diff --git a/.changeset/pg-migrator-required-empty-jsonb.md b/.changeset/pg-migrator-required-empty-jsonb.md new file mode 100644 index 0000000000..c44b8f9b48 --- /dev/null +++ b/.changeset/pg-migrator-required-empty-jsonb.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Preserve legacy empty JSON text during PostgreSQL cutover. +category: fix +dev: Required jsonb columns without defaults now retain empty, whitespace-only, malformed, and scalar SQLite values without weakening nullable/default handling. diff --git a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts index 23f1488b6b..b4c083c5b3 100644 --- a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts +++ b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts @@ -149,6 +149,20 @@ CREATE TABLE IF NOT EXISTS researchRuns ( ); `; +/** Legacy workflow rows could persist an empty IR before write validation tightened. */ +const WORKFLOWS_SQLITE_DDL = ` +CREATE TABLE IF NOT EXISTS workflows ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + ir TEXT NOT NULL, + layout TEXT NOT NULL DEFAULT '{}', + kind TEXT NOT NULL DEFAULT 'workflow', + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL +); +`; + /** * A minimal agents table so agent_heartbeats has a parent row to satisfy the * FK constraint that is re-enabled after the migration completes. Includes @@ -183,6 +197,7 @@ function buildPopulatedSqliteProject(fusionDir: string): void { db.exec(AGENTS_SQLITE_DDL); db.exec(ACTIVITY_LOG_SQLITE_DDL); db.exec(RESEARCH_RUNS_SQLITE_DDL); + db.exec(WORKFLOWS_SQLITE_DDL); // Legacy camelCase table rows — must land in project.activity_log. const insertActivity = db.prepare( @@ -205,6 +220,29 @@ function buildPopulatedSqliteProject(fusionDir: string): void { "2026-06-01T00:01:00Z", ); + const insertWorkflow = db.prepare( + `INSERT INTO workflows (id, name, description, ir, layout, kind, createdAt, updatedAt) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + ); + const workflowRows = [ + ["WF-legacy-empty-ir", "Legacy empty IR", ""], + ["WF-legacy-whitespace-ir", "Legacy whitespace IR", " \t "], + ["WF-legacy-malformed-ir", "Legacy malformed IR", "not-json"], + ["WF-legacy-scalar-ir", "Legacy scalar IR", "42"], + ] as const; + for (const [id, name, ir] of workflowRows) { + insertWorkflow.run( + id, + name, + "", + ir, + "{}", + "workflow", + "2026-06-01T00:00:00Z", + "2026-06-01T00:01:00Z", + ); + } + // Insert agents so agent_heartbeats FK is satisfiable post-migration. const insertAgent = db.prepare(`INSERT INTO agents (id, name, role, state, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?)`); insertAgent.run("agent-1", "Agent One", "coder", "idle", "2026-06-01T00:00:00Z", "2026-06-01T00:00:00Z"); @@ -1043,6 +1081,7 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { expect(rows.map((r) => r.id)).toEqual(["act-1", "act-2"]); expect(rows[0].task_id).toBe("FN-100"); expect(rows[0].metadata).toEqual({ source: "test" }); + expect(rows[1].metadata).toBeNull(); }); // FNXC:PostgresMigration 2026-06-26-16:00 (fix migration-review P1 #14): @@ -1144,6 +1183,41 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { expect(rows[0]).toEqual({ sources: [], events: [], tags: [] }); }); + it("preserves empty, whitespace, malformed, and scalar values in required jsonb without a default", async () => { + const report = await migrateTest(ctx!.db, [ + { sqlitePath: join(ctx!.fusionDir, "fusion.db"), pgSchema: "project" as const }, + ]); + + expect(report.tables.find((table) => table.table === "workflows")?.verified).toBe(true); + const rows = (await ctx!.db.execute(sql` + SELECT id, ir FROM project.workflows WHERE id LIKE 'WF-legacy-%' ORDER BY id + `)) as unknown as Array<{ id: string; ir: unknown }>; + expect(Object.fromEntries(rows.map(({ id, ir }) => [id, ir]))).toEqual({ + "WF-legacy-empty-ir": "", + "WF-legacy-malformed-ir": "not-json", + "WF-legacy-scalar-ir": 42, + "WF-legacy-whitespace-ir": " \t ", + }); + }); + + it("fails closed when a required jsonb default is declared but cannot be validated", async () => { + /* + FNXC:PostgresMigration 2026-07-14-10:43: + Function-style jsonb defaults are valid PostgreSQL expressions but are intentionally outside the migrator's literal fallback parser. Empty legacy text must not be stored as data when such a default exists. + */ + await applySchemaBaseline(ctx!.db); + await ctx!.db.execute(sql` + ALTER TABLE project.workflows + ALTER COLUMN ir SET DEFAULT jsonb_build_object() + `); + + await expect(migrateTest( + ctx!.db, + [{ sqlitePath: join(ctx!.fusionDir, "fusion.db"), pgSchema: "project" as const }], + { skipBaseline: true }, + )).rejects.toThrow(/declared default could not be validated/); + }); + // VAL-MIGRATE-003 — bytea fidelity it("round-trips bytea columns (BLOB → bytea) byte-identical", async () => { await migrateTest(ctx!.db, [ diff --git a/packages/core/src/postgres/sqlite-migrator.ts b/packages/core/src/postgres/sqlite-migrator.ts index 90670f979a..7c234857d7 100644 --- a/packages/core/src/postgres/sqlite-migrator.ts +++ b/packages/core/src/postgres/sqlite-migrator.ts @@ -116,6 +116,8 @@ interface ColumnMapping { readonly type: ColumnType; /** JSON text to use when a legacy NULL targets a NOT NULL jsonb default. */ readonly nullJsonbFallback?: string; + /** Preserve empty/whitespace source text only when required jsonb declares no default. */ + readonly preserveEmptyJsonbString: boolean; } /** A table to migrate. */ @@ -755,13 +757,14 @@ function resolveColumnMapping( continue; } const type = classifyColumnType(pgCol); + const hasJsonbDefault = type === "jsonb" && pgCol.column_default !== null; let nullJsonbFallback: string | undefined; - if (type === "jsonb" && pgCol.is_nullable === "NO" && pgCol.column_default) { + if (type === "jsonb" && pgCol.is_nullable === "NO" && hasJsonbDefault) { // FNXC:PostgresMigration 2026-07-14-05:30: // Legacy SQLite rows can contain NULL/empty JSON even when the target is // NOT NULL with a default. Materialize that default during conversion so // one stale row cannot abort the entire first-boot migration. - const match = /^'(.*)'::jsonb?$/s.exec(pgCol.column_default); + const match = /^'(.*)'::jsonb?$/s.exec(pgCol.column_default!); if (match) { const candidate = match[1].replace(/''/g, "'"); try { @@ -771,8 +774,19 @@ function resolveColumnMapping( // Leave malformed defaults to PostgreSQL rather than inventing data. } } + if (nullJsonbFallback === undefined) { + /* + FNXC:PostgresMigration 2026-07-14-10:43: + A required jsonb column with a declared but unvalidated default is not equivalent to a default-free column. Fail the cutover closed instead of converting empty legacy text into a JSON string that silently overrides the target's intended default. + */ + throw new Error( + `Cannot migrate required jsonb column ${pgTable}.${pgName}: declared default could not be validated`, + ); + } } - mapping.push({ sqliteName: sc.name, pgName, type, nullJsonbFallback }); + const preserveEmptyJsonbString = + type === "jsonb" && pgCol.is_nullable === "NO" && !hasJsonbDefault; + mapping.push({ sqliteName: sc.name, pgName, type, nullJsonbFallback, preserveEmptyJsonbString }); } return { @@ -817,8 +831,10 @@ function classifyColumnType(pgCol: { * jsonb columns (it tries to send the object as a byte string and fails), so * jsonb values MUST be passed as strings with an explicit `::jsonb` cast. * NULL stays NULL unless the target is NOT NULL with a valid jsonb default; - * in that case legacy NULL/empty-string values materialize the target default - * so the migration does not violate the target constraint. + * in that case legacy NULL values materialize the target default. Empty + * strings use that same default, or remain a JSON string scalar only when + * the required target declares no default. Declared defaults that cannot be + * validated fail the migration before conversion rather than becoming data. * - bytea: SQLite stores BLOB. We wrap it in a Buffer (postgres.js handles * Buffer natively for bytea). NULL stays NULL. * - plain: passed through verbatim. @@ -826,7 +842,12 @@ function classifyColumnType(pgCol: { * Identity and generated columns are omitted at the insert-builder level * (never passed here). */ -function convertValue(value: unknown, type: ColumnType, nullJsonbFallback?: string): unknown { +function convertValue( + value: unknown, + type: ColumnType, + nullJsonbFallback?: string, + preserveEmptyJsonbString = false, +): unknown { if (value === null || value === undefined) { return type === "jsonb" && nullJsonbFallback !== undefined ? nullJsonbFallback : null; } @@ -839,7 +860,7 @@ function convertValue(value: unknown, type: ColumnType, nullJsonbFallback?: stri if (typeof value === "string") { const trimmed = value.trim(); if (trimmed === "") { - return nullJsonbFallback ?? null; + return nullJsonbFallback ?? (preserveEmptyJsonbString ? JSON.stringify(value) : null); } try { return JSON.stringify(JSON.parse(trimmed)); @@ -1134,7 +1155,12 @@ async function migrateTable( for (const row of stmt.all() as Array>) { const converted: Record = {}; for (const col of insertableCols) { - converted[col.pgName] = convertValue(row[col.sqliteName], col.type, col.nullJsonbFallback); + converted[col.pgName] = convertValue( + row[col.sqliteName], + col.type, + col.nullJsonbFallback, + col.preserveEmptyJsonbString, + ); } batch.push(converted); if (batch.length >= INSERT_BATCH_SIZE) { @@ -1507,6 +1533,18 @@ function stableJsonStringify(value: unknown): string { .join(",")}}`; } +/** Canonicalize a converted source value as PostgreSQL will return it. */ +function canonicalizeConvertedCell(value: unknown, type: ColumnType): string { + if (type === "jsonb" && typeof value === "string") { + try { + return canonicalizeCell(JSON.parse(value)); + } catch { + // Defensive fallback: convertValue normally guarantees valid JSON text. + } + } + return canonicalizeCell(value); +} + /** * Compute a content checksum over the SQLite source rows for a table. Reads * the SAME insertable columns the copy used (so unmapped/generated columns do @@ -1539,8 +1577,13 @@ function computeSourceCanonicalRows( for (const col of cols) { const converted = col.pgName === "project_id" && partitionProjectId ? partitionProjectId - : convertValue(row[col.sqliteName], col.type, col.nullJsonbFallback); - canonical += `${canonicalizeCell(converted)}\u0001`; + : convertValue( + row[col.sqliteName], + col.type, + col.nullJsonbFallback, + col.preserveEmptyJsonbString, + ); + canonical += `${canonicalizeConvertedCell(converted, col.type)}\u0001`; } return canonical; }).sort();