fix(core): preserve required empty JSON during migration (#2099)

## Summary
- preserve empty and whitespace-only legacy SQLite text as JSON string
scalars when the PostgreSQL target is required `jsonb` without a default
- keep nullable/defaulted JSON behavior unchanged
- canonicalize converted JSON before source/target checksum comparison
- cover empty, whitespace, malformed, and scalar workflow IR values

## Test plan
- `FUSION_PG_TEST_URL_BASE=postgresql://127.0.0.1:55432 nix shell
nixpkgs#postgresql_15 -c bash -c 'corepack pnpm --filter @fusion/core
exec vitest run src/__tests__/postgres/sqlite-migrator.test.ts -t
"preserves empty, whitespace, malformed, and scalar values"
--reporter=dot'`\n- `corepack pnpm --filter @fusion/core typecheck`\n-
`corepack pnpm check:changesets --strict`\n- `corepack pnpm --filter
@runfusion/fusion build`

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **Bug Fixes**
- Improved SQLite-to-PostgreSQL migrations for required `jsonb` fields.
- Preserves empty, whitespace-only, malformed, and scalar JSON values
instead of replacing them with defaults or `NULL`.
  - Maintains existing `nullable` and default-value behavior.
  - Improved migration verification for converted `jsonb` data.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.changeset/pg-migrator-required-empty-jsonb.md

@@ -0,0 +1,7 @@
+---
+"@runfusion/fusion": patch
+---
+
+summary: Preserve legacy empty JSON text during PostgreSQL cutover.
+category: fix
+dev: Required jsonb columns without defaults now retain empty, whitespace-only, malformed, and scalar SQLite values without weakening nullable/default handling.

packages/core/src/__tests__/postgres/sqlite-migrator.test.ts

@@ -149,6 +149,20 @@ CREATE TABLE IF NOT EXISTS researchRuns (
 );
 `;
 
+/** Legacy workflow rows could persist an empty IR before write validation tightened. */
+const WORKFLOWS_SQLITE_DDL = `
+CREATE TABLE IF NOT EXISTS workflows (
+  id TEXT PRIMARY KEY,
+  name TEXT NOT NULL,
+  description TEXT NOT NULL DEFAULT '',
+  ir TEXT NOT NULL,
+  layout TEXT NOT NULL DEFAULT '{}',
+  kind TEXT NOT NULL DEFAULT 'workflow',
+  createdAt TEXT NOT NULL,
+  updatedAt TEXT NOT NULL
+);
+`;
+
 /**
  * A minimal agents table so agent_heartbeats has a parent row to satisfy the
  * FK constraint that is re-enabled after the migration completes. Includes
@@ -183,6 +197,7 @@ function buildPopulatedSqliteProject(fusionDir: string): void {
     db.exec(AGENTS_SQLITE_DDL);
     db.exec(ACTIVITY_LOG_SQLITE_DDL);
     db.exec(RESEARCH_RUNS_SQLITE_DDL);
+    db.exec(WORKFLOWS_SQLITE_DDL);
 
     // Legacy camelCase table rows — must land in project.activity_log.
     const insertActivity = db.prepare(
@@ -205,6 +220,29 @@ function buildPopulatedSqliteProject(fusionDir: string): void {
       "2026-06-01T00:01:00Z",
     );
 
+    const insertWorkflow = db.prepare(
+      `INSERT INTO workflows (id, name, description, ir, layout, kind, createdAt, updatedAt)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+    );
+    const workflowRows = [
+      ["WF-legacy-empty-ir", "Legacy empty IR", ""],
+      ["WF-legacy-whitespace-ir", "Legacy whitespace IR", " \t "],
+      ["WF-legacy-malformed-ir", "Legacy malformed IR", "not-json"],
+      ["WF-legacy-scalar-ir", "Legacy scalar IR", "42"],
+    ] as const;
+    for (const [id, name, ir] of workflowRows) {
+      insertWorkflow.run(
+        id,
+        name,
+        "",
+        ir,
+        "{}",
+        "workflow",
+        "2026-06-01T00:00:00Z",
+        "2026-06-01T00:01:00Z",
+      );
+    }
+
     // Insert agents so agent_heartbeats FK is satisfiable post-migration.
     const insertAgent = db.prepare(`INSERT INTO agents (id, name, role, state, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?)`);
     insertAgent.run("agent-1", "Agent One", "coder", "idle", "2026-06-01T00:00:00Z", "2026-06-01T00:00:00Z");
@@ -1043,6 +1081,7 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => {
     expect(rows.map((r) => r.id)).toEqual(["act-1", "act-2"]);
     expect(rows[0].task_id).toBe("FN-100");
     expect(rows[0].metadata).toEqual({ source: "test" });
+    expect(rows[1].metadata).toBeNull();
   });
 
   // FNXC:PostgresMigration 2026-06-26-16:00 (fix migration-review P1 #14):
@@ -1144,6 +1183,41 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => {
     expect(rows[0]).toEqual({ sources: [], events: [], tags: [] });
   });
 
+  it("preserves empty, whitespace, malformed, and scalar values in required jsonb without a default", async () => {
+    const report = await migrateTest(ctx!.db, [
+      { sqlitePath: join(ctx!.fusionDir, "fusion.db"), pgSchema: "project" as const },
+    ]);
+
+    expect(report.tables.find((table) => table.table === "workflows")?.verified).toBe(true);
+    const rows = (await ctx!.db.execute(sql`
+      SELECT id, ir FROM project.workflows WHERE id LIKE 'WF-legacy-%' ORDER BY id
+    `)) as unknown as Array<{ id: string; ir: unknown }>;
+    expect(Object.fromEntries(rows.map(({ id, ir }) => [id, ir]))).toEqual({
+      "WF-legacy-empty-ir": "",
+      "WF-legacy-malformed-ir": "not-json",
+      "WF-legacy-scalar-ir": 42,
+      "WF-legacy-whitespace-ir": " \t ",
+    });
+  });
+
+  it("fails closed when a required jsonb default is declared but cannot be validated", async () => {
+    /*
+    FNXC:PostgresMigration 2026-07-14-10:43:
+    Function-style jsonb defaults are valid PostgreSQL expressions but are intentionally outside the migrator's literal fallback parser. Empty legacy text must not be stored as data when such a default exists.
+    */
+    await applySchemaBaseline(ctx!.db);
+    await ctx!.db.execute(sql`
+      ALTER TABLE project.workflows
+      ALTER COLUMN ir SET DEFAULT jsonb_build_object()
+    `);
+
+    await expect(migrateTest(
+      ctx!.db,
+      [{ sqlitePath: join(ctx!.fusionDir, "fusion.db"), pgSchema: "project" as const }],
+      { skipBaseline: true },
+    )).rejects.toThrow(/declared default could not be validated/);
+  });
+
   // VAL-MIGRATE-003 — bytea fidelity
   it("round-trips bytea columns (BLOB → bytea) byte-identical", async () => {
     await migrateTest(ctx!.db, [

packages/core/src/postgres/sqlite-migrator.ts

@@ -116,6 +116,8 @@ interface ColumnMapping {
   readonly type: ColumnType;
   /** JSON text to use when a legacy NULL targets a NOT NULL jsonb default. */
   readonly nullJsonbFallback?: string;
+  /** Preserve empty/whitespace source text only when required jsonb declares no default. */
+  readonly preserveEmptyJsonbString: boolean;
 }
 
 /** A table to migrate. */
@@ -755,13 +757,14 @@ function resolveColumnMapping(
       continue;
     }
     const type = classifyColumnType(pgCol);
+    const hasJsonbDefault = type === "jsonb" && pgCol.column_default !== null;
     let nullJsonbFallback: string | undefined;
-    if (type === "jsonb" && pgCol.is_nullable === "NO" && pgCol.column_default) {
+    if (type === "jsonb" && pgCol.is_nullable === "NO" && hasJsonbDefault) {
       // FNXC:PostgresMigration 2026-07-14-05:30:
       // Legacy SQLite rows can contain NULL/empty JSON even when the target is
       // NOT NULL with a default. Materialize that default during conversion so
       // one stale row cannot abort the entire first-boot migration.
-      const match = /^'(.*)'::jsonb?$/s.exec(pgCol.column_default);
+      const match = /^'(.*)'::jsonb?$/s.exec(pgCol.column_default!);
       if (match) {
         const candidate = match[1].replace(/''/g, "'");
         try {
@@ -771,8 +774,19 @@ function resolveColumnMapping(
           // Leave malformed defaults to PostgreSQL rather than inventing data.
         }
       }
+      if (nullJsonbFallback === undefined) {
+        /*
+        FNXC:PostgresMigration 2026-07-14-10:43:
+        A required jsonb column with a declared but unvalidated default is not equivalent to a default-free column. Fail the cutover closed instead of converting empty legacy text into a JSON string that silently overrides the target's intended default.
+        */
+        throw new Error(
+          `Cannot migrate required jsonb column ${pgTable}.${pgName}: declared default could not be validated`,
+        );
+      }
     }
-    mapping.push({ sqliteName: sc.name, pgName, type, nullJsonbFallback });
+    const preserveEmptyJsonbString =
+      type === "jsonb" && pgCol.is_nullable === "NO" && !hasJsonbDefault;
+    mapping.push({ sqliteName: sc.name, pgName, type, nullJsonbFallback, preserveEmptyJsonbString });
   }
 
   return {
@@ -817,8 +831,10 @@ function classifyColumnType(pgCol: {
  *   jsonb columns (it tries to send the object as a byte string and fails), so
  *   jsonb values MUST be passed as strings with an explicit `::jsonb` cast.
  *   NULL stays NULL unless the target is NOT NULL with a valid jsonb default;
- *   in that case legacy NULL/empty-string values materialize the target default
+ *   in that case legacy NULL values materialize the target default. Empty
- *   so the migration does not violate the target constraint.
+ *   strings use that same default, or remain a JSON string scalar only when
+ *   the required target declares no default. Declared defaults that cannot be
+ *   validated fail the migration before conversion rather than becoming data.
  * - bytea: SQLite stores BLOB. We wrap it in a Buffer (postgres.js handles
  *   Buffer natively for bytea). NULL stays NULL.
  * - plain: passed through verbatim.
@@ -826,7 +842,12 @@ function classifyColumnType(pgCol: {
  * Identity and generated columns are omitted at the insert-builder level
  * (never passed here).
  */
-function convertValue(value: unknown, type: ColumnType, nullJsonbFallback?: string): unknown {
+function convertValue(
+  value: unknown,
+  type: ColumnType,
+  nullJsonbFallback?: string,
+  preserveEmptyJsonbString = false,
+): unknown {
   if (value === null || value === undefined) {
     return type === "jsonb" && nullJsonbFallback !== undefined ? nullJsonbFallback : null;
   }
@@ -839,7 +860,7 @@ function convertValue(value: unknown, type: ColumnType, nullJsonbFallback?: stri
       if (typeof value === "string") {
         const trimmed = value.trim();
         if (trimmed === "") {
-          return nullJsonbFallback ?? null;
+          return nullJsonbFallback ?? (preserveEmptyJsonbString ? JSON.stringify(value) : null);
         }
         try {
           return JSON.stringify(JSON.parse(trimmed));
@@ -1134,7 +1155,12 @@ async function migrateTable(
     for (const row of stmt.all() as Array<Record<string, unknown>>) {
       const converted: Record<string, unknown> = {};
       for (const col of insertableCols) {
-        converted[col.pgName] = convertValue(row[col.sqliteName], col.type, col.nullJsonbFallback);
+        converted[col.pgName] = convertValue(
+          row[col.sqliteName],
+          col.type,
+          col.nullJsonbFallback,
+          col.preserveEmptyJsonbString,
+        );
       }
       batch.push(converted);
       if (batch.length >= INSERT_BATCH_SIZE) {
@@ -1507,6 +1533,18 @@ function stableJsonStringify(value: unknown): string {
     .join(",")}}`;
 }
 
+/** Canonicalize a converted source value as PostgreSQL will return it. */
+function canonicalizeConvertedCell(value: unknown, type: ColumnType): string {
+  if (type === "jsonb" && typeof value === "string") {
+    try {
+      return canonicalizeCell(JSON.parse(value));
+    } catch {
+      // Defensive fallback: convertValue normally guarantees valid JSON text.
+    }
+  }
+  return canonicalizeCell(value);
+}
+
 /**
  * Compute a content checksum over the SQLite source rows for a table. Reads
  * the SAME insertable columns the copy used (so unmapped/generated columns do
@@ -1539,8 +1577,13 @@ function computeSourceCanonicalRows(
     for (const col of cols) {
       const converted = col.pgName === "project_id" && partitionProjectId
         ? partitionProjectId
-        : convertValue(row[col.sqliteName], col.type, col.nullJsonbFallback);
+        : convertValue(
-      canonical += `${canonicalizeCell(converted)}\u0001`;
+            row[col.sqliteName],
+            col.type,
+            col.nullJsonbFallback,
+            col.preserveEmptyJsonbString,
+          );
+      canonical += `${canonicalizeConvertedCell(converted, col.type)}\u0001`;
     }
     return canonical;
   }).sort();