root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(core): preserve required empty JSON during migration (#2099)

Browse Source 
## Summary
- preserve empty and whitespace-only legacy SQLite text as JSON string
scalars when the PostgreSQL target is required `jsonb` without a default
- keep nullable/defaulted JSON behavior unchanged
- canonicalize converted JSON before source/target checksum comparison
- cover empty, whitespace, malformed, and scalar workflow IR values

## Test plan
- `FUSION_PG_TEST_URL_BASE=postgresql://127.0.0.1:55432 nix shell
nixpkgs#postgresql_15 -c bash -c 'corepack pnpm --filter @fusion/core
exec vitest run src/__tests__/postgres/sqlite-migrator.test.ts -t
"preserves empty, whitespace, malformed, and scalar values"
--reporter=dot'`\n- `corepack pnpm --filter @fusion/core typecheck`\n-
`corepack pnpm check:changesets --strict`\n- `corepack pnpm --filter
@runfusion/fusion build`

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **Bug Fixes**
- Improved SQLite-to-PostgreSQL migrations for required `jsonb` fields.
- Preserves empty, whitespace-only, malformed, and scalar JSON values
instead of replacing them with defaults or `NULL`.
  - Maintains existing `nullable` and default-value behavior.
  - Improved migration verification for converted `jsonb` data.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Phil Larson
2026-07-14 13:04:40 -07:00
committed by GitHub
parent 945d629e3b
commit 379d450c38
3 changed files with 134 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.changeset/pg-migrator-required-empty-jsonb.md Normal file
View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Preserve legacy empty JSON text during PostgreSQL cutover.
category: fix
dev: Required jsonb columns without defaults now retain empty, whitespace-only, malformed, and scalar SQLite values without weakening nullable/default handling.

74
packages/core/src/__tests__/postgres/sqlite-migrator.test.ts
View File

@@ -149,6 +149,20 @@ CREATE TABLE IF NOT EXISTS researchRuns (
);
`;
/** Legacy workflow rows could persist an empty IR before write validation tightened. */
const WORKFLOWS_SQLITE_DDL = `
CREATE TABLE IF NOT EXISTS workflows (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
ir TEXT NOT NULL,
layout TEXT NOT NULL DEFAULT '{}',
kind TEXT NOT NULL DEFAULT 'workflow',
createdAt TEXT NOT NULL,
updatedAt TEXT NOT NULL
);
`;
/**
* A minimal agents table so agent_heartbeats has a parent row to satisfy the
* FK constraint that is re-enabled after the migration completes. Includes
@@ -183,6 +197,7 @@ function buildPopulatedSqliteProject(fusionDir: string): void {
db.exec(AGENTS_SQLITE_DDL);
db.exec(ACTIVITY_LOG_SQLITE_DDL);
db.exec(RESEARCH_RUNS_SQLITE_DDL);
db.exec(WORKFLOWS_SQLITE_DDL);
// Legacy camelCase table rows — must land in project.activity_log.
const insertActivity = db.prepare(
@@ -205,6 +220,29 @@ function buildPopulatedSqliteProject(fusionDir: string): void {
"2026-06-01T00:01:00Z",
);
const insertWorkflow = db.prepare(
`INSERT INTO workflows (id, name, description, ir, layout, kind, createdAt, updatedAt)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
);
const workflowRows = [
["WF-legacy-empty-ir", "Legacy empty IR", ""],
["WF-legacy-whitespace-ir", "Legacy whitespace IR", " \t "],
["WF-legacy-malformed-ir", "Legacy malformed IR", "not-json"],
["WF-legacy-scalar-ir", "Legacy scalar IR", "42"],
] as const;
for (const [id, name, ir] of workflowRows) {
insertWorkflow.run(
id,
name,
"",
ir,
"{}",
"workflow",
"2026-06-01T00:00:00Z",
"2026-06-01T00:01:00Z",
);
}
// Insert agents so agent_heartbeats FK is satisfiable post-migration.
const insertAgent = db.prepare(`INSERT INTO agents (id, name, role, state, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?)`);
insertAgent.run("agent-1", "Agent One", "coder", "idle", "2026-06-01T00:00:00Z", "2026-06-01T00:00:00Z");
@@ -1043,6 +1081,7 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => {
expect(rows.map((r) => r.id)).toEqual(["act-1", "act-2"]);
expect(rows[0].task_id).toBe("FN-100");
expect(rows[0].metadata).toEqual({ source: "test" });
expect(rows[1].metadata).toBeNull();
});
// FNXC:PostgresMigration 2026-06-26-16:00 (fix migration-review P1 #14):
@@ -1144,6 +1183,41 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => {
expect(rows[0]).toEqual({ sources: [], events: [], tags: [] });
});
it("preserves empty, whitespace, malformed, and scalar values in required jsonb without a default", async () => {
const report = await migrateTest(ctx!.db, [
{ sqlitePath: join(ctx!.fusionDir, "fusion.db"), pgSchema: "project" as const },
]);
expect(report.tables.find((table) => table.table === "workflows")?.verified).toBe(true);
const rows = (await ctx!.db.execute(sql`
SELECT id, ir FROM project.workflows WHERE id LIKE 'WF-legacy-%' ORDER BY id
`)) as unknown as Array<{ id: string; ir: unknown }>;
expect(Object.fromEntries(rows.map(({ id, ir }) => [id, ir]))).toEqual({
"WF-legacy-empty-ir": "",
"WF-legacy-malformed-ir": "not-json",
"WF-legacy-scalar-ir": 42,
"WF-legacy-whitespace-ir": " \t ",
});
});
it("fails closed when a required jsonb default is declared but cannot be validated", async () => {
/*
FNXC:PostgresMigration 2026-07-14-10:43:
Function-style jsonb defaults are valid PostgreSQL expressions but are intentionally outside the migrator's literal fallback parser. Empty legacy text must not be stored as data when such a default exists.
*/
await applySchemaBaseline(ctx!.db);
await ctx!.db.execute(sql`
ALTER TABLE project.workflows
ALTER COLUMN ir SET DEFAULT jsonb_build_object()
`);
await expect(migrateTest(
ctx!.db,
[{ sqlitePath: join(ctx!.fusionDir, "fusion.db"), pgSchema: "project" as const }],
{ skipBaseline: true },
)).rejects.toThrow(/declared default could not be validated/);
});
// VAL-MIGRATE-003 — bytea fidelity
it("round-trips bytea columns (BLOB → bytea) byte-identical", async () => {
await migrateTest(ctx!.db, [

63
packages/core/src/postgres/sqlite-migrator.ts
View File

@@ -116,6 +116,8 @@ interface ColumnMapping {
readonly type: ColumnType;
/** JSON text to use when a legacy NULL targets a NOT NULL jsonb default. */
readonly nullJsonbFallback?: string;
/** Preserve empty/whitespace source text only when required jsonb declares no default. */
readonly preserveEmptyJsonbString: boolean;
}
/** A table to migrate. */
@@ -755,13 +757,14 @@ function resolveColumnMapping(
continue;
}
const type = classifyColumnType(pgCol);
const hasJsonbDefault = type === "jsonb" && pgCol.column_default !== null;
let nullJsonbFallback: string | undefined;
if (type === "jsonb" && pgCol.is_nullable === "NO" && pgCol.column_default) {
if (type === "jsonb" && pgCol.is_nullable === "NO" && hasJsonbDefault) {
// FNXC:PostgresMigration 2026-07-14-05:30:
// Legacy SQLite rows can contain NULL/empty JSON even when the target is
// NOT NULL with a default. Materialize that default during conversion so
// one stale row cannot abort the entire first-boot migration.
const match = /^'(.*)'::jsonb?$/s.exec(pgCol.column_default);
const match = /^'(.*)'::jsonb?$/s.exec(pgCol.column_default!);
if (match) {
const candidate = match[1].replace(/''/g, "'");
try {
@@ -771,8 +774,19 @@ function resolveColumnMapping(
// Leave malformed defaults to PostgreSQL rather than inventing data.
}
}
if (nullJsonbFallback === undefined) {
/*
FNXC:PostgresMigration 2026-07-14-10:43:
A required jsonb column with a declared but unvalidated default is not equivalent to a default-free column. Fail the cutover closed instead of converting empty legacy text into a JSON string that silently overrides the target's intended default.
*/
throw new Error(
`Cannot migrate required jsonb column ${pgTable}.${pgName}: declared default could not be validated`,
);
}
}
mapping.push({ sqliteName: sc.name, pgName, type, nullJsonbFallback });
const preserveEmptyJsonbString =
type === "jsonb" && pgCol.is_nullable === "NO" && !hasJsonbDefault;
mapping.push({ sqliteName: sc.name, pgName, type, nullJsonbFallback, preserveEmptyJsonbString });
}
return {
@@ -817,8 +831,10 @@ function classifyColumnType(pgCol: {
* jsonb columns (it tries to send the object as a byte string and fails), so
* jsonb values MUST be passed as strings with an explicit `::jsonb` cast.
* NULL stays NULL unless the target is NOT NULL with a valid jsonb default;
* in that case legacy NULL/empty-string values materialize the target default
* so the migration does not violate the target constraint.
* in that case legacy NULL values materialize the target default. Empty
* strings use that same default, or remain a JSON string scalar only when
* the required target declares no default. Declared defaults that cannot be
* validated fail the migration before conversion rather than becoming data.
* - bytea: SQLite stores BLOB. We wrap it in a Buffer (postgres.js handles
* Buffer natively for bytea). NULL stays NULL.
* - plain: passed through verbatim.
@@ -826,7 +842,12 @@ function classifyColumnType(pgCol: {
* Identity and generated columns are omitted at the insert-builder level
* (never passed here).
*/
function convertValue(value: unknown, type: ColumnType, nullJsonbFallback?: string): unknown {
function convertValue(
value: unknown,
type: ColumnType,
nullJsonbFallback?: string,
preserveEmptyJsonbString = false,
): unknown {
if (value === null || value === undefined) {
return type === "jsonb" && nullJsonbFallback !== undefined ? nullJsonbFallback : null;
}
@@ -839,7 +860,7 @@ function convertValue(value: unknown, type: ColumnType, nullJsonbFallback?: stri
if (typeof value === "string") {
const trimmed = value.trim();
if (trimmed === "") {
return nullJsonbFallback ?? null;
return nullJsonbFallback ?? (preserveEmptyJsonbString ? JSON.stringify(value) : null);
}
try {
return JSON.stringify(JSON.parse(trimmed));
@@ -1134,7 +1155,12 @@ async function migrateTable(
for (const row of stmt.all() as Array<Record<string, unknown>>) {
const converted: Record<string, unknown> = {};
for (const col of insertableCols) {
converted[col.pgName] = convertValue(row[col.sqliteName], col.type, col.nullJsonbFallback);
converted[col.pgName] = convertValue(
row[col.sqliteName],
col.type,
col.nullJsonbFallback,
col.preserveEmptyJsonbString,
);
}
batch.push(converted);
if (batch.length >= INSERT_BATCH_SIZE) {
@@ -1507,6 +1533,18 @@ function stableJsonStringify(value: unknown): string {
.join(",")}}`;
}
/** Canonicalize a converted source value as PostgreSQL will return it. */
function canonicalizeConvertedCell(value: unknown, type: ColumnType): string {
if (type === "jsonb" && typeof value === "string") {
try {
return canonicalizeCell(JSON.parse(value));
} catch {
// Defensive fallback: convertValue normally guarantees valid JSON text.
}
}
return canonicalizeCell(value);
}
/**
* Compute a content checksum over the SQLite source rows for a table. Reads
* the SAME insertable columns the copy used (so unmapped/generated columns do
@@ -1539,8 +1577,13 @@ function computeSourceCanonicalRows(
for (const col of cols) {
const converted = col.pgName === "project_id" && partitionProjectId
? partitionProjectId
: convertValue(row[col.sqliteName], col.type, col.nullJsonbFallback);
canonical += `${canonicalizeCell(converted)}\u0001`;
: convertValue(
row[col.sqliteName],
col.type,
col.nullJsonbFallback,
col.preserveEmptyJsonbString,
);
canonical += `${canonicalizeConvertedCell(converted, col.type)}\u0001`;
}
return canonical;
}).sort();