From 48b0d04322a1b9357df81d741cc8c99aad5ee648 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Fri, 17 Jul 2026 20:04:22 -0700 Subject: [PATCH] fix(core): sanitize NUL (u0000) characters in SQLite-to-PostgreSQL migration Legacy SQLite databases can hold U+0000 in TEXT cells and inside stored JSON, which PostgreSQL rejects in text and jsonb columns and which aborted the first-boot auto-migration. Strip NUL from plain text cells, JSON string values and object keys, malformed-JSON scalars, and opaque legacy-preservation cells; content-checksum verification compares the sanitized source against the sanitized target so migrations still verify. Co-Authored-By: Claude Fable 5 --- .changeset/sqlite-migration-nul-sanitize.md | 7 ++ .../postgres/sqlite-migrator.test.ts | 95 +++++++++++++++++++ packages/core/src/postgres/sqlite-migrator.ts | 42 +++++++- 3 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 .changeset/sqlite-migration-nul-sanitize.md diff --git a/.changeset/sqlite-migration-nul-sanitize.md b/.changeset/sqlite-migration-nul-sanitize.md new file mode 100644 index 0000000000..25424e1a00 --- /dev/null +++ b/.changeset/sqlite-migration-nul-sanitize.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fix first-boot SQLite→PostgreSQL migration failing on legacy data containing NUL (\u0000) characters. +category: fix +dev: The migrator now strips U+0000 from plain text cells, JSON string values/keys, malformed-JSON scalars, and opaque legacy-preservation cells before insert; content-checksum verification compares sanitized source against sanitized target. diff --git a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts index 776ca202bc..a28a7c2718 100644 --- a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts +++ b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts @@ -733,6 +733,101 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { expect(rows[0].source_schema_sql).toContain("CREATE TABLE mission_feature_evidence_links"); }); + /* + FNXC:PostgresMigrationNulSanitize 2026-07-17-10:05: + Legacy SQLite data can contain U+0000 in TEXT cells and inside stored JSON documents. PostgreSQL rejects NUL in text and jsonb columns ("unsupported Unicode escape sequence" / "\u0000 cannot be converted to text"), which aborted the first-boot auto-migration. The migrator must strip NUL from plain text cells, JSON string values and object keys, malformed-JSON scalars, and opaque legacy-preservation cells — and the content-checksum verification must still pass on the sanitized rows. + */ + it("strips U+0000 from legacy text and JSON values so the migration succeeds and verifies", async () => { + const sqlitePath = join(ctx!.fusionDir, "fusion.db"); + const legacy = new DatabaseSync(sqlitePath); + try { + const insert = legacy.prepare( + `INSERT INTO tasks (id, title, description, "column", customFields, createdAt, updatedAt) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ); + insert.run( + "FN-NUL-1", + "nul\u0000title", + "desc\u0000ription", + "todo", + JSON.stringify({ ["key\u0000nul"]: "value\u0000nul", nested: ["a\u0000b"] }), + "2026-07-17T00:00:00Z", + "2026-07-17T00:00:00Z", + ); + insert.run( + "FN-NUL-2", + "clean title", + "malformed json cell", + "todo", + "not-json\u0000tail", + "2026-07-17T00:00:00Z", + "2026-07-17T00:00:00Z", + ); + } finally { + legacy.close(); + } + + const report = await migrateTest( + ctx!.db, + [{ sqlitePath, pgSchema: "project" as const }], + ); + const tasks = report.tables.find((table) => table.table === "tasks")!; + expect(tasks.verified).toBe(true); + + const rows = await ctx!.db.execute(sql` + SELECT id, title, description, custom_fields + FROM project.tasks + WHERE id IN ('FN-NUL-1', 'FN-NUL-2') + ORDER BY id + `) as unknown as Array<{ + id: string; + title: string | null; + description: string; + custom_fields: unknown; + }>; + expect(rows).toHaveLength(2); + expect(rows[0].title).toBe("nultitle"); + expect(rows[0].description).toBe("description"); + expect(rows[0].custom_fields).toEqual({ keynul: "valuenul", nested: ["ab"] }); + // Malformed JSON is preserved as a jsonb string scalar, minus the NUL. + expect(rows[1].custom_fields).toBe("not-jsontail"); + }); + + it("strips U+0000 from opaque legacy-preservation text cells", async () => { + const sqlitePath = join(ctx!.fusionDir, "mission-nul.db"); + const legacy = new DatabaseSync(sqlitePath); + try { + legacy.exec(`CREATE TABLE mission_feature_evidence_links (id TEXT, featureId TEXT)`); + legacy.prepare(`INSERT INTO mission_feature_evidence_links VALUES (?, ?)`) + .run("nul\u0000id", "feat\u0000ure"); + } finally { + legacy.close(); + } + + const report = await migrateTest( + ctx!.db, + [{ sqlitePath, pgSchema: "project" as const }], + { projectId: "project-nul" }, + ); + expect(report.tables).toContainEqual(expect.objectContaining({ + table: "mission_feature_evidence_links", + sourceRows: 1, + verified: true, + skipped: false, + })); + + const rows = await ctx!.db.execute(sql` + SELECT legacy_row FROM project.mission_feature_evidence_links + WHERE project_id = 'project-nul' + `) as unknown as Array<{ legacy_row: Record }>; + expect(rows).toEqual([{ + legacy_row: { + featureId: { type: "text", value: "feature" }, + id: { type: "text", value: "nulid" }, + }, + }]); + }); + it("requires a project identity before preserving opaque project rows", async () => { const sqlitePath = join(ctx!.fusionDir, "mission-unbound.db"); const legacy = new DatabaseSync(sqlitePath); diff --git a/packages/core/src/postgres/sqlite-migrator.ts b/packages/core/src/postgres/sqlite-migrator.ts index 916aca0c56..c0b26ead35 100644 --- a/packages/core/src/postgres/sqlite-migrator.ts +++ b/packages/core/src/postgres/sqlite-migrator.ts @@ -1212,6 +1212,35 @@ function classifyColumnType(pgCol: { return "plain"; } +/* +FNXC:PostgresMigrationNulSanitize 2026-07-17-10:05: +PostgreSQL rejects U+0000 in text/varchar ("invalid byte sequence" / "\u0000 cannot be converted to text") and in json/jsonb ("unsupported Unicode escape sequence"), but SQLite TEXT stores it freely, so legacy databases can contain NUL bytes that abort the first-boot auto-migration. Strip U+0000 from every migrated string — plain text cells, string values and object keys inside JSON documents, and opaque legacy-preservation text cells — rather than failing the cutover. Sanitization happens inside convertValue/tagLegacyCell so the content-checksum verification (computeSourceCanonicalRows reuses convertValue) compares the sanitized source against the sanitized target and still passes. +*/ +// eslint-disable-next-line no-control-regex -- matching the NUL control character is the point +const NUL_CHAR_RE = /\u0000/g; + +function stripNulChars(text: string): string { + return text.includes("\u0000") ? text.replace(NUL_CHAR_RE, "") : text; +} + +/** Recursively strip U+0000 from all string values and object keys in a parsed JSON document. */ +function deepStripNulChars(value: unknown): unknown { + if (typeof value === "string") { + return stripNulChars(value); + } + if (Array.isArray(value)) { + return value.map(deepStripNulChars); + } + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record).map( + ([key, entry]) => [stripNulChars(key), deepStripNulChars(entry)], + ), + ); + } + return value; +} + /** * Convert a SQLite value to its PostgreSQL representation based on the column * type classification. @@ -1255,14 +1284,14 @@ function convertValue( return nullJsonbFallback ?? (preserveEmptyJsonbString ? JSON.stringify(value) : null); } try { - return JSON.stringify(JSON.parse(trimmed)); + return JSON.stringify(deepStripNulChars(JSON.parse(trimmed))); } catch { // Malformed JSON — store as a JSON-encoded string scalar (valid jsonb). - return JSON.stringify(value); + return JSON.stringify(stripNulChars(value)); } } // Already a JS value (object/array/number/boolean) — stringify it. - return JSON.stringify(value); + return JSON.stringify(deepStripNulChars(value)); } case "bytea": { if (Buffer.isBuffer(value)) { @@ -1280,7 +1309,8 @@ function convertValue( case "identity": case "generated": default: - return value; + // PostgreSQL text/varchar rejects U+0000 outright; see NUL-sanitize note above. + return typeof value === "string" ? stripNulChars(value) : value; } } @@ -1330,7 +1360,9 @@ function tagLegacyCell(value: unknown): TaggedLegacyCell { if (typeof value === "number" || typeof value === "bigint") { return { type: "number", value: Object.is(value, -0) ? "-0" : String(value) }; } - return { type: "text", value: String(value) }; + // Legacy-preservation rows are stored as jsonb; jsonb rejects \u0000, so + // opaque text cells get the same NUL sanitization as regular columns. + return { type: "text", value: stripNulChars(String(value)) }; } function canonicalizeLegacyRows(