From fbcd00204a7a32b5c7d14422ae01cea54802a93d Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Mon, 13 Jul 2026 20:30:41 -0700 Subject: [PATCH] fix: snake_case legacy SQLite table names in the PG migrator and keep PG-mode boots from touching SQLite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two post-cutover fixes: 1. The SQLite -> PostgreSQL migrator matched table names verbatim while only column names were snake_cased, so all 22 legacy camelCase tables (activityLog, runAuditEvents, mergeQueue, taskClaims, projectNodePathMappings, ...) resolved zero PostgreSQL columns and were silently skipped as 'no PostgreSQL counterpart'. First observed as 'Project/node path mapping not found' on engine start because central.project_node_path_mappings was never populated. TablePlan now carries a snake_cased pgTable used for every PostgreSQL-side operation; regression test migrates a camelCase activityLog into project.activity_log. 2. The first-boot auto-migration guard opened .fusion/fusion.db with a read-write DatabaseSync on every boot (isValidSqliteDatabaseFile), which performs WAL recovery + checkpoint — writing the legacy file on each PG boot. The PG emptiness count now runs before the SQLite probe, so steady-state PG boots never open the legacy SQLite files at all. Co-Authored-By: Claude Fable 5 --- .changeset/pg-boot-no-sqlite-touch.md | 7 +++ .changeset/pg-migrator-camelcase-tables.md | 7 +++ .../postgres/sqlite-migrator.test.ts | 55 +++++++++++++++++++ packages/core/src/postgres/sqlite-migrator.ts | 46 +++++++++++----- packages/core/src/postgres/startup-factory.ts | 14 ++++- 5 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 .changeset/pg-boot-no-sqlite-touch.md create mode 100644 .changeset/pg-migrator-camelcase-tables.md diff --git a/.changeset/pg-boot-no-sqlite-touch.md b/.changeset/pg-boot-no-sqlite-touch.md new file mode 100644 index 0000000000..3301c2f482 --- /dev/null +++ b/.changeset/pg-boot-no-sqlite-touch.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Stop PostgreSQL-mode boots from opening and checkpointing the legacy SQLite files. +category: fix +dev: The first-boot auto-migration guard probed .fusion/fusion.db with a read-write DatabaseSync open on every boot, performing WAL recovery + checkpoint (file writes). The PostgreSQL emptiness count now runs first; the SQLite probe only runs on the empty-PG path where auto-migration is actually considered, so steady-state PG boots leave the legacy files byte-quiet. diff --git a/.changeset/pg-migrator-camelcase-tables.md b/.changeset/pg-migrator-camelcase-tables.md new file mode 100644 index 0000000000..c613641354 --- /dev/null +++ b/.changeset/pg-migrator-camelcase-tables.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fix SQLite → PostgreSQL migration silently skipping legacy camelCase tables. +category: fix +dev: The migrator snake_cased column names but matched TABLE names verbatim, so all 22 legacy camelCase SQLite tables (activityLog, runAuditEvents, mergeQueue, taskClaims, projectNodePathMappings, …) found no PostgreSQL counterpart and were silently skipped — surfacing as "Project/node path mapping not found" on engine start. TablePlan now carries a snake_cased pgTable used for all PostgreSQL-side operations. Re-run `fn db migrate` (idempotent) to top up databases migrated before this fix. diff --git a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts index 9b795ad1a1..e750ff4707 100644 --- a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts +++ b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts @@ -108,6 +108,27 @@ CREATE TABLE IF NOT EXISTS config ( ); `; +/* +FNXC:PostgresMigration 2026-07-13-20:30: +Legacy camelCase-named table. Older SQLite tables are camelCase (activityLog, +runAuditEvents, mergeQueue, projectNodePathMappings, …) while every PostgreSQL +table is snake_case. The migrator must snake_case the TABLE name too — a bug +where only column names were converted silently skipped all 22 such tables +("no PostgreSQL counterpart") and surfaced post-cutover as +`Project/node path mapping not found`. +*/ +const ACTIVITY_LOG_SQLITE_DDL = ` +CREATE TABLE IF NOT EXISTS activityLog ( + id TEXT PRIMARY KEY, + timestamp TEXT NOT NULL, + type TEXT NOT NULL, + taskId TEXT, + taskTitle TEXT, + details TEXT NOT NULL, + metadata TEXT +); +`; + /** * A minimal agents table so agent_heartbeats has a parent row to satisfy the * FK constraint that is re-enabled after the migration completes. Includes @@ -140,6 +161,14 @@ function buildPopulatedSqliteProject(fusionDir: string): void { db.exec(AGENT_HEARTBEATS_SQLITE_DDL); db.exec(CONFIG_SQLITE_DDL); db.exec(AGENTS_SQLITE_DDL); + db.exec(ACTIVITY_LOG_SQLITE_DDL); + + // Legacy camelCase table rows — must land in project.activity_log. + const insertActivity = db.prepare( + `INSERT INTO activityLog (id, timestamp, type, taskId, taskTitle, details, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ); + insertActivity.run("act-1", "2026-06-01T00:00:00Z", "task:created", "FN-100", "First task", "created", JSON.stringify({ source: "test" })); + insertActivity.run("act-2", "2026-06-01T01:00:00Z", "task:moved", "FN-100", "First task", "todo -> in-progress", null); // Insert agents so agent_heartbeats FK is satisfiable post-migration. const insertAgent = db.prepare(`INSERT INTO agents (id, name, role, state, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?)`); @@ -340,6 +369,32 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { expect(archived.targetRows).toBe(1); }); + // FNXC:PostgresMigration 2026-07-13-20:30: + // Legacy camelCase TABLE names (activityLog, runAuditEvents, mergeQueue, + // projectNodePathMappings, …) must be snake_cased when matched against + // PostgreSQL, exactly like column names. A bug that matched table names + // verbatim silently skipped all 22 legacy camelCase tables ("no PostgreSQL + // counterpart"), surfacing post-cutover as + // `Project/node path mapping not found` on engine start. + it("migrates legacy camelCase-named tables into their snake_case PostgreSQL counterparts", async () => { + const report = await migrateSqliteToPostgres(ctx!.db, [ + { sqlitePath: join(ctx!.fusionDir, "fusion.db"), pgSchema: "project" as const }, + ]); + + const activity = report.tables.find((t) => t.table === "activity_log"); + expect(activity, "activityLog must not be silently skipped").toBeDefined(); + expect(activity!.sourceRows).toBe(2); + expect(activity!.targetRows).toBe(2); + expect(activity!.verified).toBe(true); + + const rows = (await ctx!.db.execute(sql` + SELECT id, task_id, metadata FROM project.activity_log ORDER BY id + `)) as unknown as Array<{ id: string; task_id: string | null; metadata: unknown }>; + expect(rows.map((r) => r.id)).toEqual(["act-1", "act-2"]); + expect(rows[0].task_id).toBe("FN-100"); + expect(rows[0].metadata).toEqual({ source: "test" }); + }); + // FNXC:PostgresMigration 2026-06-26-16:00 (fix migration-review P1 #14): // The `data` column appears in MULTIPLE tables with DIFFERENT types: it is // `jsonb` in agents/workflow_work_items/etc but would be `text` in a diff --git a/packages/core/src/postgres/sqlite-migrator.ts b/packages/core/src/postgres/sqlite-migrator.ts index ac98bdc4d0..8617595cbf 100644 --- a/packages/core/src/postgres/sqlite-migrator.ts +++ b/packages/core/src/postgres/sqlite-migrator.ts @@ -119,8 +119,20 @@ interface ColumnMapping { /** A table to migrate. */ interface TablePlan { readonly pgSchema: string; - /** The table name (identical in SQLite and PostgreSQL). */ + /** The SQLite table name (legacy tables are camelCase, e.g. `activityLog`). */ readonly table: string; + /* + FNXC:PostgresMigration 2026-07-13-20:30: + The PostgreSQL table name (snake_case). Table names were previously assumed + identical across both engines, but legacy SQLite tables are camelCase + (activityLog, runAuditEvents, mergeQueue, taskClaims, projectNodePathMappings, + …) while every PostgreSQL table is snake_case. The old single-name plan made + resolveColumnMapping find zero PG columns for all 22 camelCase tables, and the + migrator silently skipped them as "no PostgreSQL counterpart" — first + observed as `Project/node path mapping not found` because + central.project_node_path_mappings was never populated. + */ + readonly pgTable: string; readonly columns: readonly ColumnMapping[]; } @@ -227,7 +239,7 @@ export async function migrateSqliteToPostgres( if (!dryRun && !result.skipped && result.sourceRows > 0) { const identityCols = tablePlan.columns.filter((c) => c.type === "identity"); for (const col of identityCols) { - const bump = await bumpIdentitySequence(migrationDb, tablePlan.pgSchema, tablePlan.table, col.pgName); + const bump = await bumpIdentitySequence(migrationDb, tablePlan.pgSchema, tablePlan.pgTable, col.pgName); if (bump) { sequenceBumps.push({ schema: tablePlan.pgSchema, @@ -288,14 +300,17 @@ async function buildMigrationPlan( const tables = listSqliteTables(sqlite); const plans: TablePlan[] = []; for (const table of tables) { - const cols = await resolveColumnMapping(db, source.pgSchema, table, sqlite); + // Legacy SQLite table names are camelCase; PostgreSQL tables are + // snake_case. toSnakeCase is the identity for already-snake names. + const pgTable = toSnakeCase(table); + const cols = await resolveColumnMapping(db, source.pgSchema, pgTable, table, sqlite); if (cols.length === 0) { // Table exists in SQLite but has no mappable columns in PostgreSQL — // skip it (e.g. FTS5 shadow tables). Logged at the table-migration // step, not here. continue; } - plans.push({ pgSchema: source.pgSchema, table, columns: cols }); + plans.push({ pgSchema: source.pgSchema, table, pgTable, columns: cols }); } return plans; } finally { @@ -363,6 +378,7 @@ function listSqliteTables(db: DatabaseSync): string[] { async function resolveColumnMapping( db: PostgresJsDatabase>, pgSchema: string, + pgTable: string, table: string, sqlite: DatabaseSync, ): Promise { @@ -393,7 +409,7 @@ async function resolveColumnMapping( JOIN pg_class cls ON cls.oid = a.attrelid JOIN pg_namespace n ON n.oid = cls.relnamespace WHERE c.table_schema = ${pgSchema} - AND c.table_name = ${table} + AND c.table_name = ${pgTable} AND n.nspname = c.table_schema AND cls.relname = c.table_name AND a.attnum > 0 @@ -546,10 +562,10 @@ async function migrateTable( if (insertableCols.length === 0) { // No insertable columns (e.g. a pure-generated table). Verify the target // exists but copy nothing. - const targetRows = await countTargetRows(db, plan.pgSchema, plan.table); + const targetRows = await countTargetRows(db, plan.pgSchema, plan.pgTable); return { schema: plan.pgSchema, - table: plan.table, + table: plan.pgTable, sourceRows: 0, insertedRows: 0, targetRows, @@ -576,10 +592,10 @@ async function migrateTable( // Dry-run: report the plan without writing. return { schema: plan.pgSchema, - table: plan.table, + table: plan.pgTable, sourceRows, insertedRows: 0, - targetRows: dryRun ? 0 : await countTargetRows(db, plan.pgSchema, plan.table), + targetRows: dryRun ? 0 : await countTargetRows(db, plan.pgSchema, plan.pgTable), verified: dryRun ? false : true, skipped: dryRun ? true : false, skipReason: dryRun ? "dry-run" : "no source rows", @@ -624,7 +640,7 @@ async function migrateTable( // Both layers must pass for `verified: true`. The MD5 is computed in SQL // (md5(string_agg(...)) on PostgreSQL, and a Node-side md5 over the SQLite // converted stream) so the comparison is a single short string per side. - const targetRows = await countTargetRows(db, plan.pgSchema, plan.table); + const targetRows = await countTargetRows(db, plan.pgSchema, plan.pgTable); const rowCountOk = targetRows === sourceRows; let contentOk = true; if (rowCountOk && sourceRows > 0) { @@ -632,26 +648,26 @@ async function migrateTable( const targetChecksum = await computeTargetContentChecksum( db, plan.pgSchema, - plan.table, + plan.pgTable, insertableCols, ); contentOk = sourceChecksum === targetChecksum; if (!contentOk) { log.warn( - `Content checksum mismatch for ${plan.pgSchema}.${plan.table}: ` + + `Content checksum mismatch for ${plan.pgSchema}.${plan.pgTable}: ` + `source=${sourceChecksum}, target=${targetChecksum}`, ); } } else if (!rowCountOk) { log.warn( - `Row-count mismatch for ${plan.pgSchema}.${plan.table}: source=${sourceRows}, target=${targetRows}`, + `Row-count mismatch for ${plan.pgSchema}.${plan.pgTable}: source=${sourceRows}, target=${targetRows}`, ); } const verified = rowCountOk && contentOk; return { schema: plan.pgSchema, - table: plan.table, + table: plan.pgTable, sourceRows, insertedRows, targetRows, @@ -685,7 +701,7 @@ async function insertBatch( ): Promise { if (rows.length === 0) return 0; const colList = cols.map((c) => quoteIdent(c.pgName)).join(", "); - const schemaQualifiedTable = `${quoteIdent(plan.pgSchema)}.${quoteIdent(plan.table)}`; + const schemaQualifiedTable = `${quoteIdent(plan.pgSchema)}.${quoteIdent(plan.pgTable)}`; // OVERRIDING SYSTEM VALUE lets us write explicit values into GENERATED ALWAYS // AS IDENTITY columns so the SQLite id is preserved (VAL-MIGRATE-002/004). const overridingClause = hasIdentityCol ? " OVERRIDING SYSTEM VALUE" : ""; diff --git a/packages/core/src/postgres/startup-factory.ts b/packages/core/src/postgres/startup-factory.ts index 623909f640..d29d7ac5e9 100644 --- a/packages/core/src/postgres/startup-factory.ts +++ b/packages/core/src/postgres/startup-factory.ts @@ -374,7 +374,7 @@ export async function createTaskStoreForBackend( try { const fusionDir = join(rootDir, ".fusion"); const legacySqlitePath = join(fusionDir, "fusion.db"); - if (existsSync(legacySqlitePath) && isValidSqliteDatabaseFile(legacySqlitePath)) { + if (existsSync(legacySqlitePath)) { /* FNXC:MultiProjectIsolation 2026-07-11: With per-project task partitioning (project_id on project.tasks), the @@ -385,6 +385,16 @@ export async function createTaskStoreForBackend( project_id rows are counted as blocking: they may be this project's pre-isolation data, and migrating on top of them risks id collisions. Without a bound projectId the pre-isolation whole-table check applies. + + FNXC:PostgresCutover 2026-07-13-20:50: + Order matters: the PostgreSQL emptiness count runs BEFORE the SQLite + validity probe. isValidSqliteDatabaseFile opens the file with a + read-write DatabaseSync, and that open/close performs WAL recovery and + a checkpoint — i.e. it WRITES to the legacy fusion.db on every boot. + Post-cutover the legacy files must stay byte-quiet: steady-state boots + (PG already populated) must not open SQLite at all. The probe now runs + only on the rare empty-PG path where auto-migration is actually being + considered. */ const countRows = (await connections.migration.execute( options.projectId @@ -392,7 +402,7 @@ export async function createTaskStoreForBackend( : drizzleSql`SELECT count(*)::int AS count FROM project.tasks`, )) as Array<{ count: number }>; const pgTaskCount = Number(countRows[0]?.count ?? 0); - if (pgTaskCount === 0) { + if (pgTaskCount === 0 && isValidSqliteDatabaseFile(legacySqlitePath)) { const { migrateSqliteToPostgres, defaultMigrationSources } = await import("./sqlite-migrator.js"); // The central (global-dir) source is optional: when no global dir is // resolvable (e.g. tests without an explicit dir), migrate only the