From 678265a52650107b4fc892ef95e1132b003d2883 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Tue, 14 Jul 2026 14:05:39 -0700 Subject: [PATCH] fix(cli): show live SQLite migration progress Report source scans, per-table copy milestones, checksum phases, verification outcomes, and unambiguous failure or finalization status during first-boot and manual migrations. --- .changeset/bright-migrations-report.md | 7 + packages/cli/src/commands/db.ts | 4 + .../postgres/sqlite-migrator.test.ts | 128 ++++++++- packages/core/src/index.ts | 3 + packages/core/src/postgres/index.ts | 3 + packages/core/src/postgres/sqlite-migrator.ts | 248 +++++++++++++++++- packages/core/src/postgres/startup-factory.ts | 9 +- 7 files changed, 391 insertions(+), 11 deletions(-) create mode 100644 .changeset/bright-migrations-report.md diff --git a/.changeset/bright-migrations-report.md b/.changeset/bright-migrations-report.md new file mode 100644 index 0000000000..55417ed5c1 --- /dev/null +++ b/.changeset/bright-migrations-report.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Show live phase, table, row-copy, verification, and failure progress during SQLite migration. +category: fix +dev: Adds structured migration progress events to first-boot startup and `fn db migrate` terminal output. diff --git a/packages/cli/src/commands/db.ts b/packages/cli/src/commands/db.ts index 55f31725b0..b8722cb001 100644 --- a/packages/cli/src/commands/db.ts +++ b/packages/cli/src/commands/db.ts @@ -4,6 +4,7 @@ import { vacuumAnalyze, resolveBackend, migrateSqliteToPostgres, + formatMigrationProgress, completeSqliteMigration, defaultMigrationSources, stampMigratedProjectRows, @@ -278,6 +279,9 @@ export async function runDbMigrate( projectId: registeredProjectId, migrationKey: `project:${registeredProjectId}`, deferCompletion: true, + onProgress: (event) => { + console.log(`fn db migrate: ${formatMigrationProgress(event)}`); + }, }); } catch (error) { console.error(`fn db migrate: migration failed: ${(error as Error).message}`); diff --git a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts index 43478466f2..8b0fe52486 100644 --- a/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts +++ b/packages/core/src/__tests__/postgres/sqlite-migrator.test.ts @@ -34,8 +34,10 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { DatabaseSync } from "../../sqlite-adapter.js"; import { + formatMigrationProgress, migrateSqliteToPostgres, toSnakeCase, + type MigrationProgressEvent, } from "../../postgres/sqlite-migrator.js"; import { applySchemaBaseline } from "../../postgres/schema-applier.js"; @@ -46,6 +48,35 @@ const PG_AVAILABLE = const pgDescribe = PG_AVAILABLE ? describe : describe.skip; +describe("SQLite migration CLI progress", () => { + it("formats table copy progress with position, row counts, and percentage", () => { + expect(formatMigrationProgress({ + phase: "table-progress", + sourceSchema: "project", + table: "run_audit_events", + tableIndex: 42, + tableCount: 124, + processedRows: 63_400, + sourceRows: 252_947, + })).toBe("[42/124] project.run_audit_events: processed 63,400/252,947 rows (25%)"); + }); + + it("makes transaction rollback explicit on failure", () => { + expect(formatMigrationProgress({ + phase: "failed", + error: "project.tasks failed verification", + })).toBe("FAILED — migration transaction rolled back: project.tasks failed verification"); + }); + + it("distinguishes committed verification failures from transaction rollbacks", () => { + expect(formatMigrationProgress({ + phase: "failed", + tableCount: 124, + failedTables: 2, + })).toBe("FAILED — 2/124 tables failed verification; migration will not be marked complete."); + }); +}); + /** * FNXC:PostgresMigration 2026-06-24-09:05: * Create a uniquely-named fresh PostgreSQL database. Mirrors the @@ -448,6 +479,90 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { expect(archived.targetRows).toBe(1); }); + /* + FNXC:CliMigrationProgress 2026-07-14-13:47: + CLI-triggered first-boot migrations must expose schema preparation, source discovery, per-table copy/verification, and terminal success so operators can distinguish a long-running copy from a stalled or failed startup. + */ + it("reports structured progress through planning, table verification, and completion", async () => { + const progress: MigrationProgressEvent[] = []; + let rejectFirstCallback = true; + const report = await migrateTest( + ctx!.db, + [{ sqlitePath: join(ctx!.fusionDir, "fusion.db"), pgSchema: "project" as const }], + { + onProgress: (event) => { + progress.push(event); + if (rejectFirstCallback) { + rejectFirstCallback = false; + return Promise.reject(new Error("test progress sink failure")); + } + }, + }, + ); + + expect(progress[0]?.phase).toBe("preparing-schema"); + expect(progress.some((event) => event.phase === "scanning-source")).toBe(true); + expect(progress.some((event) => event.phase === "copy-started" && event.tableCount === report.tables.length)).toBe(true); + expect(progress.some((event) => event.phase === "table-started" && event.table === "tasks")).toBe(true); + expect(progress.some((event) => event.phase === "table-verifying" && event.table === "tasks")).toBe(true); + expect(progress.some((event) => event.phase === "table-complete" && event.table === "tasks")).toBe(true); + expect(progress.at(-1)?.phase).toBe("copy-complete"); + expect(progress.filter((event) => event.phase === "table-complete")).toHaveLength(report.tables.length); + }); + + it("reports bounded quarter progress for a multi-batch table without a redundant 100% event", async () => { + const sqlitePath = join(ctx!.fusionDir, "fusion.db"); + const legacy = new DatabaseSync(sqlitePath); + try { + const insert = legacy.prepare( + `INSERT INTO tasks (id, title, description, "column", createdAt, updatedAt) + VALUES (?, ?, ?, ?, ?, ?)`, + ); + for (let index = 0; index < 648; index += 1) { + insert.run( + `FN-PROGRESS-${index}`, + `Progress ${index}`, + "progress fixture", + "todo", + "2026-07-14T00:00:00Z", + "2026-07-14T00:00:00Z", + ); + } + } finally { + legacy.close(); + } + + const progress: MigrationProgressEvent[] = []; + await migrateTest( + ctx!.db, + [{ sqlitePath, pgSchema: "project" as const }], + { onProgress: (event) => { progress.push(event); } }, + ); + const taskProgress = progress.filter( + (event): event is Extract => + event.phase === "table-progress" && event.table === "tasks", + ); + + expect(taskProgress.map((event) => Math.floor((event.processedRows / event.sourceRows) * 4))).toEqual([1, 2, 3]); + expect(taskProgress.every((event) => event.processedRows < event.sourceRows)).toBe(true); + }); + + it("reports a terminal dry-run plan without implying that data was written", async () => { + const progress: MigrationProgressEvent[] = []; + const report = await migrateTest( + ctx!.db, + [{ sqlitePath: join(ctx!.fusionDir, "fusion.db"), pgSchema: "project" as const }], + { dryRun: true, onProgress: (event) => { progress.push(event); } }, + ); + + expect(report.dryRun).toBe(true); + expect(progress.at(-1)).toMatchObject({ + phase: "dry-run-complete", + tableCount: report.tables.length, + }); + expect(formatMigrationProgress(progress.at(-1)!)).toContain("no data written"); + }); + /* FNXC:PostgresMigration 2026-07-13-22:37: Every user table in a legacy SQLite database must be represented in the migration report. An unknown table is retained as an explicit failed verification so startup cannot claim a complete cutover while silently abandoning operator data. @@ -462,9 +577,12 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { legacy.close(); } - const report = await migrateTest(ctx!.db, [ - { sqlitePath, pgSchema: "project" as const }, - ]); + const progress: MigrationProgressEvent[] = []; + const report = await migrateTest( + ctx!.db, + [{ sqlitePath, pgSchema: "project" as const }], + { onProgress: (event) => { progress.push(event); } }, + ); expect(report.tables).toContainEqual( expect.objectContaining({ @@ -476,6 +594,10 @@ pgDescribe("SQLite-to-PostgreSQL migrator", () => { skipped: false, }), ); + expect(progress.at(-1)).toMatchObject({ + phase: "failed", + failedTables: 1, + }); }); /* diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f518794bb9..69c98ba595 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2239,6 +2239,7 @@ export { isSqliteMigrationComplete, completeSqliteMigration, defaultMigrationSources, + formatMigrationProgress, // FNXC:CentralProjectIdentity 2026-07-13-23:10: // Post-migration project-partition stamping, shared by the startup-factory // first-boot auto-migration and `fn db migrate` so migrated rows are re-keyed @@ -2281,6 +2282,8 @@ export type { SqliteMigrationSource, SchemaName, MigrationReport, + MigrationProgressEvent, + MigrationProgressPhase, TableMigrationResult, StampMigratedProjectRowsInput, StampMigratedProjectRowsResult, diff --git a/packages/core/src/postgres/index.ts b/packages/core/src/postgres/index.ts index dfabb233cb..1cf0874084 100644 --- a/packages/core/src/postgres/index.ts +++ b/packages/core/src/postgres/index.ts @@ -159,11 +159,14 @@ export { isSqliteMigrationComplete, completeSqliteMigration, defaultMigrationSources, + formatMigrationProgress, toSnakeCase, type SqliteMigrationSource, type SchemaName, type MigrationOptions, type MigrationReport, + type MigrationProgressEvent, + type MigrationProgressPhase, type TableMigrationResult, } from "./sqlite-migrator.js"; diff --git a/packages/core/src/postgres/sqlite-migrator.ts b/packages/core/src/postgres/sqlite-migrator.ts index 7c234857d7..92a03d8e43 100644 --- a/packages/core/src/postgres/sqlite-migrator.ts +++ b/packages/core/src/postgres/sqlite-migrator.ts @@ -56,6 +56,7 @@ import { DatabaseSync } from "../sqlite-adapter.js"; import type { PostgresJsDatabase } from "drizzle-orm/postgres-js"; import { sql } from "drizzle-orm"; import { createHash } from "node:crypto"; +import { basename } from "node:path"; import { applySchemaBaseline } from "./schema-applier.js"; import { PROJECT_SCHEMA, @@ -63,6 +64,7 @@ import { ARCHIVE_SCHEMA, } from "./schema/_shared.js"; import { createLogger } from "../logger.js"; +import { getErrorMessage } from "../error-message.js"; const log = createLogger("sqlite-migrator"); @@ -176,6 +178,110 @@ export interface MigrationReport { readonly appliedBaseline: boolean; } +interface TableProgressCoordinates { + readonly sourceSchema: SchemaName; + readonly table: string; + readonly tableIndex: number; + readonly tableCount: number; +} + +/** Structured status emitted to CLI callers during a potentially long migration. */ +export type MigrationProgressEvent = + | { readonly phase: "preparing-schema" } + | { + readonly phase: "scanning-source"; + readonly sourcePath: string; + readonly sourceSchema: SchemaName; + readonly sourceIndex: number; + readonly sourceCount: number; + } + | { readonly phase: "copy-started"; readonly tableCount: number } + | ({ readonly phase: "table-started" } & TableProgressCoordinates) + | ({ + readonly phase: "table-progress"; + readonly processedRows: number; + readonly sourceRows: number; + } & TableProgressCoordinates) + | ({ + readonly phase: "table-verifying"; + readonly sourceRows: number; + readonly verificationStage: "target-count" | "source-content" | "target-content"; + } & TableProgressCoordinates) + | ({ + readonly phase: "table-complete"; + readonly sourceRows: number; + readonly insertedRows: number; + readonly targetRows: number; + readonly verified: boolean; + readonly skipped: boolean; + readonly skipReason?: string; + } & TableProgressCoordinates) + | { + readonly phase: "copy-complete"; + readonly tableCount: number; + readonly verifiedTables: number; + readonly failedTables: number; + readonly sequenceBumps: number; + } + | { + readonly phase: "dry-run-complete"; + readonly tableCount: number; + readonly sourceRows: number; + } + | { readonly phase: "failed"; readonly error: string } + | { + readonly phase: "failed"; + readonly tableCount: number; + readonly verifiedTables: number; + readonly failedTables: number; + }; + +export type MigrationProgressPhase = MigrationProgressEvent["phase"]; + +function formatTableProgressPrefix(event: TableProgressCoordinates): string { + return `[${event.tableIndex}/${event.tableCount}] ${event.sourceSchema}.${event.table}`; +} + +/** Format a structured migration event for terminal output. */ +export function formatMigrationProgress(event: MigrationProgressEvent): string { + switch (event.phase) { + case "preparing-schema": + return "Preparing PostgreSQL schema…"; + case "scanning-source": + return `Scanning source ${event.sourceIndex}/${event.sourceCount}: ${basename(event.sourcePath)} → ${event.sourceSchema}…`; + case "copy-started": + return `Found ${event.tableCount} tables. Copying and verifying data…`; + case "table-started": + return `${formatTableProgressPrefix(event)}: starting…`; + case "table-progress": { + const percent = Math.floor((event.processedRows / event.sourceRows) * 100); + return `${formatTableProgressPrefix(event)}: processed ${event.processedRows.toLocaleString()}/${event.sourceRows.toLocaleString()} rows (${percent}%)`; + } + case "table-verifying": { + const stage = { + "target-count": "counting migrated rows", + "source-content": "checksumming SQLite source", + "target-content": "checksumming PostgreSQL target", + }[event.verificationStage]; + return `${formatTableProgressPrefix(event)}: processed ${event.sourceRows.toLocaleString()} rows; ${stage}…`; + } + case "table-complete": + if (event.skipped) return `${formatTableProgressPrefix(event)}: skipped — ${event.skipReason ?? "not required"}`; + if (!event.verified) { + return `${formatTableProgressPrefix(event)}: VERIFICATION FAILED — source=${event.sourceRows}, target=${event.targetRows}${event.skipReason ? ` (${event.skipReason})` : ""}`; + } + return `${formatTableProgressPrefix(event)}: verified — ${event.sourceRows.toLocaleString()} rows (${event.insertedRows.toLocaleString()} inserted)`; + case "copy-complete": + return `Copy and verification complete — ${event.verifiedTables}/${event.tableCount} tables verified; ${event.sequenceBumps} sequences updated. Finalizing migration…`; + case "dry-run-complete": + return `Dry run complete — ${event.tableCount} tables and ${event.sourceRows.toLocaleString()} source rows planned; no data written.`; + case "failed": + return "error" in event + ? `FAILED — migration transaction rolled back: ${event.error}` + : `FAILED — ${event.failedTables}/${event.tableCount} tables failed verification; migration will not be marked complete.`; + } +} + /** Options for the migration. */ export interface MigrationOptions { /** If true, report the planned copy without modifying PostgreSQL. */ @@ -192,6 +298,21 @@ export interface MigrationOptions { readonly migrationKey?: string; /** Leave a verified migration running until caller-side project stamping succeeds. */ readonly deferCompletion?: boolean; + /** Receives CLI-safe progress events; callback failures never abort migration. */ + readonly onProgress?: (event: MigrationProgressEvent) => void | Promise; +} + +function emitMigrationProgress(options: MigrationOptions, event: MigrationProgressEvent): void { + try { + const pending = options.onProgress?.(event); + if (pending) { + void pending.catch((error) => { + log.warn(`Migration progress callback failed: ${getErrorMessage(error)}`); + }); + } + } catch (error) { + log.warn(`Migration progress callback failed: ${getErrorMessage(error)}`); + } } const SQLITE_MIGRATION_STATE_TABLE = "fusion_sqlite_migrations"; @@ -281,6 +402,11 @@ export async function migrateSqliteToPostgres( ), ); } catch (error) { + const errorMessage = getErrorMessage(error); + emitMigrationProgress(options, { + phase: "failed", + error: errorMessage, + }); if (options.dryRun !== true) { await migrationDb.transaction(async (tx) => { await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext('fusion:sqlite-migration-state'))`); @@ -288,7 +414,7 @@ export async function migrateSqliteToPostgres( await tx.execute(sql` INSERT INTO public.${sql.identifier(SQLITE_MIGRATION_STATE_TABLE)} (migration_key, project_id, status, last_error, updated_at) - VALUES (${migrationKey}, ${options.projectId ?? null}, 'failed', ${error instanceof Error ? error.message : String(error)}, now()) + VALUES (${migrationKey}, ${options.projectId ?? null}, 'failed', ${errorMessage}, now()) ON CONFLICT (migration_key) DO UPDATE SET project_id = EXCLUDED.project_id, status = 'failed', last_error = EXCLUDED.last_error, updated_at = now() `); @@ -305,6 +431,7 @@ async function migrateSqliteToPostgresOnSession( ): Promise { const dryRun = options.dryRun === true; const migrationKey = options.migrationKey ?? `project:${options.projectId ?? "unbound"}`; + emitMigrationProgress(options, { phase: "preparing-schema" }); /* * FNXC:PostgresMigration 2026-07-14-00:05: @@ -345,7 +472,7 @@ async function migrateSqliteToPostgresOnSession( if (!dryRun) { await migrationDb.execute(sql` UPDATE public.${sql.identifier(SQLITE_MIGRATION_STATE_TABLE)} - SET status = 'failed', last_error = ${error instanceof Error ? error.message : String(error)}, updated_at = now() + SET status = 'failed', last_error = ${getErrorMessage(error)}, updated_at = now() WHERE migration_key = ${migrationKey} `); } @@ -375,7 +502,7 @@ async function migrateSqliteToPostgresOnSession( } catch (error) { log.warn( `Could not set session_replication_role = replica (FK deferral requires SUPERUSER/REPLICATION): ` + - `${error instanceof Error ? error.message : String(error)}. ` + + `${getErrorMessage(error)}. ` + `Tables will be copied in name order; FK violations may surface if order is wrong.`, ); } @@ -383,11 +510,62 @@ async function migrateSqliteToPostgresOnSession( let copyError: unknown; try { - for (const source of sources) { + const plannedSources: Array<{ source: SqliteMigrationSource; plan: readonly TablePlan[] }> = []; + for (const [sourceOffset, source] of sources.entries()) { + emitMigrationProgress(options, { + phase: "scanning-source", + sourcePath: source.sqlitePath, + sourceSchema: source.pgSchema, + sourceIndex: sourceOffset + 1, + sourceCount: sources.length, + }); const plan = await buildMigrationPlan(migrationDb, source, options.projectId); + plannedSources.push({ source, plan }); + } + const tableCount = plannedSources.reduce((sum, planned) => sum + planned.plan.length, 0); + emitMigrationProgress(options, { phase: "copy-started", tableCount }); + let tableIndex = 0; + for (const { source, plan } of plannedSources) { for (const tablePlan of plan) { - const result = await migrateTable(migrationDb, source, tablePlan, dryRun); + tableIndex += 1; + const progressBase = { + sourceSchema: source.pgSchema, + table: tablePlan.pgTable, + tableIndex, + tableCount, + } as const; + emitMigrationProgress(options, { phase: "table-started", ...progressBase }); + const result = await migrateTable( + migrationDb, + source, + tablePlan, + dryRun, + { + onCopyProgress: (processedRows, sourceRows) => emitMigrationProgress(options, { + phase: "table-progress", + ...progressBase, + processedRows, + sourceRows, + }), + onVerifying: (verificationStage, sourceRows) => emitMigrationProgress(options, { + phase: "table-verifying", + ...progressBase, + sourceRows, + verificationStage, + }), + }, + ); tableResults.push(result); + emitMigrationProgress(options, { + phase: "table-complete", + ...progressBase, + sourceRows: result.sourceRows, + insertedRows: result.insertedRows, + targetRows: result.targetRows, + verified: result.verified, + skipped: result.skipped, + skipReason: result.skipReason, + }); // Bump identity sequences after a real (non-dry-run) copy. if (!dryRun && !result.skipped && result.sourceRows > 0) { @@ -440,7 +618,13 @@ async function migrateSqliteToPostgresOnSession( }; if (dryRun) { - log.log(`[dry-run] Migration plan: ${tableResults.length} tables, ${tableResults.reduce((n, t) => n + t.sourceRows, 0)} source rows planned. No writes performed.`); + const sourceRows = tableResults.reduce((n, t) => n + t.sourceRows, 0); + log.log(`[dry-run] Migration plan: ${tableResults.length} tables, ${sourceRows} source rows planned. No writes performed.`); + emitMigrationProgress(options, { + phase: "dry-run-complete", + tableCount: tableResults.length, + sourceRows, + }); } else { const ok = tableResults.filter((t) => t.verified).length; const bad = tableResults.length - ok; @@ -452,6 +636,18 @@ async function migrateSqliteToPostgresOnSession( WHERE migration_key = ${migrationKey} `); log.log(`Migration complete: ${ok}/${tableResults.length} tables verified (${bad} failed verification). ${sequenceBumps.length} sequences bumped.`); + emitMigrationProgress(options, bad === 0 ? { + phase: "copy-complete", + tableCount: tableResults.length, + verifiedTables: ok, + failedTables: bad, + sequenceBumps: sequenceBumps.length, + } : { + phase: "failed", + tableCount: tableResults.length, + verifiedTables: ok, + failedTables: bad, + }); } return report; @@ -903,6 +1099,30 @@ interface CanonicalLegacyRow { readonly json: string; } +interface TableMigrationProgressCallbacks { + readonly onCopyProgress?: (processedRows: number, sourceRows: number) => void; + readonly onVerifying?: ( + stage: "target-count" | "source-content" | "target-content", + sourceRows: number, + ) => void; +} + +function createQuarterProgressReporter( + sourceRows: number, + onCopyProgress?: (processedRows: number, sourceRows: number) => void, +): (processedRows: number) => void { + let lastProgressQuarter = 0; + return (processedRows) => { + // The verifying and complete events represent the terminal 100% state. + if (processedRows >= sourceRows) return; + const progressQuarter = Math.floor((processedRows / sourceRows) * 4); + if (progressQuarter > lastProgressQuarter) { + lastProgressQuarter = progressQuarter; + onCopyProgress?.(processedRows, sourceRows); + } + }; +} + function tagLegacyCell(value: unknown): TaggedLegacyCell { if (value === null || value === undefined) return { type: "null" }; if (Buffer.isBuffer(value)) { @@ -940,6 +1160,7 @@ async function migrateLegacyPreservationTable( source: SqliteMigrationSource, plan: TablePlan, dryRun: boolean, + progress: TableMigrationProgressCallbacks, ): Promise { const projectId = plan.partitionProjectId; const legacyPreservation = plan.legacyPreservation; @@ -969,6 +1190,7 @@ async function migrateLegacyPreservationTable( const sourceColumnOrder = (sqlite.prepare( `PRAGMA table_info(${quoteIdent(plan.table)})`, ).all() as Array<{ name: string }>).map(({ name }) => quoteIdent(name)).join(", "); + const reportCopyProgress = createQuarterProgressReporter(sourceRows, progress.onCopyProgress); for (let offset = 0; offset < sourceRows; offset += INSERT_BATCH_SIZE) { const rawBatch = sqlite.prepare( `SELECT * FROM ${quoteIdent(plan.table)} ORDER BY ${sourceColumnOrder} LIMIT ? OFFSET ?`, @@ -1005,8 +1227,11 @@ async function migrateLegacyPreservationTable( expectedBatch.get(row.legacy_row_hash) === canonicalizeCell(row.legacy_row) && row.source_schema_sql === legacyPreservation.sourceSchemaSql, ); + const processedRows = Math.min(offset + rawBatch.length, sourceRows); + reportCopyProgress(processedRows); } + progress.onVerifying?.("target-count", sourceRows); const targetCountRows = await db.execute(sql` SELECT COUNT(*)::int AS n FROM ${sql.raw(quoteIdent(plan.pgSchema))}.${sql.raw(quoteIdent(plan.pgTable))} @@ -1047,9 +1272,10 @@ async function migrateTable( source: SqliteMigrationSource, plan: TablePlan, dryRun: boolean, + progress: TableMigrationProgressCallbacks = {}, ): Promise { if (plan.legacyPreservation) { - return migrateLegacyPreservationTable(db, source, plan, dryRun); + return migrateLegacyPreservationTable(db, source, plan, dryRun, progress); } if (plan.unmappedSourceColumns.length > 0) { const sqlite = openSqlite(source.sqlitePath); @@ -1145,11 +1371,16 @@ async function migrateTable( // Stream rows in batches. const stmt = sqlite.prepare(`SELECT ${selectableCols} FROM ${quoteIdent(plan.table)}`); const batch: Record[] = []; + let processedRows = 0; + const reportCopyProgress = createQuarterProgressReporter(sourceRows, progress.onCopyProgress); const flush = async (): Promise => { if (batch.length === 0) return; + const batchSize = batch.length; const inserted = await insertBatch(db, plan, insertableCols, batch, hasIdentityCol); insertedRows += inserted; + processedRows += batchSize; batch.length = 0; + reportCopyProgress(processedRows); }; for (const row of stmt.all() as Array>) { @@ -1168,6 +1399,7 @@ async function migrateTable( } } await flush(); + progress.onVerifying?.("target-count", sourceRows); // Verify the migration. // FNXC:PostgresMigration 2026-06-26-15:40 (fix migration-review P1 #15): @@ -1197,9 +1429,11 @@ async function migrateTable( : targetRows === sourceRows; let contentOk = true; if (rowCountOk && sourceRows > 0) { + progress.onVerifying?.("source-content", sourceRows); const sourceCanonicalRows = computeSourceCanonicalRows( sqlite, plan.table, insertableCols, plan.partitionProjectId, ); + progress.onVerifying?.("target-content", sourceRows); const targetCanonicalRows = await computeTargetCanonicalRows( db, plan.pgSchema, plan.pgTable, insertableCols, plan.partitionProjectId, ); diff --git a/packages/core/src/postgres/startup-factory.ts b/packages/core/src/postgres/startup-factory.ts index 9db5bb2ff5..0261cda537 100644 --- a/packages/core/src/postgres/startup-factory.ts +++ b/packages/core/src/postgres/startup-factory.ts @@ -463,7 +463,7 @@ export async function createTaskStoreForBackend( considered. */ const migrationKey = `project:${migrationProjectId ?? rootDir}`; - const { migrateSqliteToPostgres, defaultMigrationSources, isSqliteMigrationComplete, completeSqliteMigration, recordSqliteMigrationComplete, CENTRAL_SQLITE_MIGRATION_KEY } = await import("./sqlite-migrator.js"); + const { migrateSqliteToPostgres, defaultMigrationSources, formatMigrationProgress, isSqliteMigrationComplete, completeSqliteMigration, recordSqliteMigrationComplete, CENTRAL_SQLITE_MIGRATION_KEY } = await import("./sqlite-migrator.js"); const migrationComplete = await isSqliteMigrationComplete(connections.migration, migrationKey); if (!migrationComplete && isValidSqliteDatabaseFile(legacySqlitePath)) { // The central (global-dir) source is optional: when no global dir is @@ -486,6 +486,13 @@ export async function createTaskStoreForBackend( projectId: migrationProjectId, migrationKey, deferCompletion: true, + /* + FNXC:CliMigrationProgress 2026-07-14-13:47: + First-boot migration can copy hundreds of thousands of rows. Forward structured phase, table, quarter-copy, and terminal events to the CLI logger so an operator sees forward progress and an explicit rollback instead of a silent startup wait. + */ + onProgress: (event) => { + log.log(`startup-factory: SQLite migration — ${formatMigrationProgress(event)}`); + }, }); /* FNXC:PostgresMigrationVerification 2026-07-13-22:37: