diff --git a/.changeset/fn-7095-backup-detail.md b/.changeset/fn-7095-backup-detail.md new file mode 100644 index 0000000000..634b8d50ca --- /dev/null +++ b/.changeset/fn-7095-backup-detail.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Database backup automation failures now report which database and the underlying cause. +category: fix +dev: Hardens runBackupCommand + routine/cron in-process backup branches so AutomationRunResult.error is always actionable. diff --git a/docs/storage.md b/docs/storage.md index dbef183f75..db4edcaaf4 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -396,6 +396,8 @@ Backups in `.fusion/backups/` now capture the project DB and (when present) the `BackupManager` supports `includeCentralDb` (default `true`). If central DB is missing or disabled, project backup still succeeds and records a skip reason. Retention (`autoBackupRetention`) is still computed from project backups; when an old project backup is pruned, its matching `fusion-central-*` sibling is pruned too. Restoring a project backup also restores the paired central backup when available; restoring a `fusion-central-*` file restores the central DB only. Pre-restore snapshots use `fusion-pre-restore-.db` and `fusion-central-pre-restore-.db`. +Database Backup automation failures are surfaced with DB-qualified detail. Project backup failures include the project DB source path, backup target or backup directory when available, and the underlying cause; central DB sub-failures keep the project backup run successful but include `Central DB backup failed` plus central source/target/cause detail in the run output. + ## 4) SQLite Tables Inventory (`packages/core/src/db.ts`) | Table | Purpose | diff --git a/packages/core/src/__tests__/backup.test.ts b/packages/core/src/__tests__/backup.test.ts index c2458c7457..bbee731e94 100644 --- a/packages/core/src/__tests__/backup.test.ts +++ b/packages/core/src/__tests__/backup.test.ts @@ -878,7 +878,24 @@ describe("runBackupCommand", () => { expect(result.output).toContain("Backup created"); }); - it("should return failure for invalid schedule", async () => { + it("reports central DB missing as an explicit successful skip", async () => { + const settings: ProjectSettings = { + maxConcurrent: 2, + maxWorktrees: 4, + pollIntervalMs: 15000, + groupOverlappingFiles: false, + autoMerge: true, + autoBackupEnabled: true, + autoBackupRetention: 7, + }; + + const result = await runBackupCommand(fusionDir, settings); + + expect(result.success).toBe(true); + expect(result.output).toContain("Central DB skipped: missing"); + }); + + it("returns DB-qualified failure for invalid schedule", async () => { const settings: ProjectSettings = { maxConcurrent: 2, maxWorktrees: 4, @@ -892,7 +909,9 @@ describe("runBackupCommand", () => { const result = await runBackupCommand(fusionDir, settings); expect(result.success).toBe(false); - expect(result.output).toContain("Invalid backup schedule"); + expect(result.output).toContain("project DB"); + expect(result.output).toContain(join(fusionDir, "fusion.db")); + expect(result.output).toContain("invalid cron expression: invalid-cron"); }); it("should cleanup old backups after creation", async () => { @@ -921,7 +940,7 @@ describe("runBackupCommand", () => { expect(result.deletedCount).toBeGreaterThanOrEqual(1); }); - it("should report central copy failure while keeping success true", async () => { + it("reports central copy failure with DB and path detail while keeping success true", async () => { const settings: ProjectSettings = { maxConcurrent: 2, maxWorktrees: 4, @@ -938,9 +957,13 @@ describe("runBackupCommand", () => { expect(result.success).toBe(true); expect(result.output).toContain("Central DB backup failed"); + expect(result.output).toContain("central DB"); + expect(result.output).toContain("source:"); + expect(result.output).toContain("target:"); + expect(result.output).toContain("cause:"); }); - it("should return failure when database file is missing", async () => { + it("returns DB-qualified failure when the project database file is missing", async () => { // Remove the database await rm(join(fusionDir, "fusion.db")); @@ -956,6 +979,87 @@ describe("runBackupCommand", () => { const result = await runBackupCommand(fusionDir, settings); expect(result.success).toBe(false); - expect(result.output).toContain("failed"); + expect(result.output).toContain("project DB"); + expect(result.output).toContain(`source: ${join(fusionDir, "fusion.db")}`); + expect(result.output).toContain("target:"); + expect(result.output).toContain("cause:"); + expect(result.output).not.toMatch(/Backup failed:\s*$/); + }); + + it("returns DB-qualified failure when the backup directory cannot be created", async () => { + const blockedBackupDir = join(tempDir, "blocked-backups"); + writeFileSync(blockedBackupDir, "not a directory"); + const settings: ProjectSettings = { + maxConcurrent: 2, + maxWorktrees: 4, + pollIntervalMs: 15000, + groupOverlappingFiles: false, + autoMerge: true, + autoBackupEnabled: true, + autoBackupDir: "blocked-backups", + }; + + const result = await runBackupCommand(fusionDir, settings); + + expect(result.success).toBe(false); + expect(result.output).toContain("project DB"); + expect(result.output).toContain(`source: ${join(fusionDir, "fusion.db")}`); + expect(result.output).toContain(`backup directory: ${blockedBackupDir}`); + expect(result.output).toContain("cause:"); + }); + + it("returns DB-qualified failure when project backup verification quarantines a corrupt copy", async () => { + const probe = spawnSync("sqlite3", ["--version"], { encoding: "utf-8" }); + if (probe.error || probe.status !== 0) return; + writeFileSync(join(fusionDir, "fusion.db"), "not sqlite"); + const settings: ProjectSettings = { + maxConcurrent: 2, + maxWorktrees: 4, + pollIntervalMs: 15000, + groupOverlappingFiles: false, + autoMerge: true, + autoBackupEnabled: true, + }; + + const result = await runBackupCommand(fusionDir, settings); + + expect(result.success).toBe(false); + expect(result.output).toContain("project DB"); + expect(result.output).toContain(`source: ${join(fusionDir, "fusion.db")}`); + expect(result.output).toContain("quarantined as *.corrupt"); + expect(result.output).toContain("cause:"); + }); + + it("does not report sqlite3-unavailable verification degradation as a backup failure", async () => { + vi.resetModules(); + vi.doMock("node:child_process", () => ({ + spawnSync: vi.fn(() => ({ + error: Object.assign(new Error("spawn sqlite3 ENOENT"), { code: "ENOENT" }), + stdout: "", + stderr: "", + status: null, + })), + })); + try { + const { runBackupCommand: runBackupCommandWithMissingSqlite } = await import("../backup.js"); + writeFileSync(join(fusionDir, "fusion.db"), "not sqlite but sqlite3 is unavailable"); + const settings: ProjectSettings = { + maxConcurrent: 2, + maxWorktrees: 4, + pollIntervalMs: 15000, + groupOverlappingFiles: false, + autoMerge: true, + autoBackupEnabled: true, + }; + + const result = await runBackupCommandWithMissingSqlite(fusionDir, settings); + + expect(result.success).toBe(true); + expect(result.output).toContain("Backup created"); + expect(result.output).not.toContain("failed"); + } finally { + vi.doUnmock("node:child_process"); + vi.resetModules(); + } }); }); diff --git a/packages/core/src/backup.ts b/packages/core/src/backup.ts index 1c1f261f25..6a92776875 100644 --- a/packages/core/src/backup.ts +++ b/packages/core/src/backup.ts @@ -67,7 +67,16 @@ export class BackupManager { async createBackup(): Promise { const sourcePath = join(this.fusionDir, "fusion.db"); const backupDirPath = this.getBackupDirPath(); - await mkdir(backupDirPath, { recursive: true }); + try { + await mkdir(backupDirPath, { recursive: true }); + } catch (err) { + throw new Error(formatBackupError({ + dbLabel: "project DB", + action: "prepare backup directory", + backupDirPath, + cause: err, + })); + } const timestamp = currentBackupTimestamp(); let counter = 0; @@ -96,21 +105,31 @@ export class BackupManager { const filename = generateBackupFilename(timestamp, counter); const targetPath = join(backupDirPath, filename); - await copyLiveDatabase(sourcePath, targetPath); + try { + await copyLiveDatabase(sourcePath, targetPath); - // Verify the freshly-written copy. A copy of a live WAL db can capture a - // torn/corrupt main file; refusing to keep a corrupt backup guarantees - // that every retained `fusion-*.db` is restorable and that a corrupt copy - // is never counted as the "last known-good" by cleanupOldBackups(). - if (this.verifyIntegrity) { - const integrity = verifyDatabaseIntegrity(targetPath); - if (!integrity.ok) { - await quarantineCorruptBackup(targetPath); - throw new Error( - `Backup verification failed for ${filename}: ${integrity.error ?? "database disk image is malformed"}. ` + - "The source database may be corrupt; the unusable copy was quarantined as *.corrupt.", - ); + // Verify the freshly-written copy. A copy of a live WAL db can capture a + // torn/corrupt main file; refusing to keep a corrupt backup guarantees + // that every retained `fusion-*.db` is restorable and that a corrupt copy + // is never counted as the "last known-good" by cleanupOldBackups(). + if (this.verifyIntegrity) { + const integrity = verifyDatabaseIntegrity(targetPath); + if (!integrity.ok) { + await quarantineCorruptBackup(targetPath); + throw new Error( + `verification failed: ${integrity.error ?? "database disk image is malformed"}. ` + + "The source database may be corrupt; the unusable copy was quarantined as *.corrupt.", + ); + } } + } catch (err) { + throw new Error(formatBackupError({ + dbLabel: "project DB", + action: "create backup", + sourcePath, + targetPath, + cause: err, + })); } const stats = await stat(targetPath); @@ -142,7 +161,8 @@ export class BackupManager { if (!centralIntegrity.ok) { await quarantineCorruptBackup(centralTargetPath); throw new Error( - `central DB verification failed: ${centralIntegrity.error ?? "database disk image is malformed"}`, + `verification failed: ${centralIntegrity.error ?? "database disk image is malformed"}. ` + + "The source database may be corrupt; the unusable copy was quarantined as *.corrupt.", ); } } @@ -156,7 +176,13 @@ export class BackupManager { }; } catch (err) { backup.centralBackup = { - failed: (err as Error).message, + failed: formatBackupError({ + dbLabel: "central DB", + action: "create backup", + sourcePath: this.centralDbPath, + targetPath: centralTargetPath, + cause: err, + }), }; } @@ -573,10 +599,16 @@ export async function runBackupCommand( fusionDir: string, settings: ProjectSettings ): Promise<{ success: boolean; output: string; backupPath?: string; deletedCount?: number }> { + const projectDbPath = join(fusionDir, "fusion.db"); if (settings.autoBackupSchedule && !validateBackupSchedule(settings.autoBackupSchedule)) { return { success: false, - output: `Invalid backup schedule: ${settings.autoBackupSchedule}`, + output: formatBackupError({ + dbLabel: "project DB", + action: "validate backup schedule", + sourcePath: projectDbPath, + cause: `invalid cron expression: ${settings.autoBackupSchedule}`, + }), }; } @@ -613,11 +645,55 @@ export async function runBackupCommand( } catch (err) { return { success: false, - output: `Backup failed: ${(err as Error).message}`, + output: formatBackupError({ + dbLabel: "project DB", + action: "run backup command", + sourcePath: projectDbPath, + cause: err, + }), }; } } +/* +FNXC:DatabaseBackup 2026-06-26-12:00: +Database Backup automations are operator-facing data-safety signals. Every failure must name the affected DB, relevant path, and cause so CLI, dashboard, routine, and cron surfaces never persist a detail-less "Backup failed" result. +*/ +function formatBackupError(input: { + dbLabel: "project DB" | "central DB"; + action: string; + sourcePath?: string; + targetPath?: string; + backupDirPath?: string; + cause: unknown; +}): string { + const parts = [`${input.dbLabel} ${input.action} failed`]; + if (input.sourcePath) parts.push(`source: ${input.sourcePath}`); + if (input.targetPath) parts.push(`target: ${input.targetPath}`); + if (input.backupDirPath) parts.push(`backup directory: ${input.backupDirPath}`); + parts.push(`cause: ${describeError(input.cause)}`); + return parts.join("; "); +} + +function describeError(err: unknown): string { + if (err instanceof Error) { + return err.message.trim() || err.name || "unknown error"; + } + if (typeof err === "string") { + return err.trim() || "unknown error"; + } + if (err === null || err === undefined) { + return "unknown error"; + } + try { + const serialized = JSON.stringify(err); + if (serialized && serialized !== "{}") return serialized; + } catch { + // Fall through to String(). + } + return String(err).trim() || "unknown error"; +} + function formatBytes(bytes: number): string { if (bytes === 0) return "0 B"; const k = 1024; diff --git a/packages/engine/src/__tests__/cron-runner.test.ts b/packages/engine/src/__tests__/cron-runner.test.ts index acd937b0e6..829104b53b 100644 --- a/packages/engine/src/__tests__/cron-runner.test.ts +++ b/packages/engine/src/__tests__/cron-runner.test.ts @@ -2038,6 +2038,41 @@ describe("CronRunner", () => { expect(runResult.success).toBe(true); expect(runResult.output).toContain("fusion-central-"); }); + + it("normalizes empty runBackupCommand failure output for legacy schedules", async () => { + coreModuleMocks.runBackupCommand.mockResolvedValueOnce({ success: false, output: "" }); + const store = createMockStore() as unknown as TaskStore & { getFusionDir: () => string }; + store.getFusionDir = vi.fn().mockReturnValue("/tmp/.fusion"); + const schedule = createMockSchedule({ id: "db-bk-empty", command: "fn backup --create" }); + runner = new CronRunner(store, createMockAutomationStore()); + + const runResult = await (runner as unknown as { executeLegacyCommand: (s: ScheduledTask, startedAt: string) => Promise }) + .executeLegacyCommand(schedule, new Date().toISOString()); + + expect(runResult.success).toBe(false); + expect(runResult.error).toBe("project DB run backup command failed; source: /tmp/.fusion/fusion.db; cause: unknown error"); + expect(runResult.output).toBe(""); + }); + + it("normalizes empty thrown errors for backup command steps", async () => { + coreModuleMocks.runBackupCommand.mockRejectedValueOnce(new Error("")); + const store = createMockStore() as unknown as TaskStore & { getFusionDir: () => string }; + store.getFusionDir = vi.fn().mockReturnValue("/tmp/.fusion"); + runner = new CronRunner(store, createMockAutomationStore()); + const step: AutomationStep = { + id: "step-db-bk", + name: "Database Backup", + type: "command", + command: "fn backup --create", + }; + + const stepResult = await (runner as unknown as { executeCommandStep: (s: AutomationStep, i: number, t: number, startedAt: string) => Promise }) + .executeCommandStep(step, 0, 30_000, new Date().toISOString()); + + expect(stepResult.success).toBe(false); + expect(stepResult.error).toBe("project DB run backup command failed; source: /tmp/.fusion/fusion.db; cause: unknown error"); + expect(stepResult.output).toBe(""); + }); }); describe("memory backup in-process dispatch", () => { diff --git a/packages/engine/src/__tests__/routine-runner.test.ts b/packages/engine/src/__tests__/routine-runner.test.ts index 8a58eaa33a..b1c8fc792f 100644 --- a/packages/engine/src/__tests__/routine-runner.test.ts +++ b/packages/engine/src/__tests__/routine-runner.test.ts @@ -9,6 +9,10 @@ import type { Settings, } from "@fusion/core"; import type { HeartbeatMonitor } from "../agent-heartbeat.js"; +import { mkdtempSync } from "node:fs"; +import { mkdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; // Default settings inline to avoid @fusion/core build dependency during tests const DEFAULT_SETTINGS: Settings = { @@ -111,9 +115,10 @@ function createMockAgentStore(): AgentStore { } as unknown as AgentStore; } -function createMockTaskStore(): TaskStore { +function createMockTaskStore(overrides: { fusionDir?: string; settings?: Partial } = {}): TaskStore { return { - getSettings: vi.fn().mockResolvedValue(DEFAULT_SETTINGS), + getFusionDir: vi.fn().mockReturnValue(overrides.fusionDir ?? "/tmp/.fusion"), + getSettings: vi.fn().mockResolvedValue({ ...DEFAULT_SETTINGS, ...overrides.settings }), on: vi.fn(), off: vi.fn(), } as unknown as TaskStore; @@ -238,6 +243,42 @@ describe("RoutineRunner", () => { ); }); + it("persists an actionable error for in-process Database Backup failures", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "routine-backup-detail-")); + const fusionDir = join(tempDir, ".fusion"); + await mkdir(fusionDir, { recursive: true }); + const routine = createMockRoutine({ + id: "routine-backup-missing-db", + command: "fn backup --create", + agentId: "", + }); + const routineStore = createMockRoutineStore([routine]); + const runner = createRoutineRunner({ + routineStore, + taskStore: createMockTaskStore({ fusionDir }), + }); + + try { + const result = await runner.executeRoutine("routine-backup-missing-db", "cron"); + + expect(result.success).toBe(false); + expect(result.error).toContain("project DB"); + expect(result.error).toContain(`source: ${join(fusionDir, "fusion.db")}`); + expect(result.error).toContain("cause:"); + expect(result.error).not.toBe(""); + expect(routineStore.completeRoutineExecution).toHaveBeenCalledWith( + "routine-backup-missing-db", + expect.objectContaining({ + success: false, + error: result.error, + output: expect.stringContaining("project DB"), + }), + ); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); + it("marks execution as failed when executeHeartbeat rejects", async () => { const routine = createMockRoutine({ id: "routine-fail" }); const routineStore = createMockRoutineStore([routine]); diff --git a/packages/engine/src/cron-runner.ts b/packages/engine/src/cron-runner.ts index a2142dc135..de2f17eef8 100644 --- a/packages/engine/src/cron-runner.ts +++ b/packages/engine/src/cron-runner.ts @@ -608,13 +608,14 @@ export class CronRunner { const fusionDir = this.store.getFusionDir(); const settings = await this.store.getSettings(); const result = await runBackupCommand(fusionDir, settings); + const output = truncateOutput(result.output ?? "", ""); return { success: result.success, - output: truncateOutput(result.output ?? "", ""), - error: result.success ? undefined : result.output, + output, + error: result.success ? undefined : formatInProcessBackupError(output, fusionDir), }; } catch (err) { - const message = err instanceof Error ? err.message : String(err); + const message = formatInProcessBackupError(err, this.store.getFusionDir()); return { success: false, output: "", error: message }; } } @@ -1056,6 +1057,19 @@ export async function createAiPromptExecutor(cwd: string, store?: TaskStore): Pr }; } +/* +FNXC:DatabaseBackup 2026-06-26-12:00: +Cron-runner in-process backups feed automation run history and step errors. Normalize empty thrown values and empty command output before they become operator-visible Database Backup failures. +*/ +function formatInProcessBackupError(err: unknown, fusionDir: string): string { + const message = err instanceof Error ? err.message.trim() : String(err ?? "").trim(); + const cause = message || "unknown error"; + if (cause.includes("project DB") || cause.includes("central DB")) { + return cause; + } + return `project DB run backup command failed; source: ${fusionDir}/fusion.db; cause: ${cause}`; +} + /** Combine and truncate stdout/stderr to stay within storage limits. */ function truncateOutput(stdout: string | null | undefined, stderr: string | null | undefined): string { const out = stdout ?? ""; diff --git a/packages/engine/src/routine-runner.ts b/packages/engine/src/routine-runner.ts index e21a3e26e4..8cfa60cd36 100644 --- a/packages/engine/src/routine-runner.ts +++ b/packages/engine/src/routine-runner.ts @@ -306,15 +306,16 @@ export class RoutineRunner { const fusionDir = this.options.taskStore.getFusionDir(); const settings = await this.options.taskStore.getSettings(); const result = await runBackupCommand(fusionDir, settings); + const output = truncateOutput(result.output ?? "", ""); return { success: result.success, - output: truncateOutput(result.output ?? "", ""), - error: result.success ? undefined : result.output, + output, + error: result.success ? undefined : formatInProcessBackupError(output, fusionDir), startedAt, completedAt: new Date().toISOString(), }; } catch (err) { - const message = err instanceof Error ? err.message : String(err); + const message = formatInProcessBackupError(err, this.options.taskStore.getFusionDir()); return { success: false, output: "", @@ -633,6 +634,19 @@ export class RoutineRunner { } } +/* +FNXC:DatabaseBackup 2026-06-26-12:00: +Routine-runner in-process backups persist AutomationRunResult.error directly to lastRunResult. Normalize empty or opaque failures here so Database Backup cards always show a DB-qualified cause. +*/ +function formatInProcessBackupError(err: unknown, fusionDir: string): string { + const message = err instanceof Error ? err.message.trim() : String(err ?? "").trim(); + const cause = message || "unknown error"; + if (cause.includes("project DB") || cause.includes("central DB")) { + return cause; + } + return `project DB run backup command failed; source: ${fusionDir}/fusion.db; cause: ${cause}`; +} + function truncateOutput(stdout: string, stderr: string): string { let output = stdout; if (stderr) {