root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

FN-7095: add actionable backup failure details

Browse Source 
Database Backup automation failures now expose the affected database, paths, and root cause.

- Add DB-qualified error formatting for project and central backup creation, verification, schedule validation, and command failures.
- Normalize routine and cron in-process backup failures so empty or opaque errors become actionable AutomationRunResult errors.
- Cover missing database files, backup directory failures, central copy failures, corrupt backup quarantine, and runner normalization with regression tests.
- Document backup failure detail behavior and add a patch changeset.

Files changed:
 .changeset/fn-7095-backup-detail.md                |   7 ++
 docs/storage.md                                    |   2 +
 packages/core/src/__tests__/backup.test.ts         | 114 ++++++++++++++++++++-
 packages/core/src/backup.ts                        | 112 +++++++++++++++-----
 packages/engine/src/__tests__/cron-runner.test.ts  |  35 +++++++
 .../engine/src/__tests__/routine-runner.test.ts    |  45 ++++++++-
 packages/engine/src/cron-runner.ts                 |  20 +++-
 packages/engine/src/routine-runner.ts              |  20 +++-
 8 files changed, 324 insertions(+), 31 deletions(-)

Fusion-Task-Id: FN-7095

Fusion-Task-Lineage: 368e3edf-20d3-4756-97be-75734dbff703

Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
gsxdsm
2026-06-26 23:35:16 -07:00
parent f4b25dd2c9
commit ee3a06ecfd
8 changed files with 324 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.changeset/fn-7095-backup-detail.md Normal file
View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Database backup automation failures now report which database and the underlying cause.
category: fix
dev: Hardens runBackupCommand + routine/cron in-process backup branches so AutomationRunResult.error is always actionable.

2
docs/storage.md
View File

@@ -396,6 +396,8 @@ Backups in `.fusion/backups/` now capture the project DB and (when present) the
`BackupManager` supports `includeCentralDb` (default `true`). If central DB is missing or disabled, project backup still succeeds and records a skip reason. Retention (`autoBackupRetention`) is still computed from project backups; when an old project backup is pruned, its matching `fusion-central-*` sibling is pruned too. Restoring a project backup also restores the paired central backup when available; restoring a `fusion-central-*` file restores the central DB only. Pre-restore snapshots use `fusion-pre-restore-<timestamp>.db` and `fusion-central-pre-restore-<timestamp>.db`.
Database Backup automation failures are surfaced with DB-qualified detail. Project backup failures include the project DB source path, backup target or backup directory when available, and the underlying cause; central DB sub-failures keep the project backup run successful but include `Central DB backup failed` plus central source/target/cause detail in the run output.
## 4) SQLite Tables Inventory (`packages/core/src/db.ts`)
| Table | Purpose |

114
packages/core/src/__tests__/backup.test.ts
View File

@@ -878,7 +878,24 @@ describe("runBackupCommand", () => {
expect(result.output).toContain("Backup created");
});
it("should return failure for invalid schedule", async () => {
it("reports central DB missing as an explicit successful skip", async () => {
const settings: ProjectSettings = {
maxConcurrent: 2,
maxWorktrees: 4,
pollIntervalMs: 15000,
groupOverlappingFiles: false,
autoMerge: true,
autoBackupEnabled: true,
autoBackupRetention: 7,
};
const result = await runBackupCommand(fusionDir, settings);
expect(result.success).toBe(true);
expect(result.output).toContain("Central DB skipped: missing");
});
it("returns DB-qualified failure for invalid schedule", async () => {
const settings: ProjectSettings = {
maxConcurrent: 2,
maxWorktrees: 4,
@@ -892,7 +909,9 @@ describe("runBackupCommand", () => {
const result = await runBackupCommand(fusionDir, settings);
expect(result.success).toBe(false);
expect(result.output).toContain("Invalid backup schedule");
expect(result.output).toContain("project DB");
expect(result.output).toContain(join(fusionDir, "fusion.db"));
expect(result.output).toContain("invalid cron expression: invalid-cron");
});
it("should cleanup old backups after creation", async () => {
@@ -921,7 +940,7 @@ describe("runBackupCommand", () => {
expect(result.deletedCount).toBeGreaterThanOrEqual(1);
});
it("should report central copy failure while keeping success true", async () => {
it("reports central copy failure with DB and path detail while keeping success true", async () => {
const settings: ProjectSettings = {
maxConcurrent: 2,
maxWorktrees: 4,
@@ -938,9 +957,13 @@ describe("runBackupCommand", () => {
expect(result.success).toBe(true);
expect(result.output).toContain("Central DB backup failed");
expect(result.output).toContain("central DB");
expect(result.output).toContain("source:");
expect(result.output).toContain("target:");
expect(result.output).toContain("cause:");
});
it("should return failure when database file is missing", async () => {
it("returns DB-qualified failure when the project database file is missing", async () => {
// Remove the database
await rm(join(fusionDir, "fusion.db"));
@@ -956,6 +979,87 @@ describe("runBackupCommand", () => {
const result = await runBackupCommand(fusionDir, settings);
expect(result.success).toBe(false);
expect(result.output).toContain("failed");
expect(result.output).toContain("project DB");
expect(result.output).toContain(`source: ${join(fusionDir, "fusion.db")}`);
expect(result.output).toContain("target:");
expect(result.output).toContain("cause:");
expect(result.output).not.toMatch(/Backup failed:\s*$/);
});
it("returns DB-qualified failure when the backup directory cannot be created", async () => {
const blockedBackupDir = join(tempDir, "blocked-backups");
writeFileSync(blockedBackupDir, "not a directory");
const settings: ProjectSettings = {
maxConcurrent: 2,
maxWorktrees: 4,
pollIntervalMs: 15000,
groupOverlappingFiles: false,
autoMerge: true,
autoBackupEnabled: true,
autoBackupDir: "blocked-backups",
};
const result = await runBackupCommand(fusionDir, settings);
expect(result.success).toBe(false);
expect(result.output).toContain("project DB");
expect(result.output).toContain(`source: ${join(fusionDir, "fusion.db")}`);
expect(result.output).toContain(`backup directory: ${blockedBackupDir}`);
expect(result.output).toContain("cause:");
});
it("returns DB-qualified failure when project backup verification quarantines a corrupt copy", async () => {
const probe = spawnSync("sqlite3", ["--version"], { encoding: "utf-8" });
if (probe.error || probe.status !== 0) return;
writeFileSync(join(fusionDir, "fusion.db"), "not sqlite");
const settings: ProjectSettings = {
maxConcurrent: 2,
maxWorktrees: 4,
pollIntervalMs: 15000,
groupOverlappingFiles: false,
autoMerge: true,
autoBackupEnabled: true,
};
const result = await runBackupCommand(fusionDir, settings);
expect(result.success).toBe(false);
expect(result.output).toContain("project DB");
expect(result.output).toContain(`source: ${join(fusionDir, "fusion.db")}`);
expect(result.output).toContain("quarantined as *.corrupt");
expect(result.output).toContain("cause:");
});
it("does not report sqlite3-unavailable verification degradation as a backup failure", async () => {
vi.resetModules();
vi.doMock("node:child_process", () => ({
spawnSync: vi.fn(() => ({
error: Object.assign(new Error("spawn sqlite3 ENOENT"), { code: "ENOENT" }),
stdout: "",
stderr: "",
status: null,
})),
}));
try {
const { runBackupCommand: runBackupCommandWithMissingSqlite } = await import("../backup.js");
writeFileSync(join(fusionDir, "fusion.db"), "not sqlite but sqlite3 is unavailable");
const settings: ProjectSettings = {
maxConcurrent: 2,
maxWorktrees: 4,
pollIntervalMs: 15000,
groupOverlappingFiles: false,
autoMerge: true,
autoBackupEnabled: true,
};
const result = await runBackupCommandWithMissingSqlite(fusionDir, settings);
expect(result.success).toBe(true);
expect(result.output).toContain("Backup created");
expect(result.output).not.toContain("failed");
} finally {
vi.doUnmock("node:child_process");
vi.resetModules();
}
});
});

112
packages/core/src/backup.ts
View File

@@ -67,7 +67,16 @@ export class BackupManager {
async createBackup(): Promise<BackupInfo> {
const sourcePath = join(this.fusionDir, "fusion.db");
const backupDirPath = this.getBackupDirPath();
await mkdir(backupDirPath, { recursive: true });
try {
await mkdir(backupDirPath, { recursive: true });
} catch (err) {
throw new Error(formatBackupError({
dbLabel: "project DB",
action: "prepare backup directory",
backupDirPath,
cause: err,
}));
}
const timestamp = currentBackupTimestamp();
let counter = 0;
@@ -96,21 +105,31 @@ export class BackupManager {
const filename = generateBackupFilename(timestamp, counter);
const targetPath = join(backupDirPath, filename);
await copyLiveDatabase(sourcePath, targetPath);
try {
await copyLiveDatabase(sourcePath, targetPath);
// Verify the freshly-written copy. A copy of a live WAL db can capture a
// torn/corrupt main file; refusing to keep a corrupt backup guarantees
// that every retained `fusion-*.db` is restorable and that a corrupt copy
// is never counted as the "last known-good" by cleanupOldBackups().
if (this.verifyIntegrity) {
const integrity = verifyDatabaseIntegrity(targetPath);
if (!integrity.ok) {
await quarantineCorruptBackup(targetPath);
throw new Error(
`Backup verification failed for ${filename}: ${integrity.error ?? "database disk image is malformed"}. ` +
"The source database may be corrupt; the unusable copy was quarantined as *.corrupt.",
);
// Verify the freshly-written copy. A copy of a live WAL db can capture a
// torn/corrupt main file; refusing to keep a corrupt backup guarantees
// that every retained `fusion-*.db` is restorable and that a corrupt copy
// is never counted as the "last known-good" by cleanupOldBackups().
if (this.verifyIntegrity) {
const integrity = verifyDatabaseIntegrity(targetPath);
if (!integrity.ok) {
await quarantineCorruptBackup(targetPath);
throw new Error(
`verification failed: ${integrity.error ?? "database disk image is malformed"}. ` +
"The source database may be corrupt; the unusable copy was quarantined as *.corrupt.",
);
}
}
} catch (err) {
throw new Error(formatBackupError({
dbLabel: "project DB",
action: "create backup",
sourcePath,
targetPath,
cause: err,
}));
}
const stats = await stat(targetPath);
@@ -142,7 +161,8 @@ export class BackupManager {
if (!centralIntegrity.ok) {
await quarantineCorruptBackup(centralTargetPath);
throw new Error(
`central DB verification failed: ${centralIntegrity.error ?? "database disk image is malformed"}`,
`verification failed: ${centralIntegrity.error ?? "database disk image is malformed"}. ` +
"The source database may be corrupt; the unusable copy was quarantined as *.corrupt.",
);
}
}
@@ -156,7 +176,13 @@ export class BackupManager {
};
} catch (err) {
backup.centralBackup = {
failed: (err as Error).message,
failed: formatBackupError({
dbLabel: "central DB",
action: "create backup",
sourcePath: this.centralDbPath,
targetPath: centralTargetPath,
cause: err,
}),
};
}
@@ -573,10 +599,16 @@ export async function runBackupCommand(
fusionDir: string,
settings: ProjectSettings
): Promise<{ success: boolean; output: string; backupPath?: string; deletedCount?: number }> {
const projectDbPath = join(fusionDir, "fusion.db");
if (settings.autoBackupSchedule && !validateBackupSchedule(settings.autoBackupSchedule)) {
return {
success: false,
output: `Invalid backup schedule: ${settings.autoBackupSchedule}`,
output: formatBackupError({
dbLabel: "project DB",
action: "validate backup schedule",
sourcePath: projectDbPath,
cause: `invalid cron expression: ${settings.autoBackupSchedule}`,
}),
};
}
@@ -613,11 +645,55 @@ export async function runBackupCommand(
} catch (err) {
return {
success: false,
output: `Backup failed: ${(err as Error).message}`,
output: formatBackupError({
dbLabel: "project DB",
action: "run backup command",
sourcePath: projectDbPath,
cause: err,
}),
};
}
}
/*
FNXC:DatabaseBackup 2026-06-26-12:00:
Database Backup automations are operator-facing data-safety signals. Every failure must name the affected DB, relevant path, and cause so CLI, dashboard, routine, and cron surfaces never persist a detail-less "Backup failed" result.
*/
function formatBackupError(input: {
dbLabel: "project DB" | "central DB";
action: string;
sourcePath?: string;
targetPath?: string;
backupDirPath?: string;
cause: unknown;
}): string {
const parts = [`${input.dbLabel} ${input.action} failed`];
if (input.sourcePath) parts.push(`source: ${input.sourcePath}`);
if (input.targetPath) parts.push(`target: ${input.targetPath}`);
if (input.backupDirPath) parts.push(`backup directory: ${input.backupDirPath}`);
parts.push(`cause: ${describeError(input.cause)}`);
return parts.join("; ");
}
function describeError(err: unknown): string {
if (err instanceof Error) {
return err.message.trim() || err.name || "unknown error";
}
if (typeof err === "string") {
return err.trim() || "unknown error";
}
if (err === null || err === undefined) {
return "unknown error";
}
try {
const serialized = JSON.stringify(err);
if (serialized && serialized !== "{}") return serialized;
} catch {
// Fall through to String().
}
return String(err).trim() || "unknown error";
}
function formatBytes(bytes: number): string {
if (bytes === 0) return "0 B";
const k = 1024;

35
packages/engine/src/__tests__/cron-runner.test.ts
View File

@@ -2038,6 +2038,41 @@ describe("CronRunner", () => {
expect(runResult.success).toBe(true);
expect(runResult.output).toContain("fusion-central-");
});
it("normalizes empty runBackupCommand failure output for legacy schedules", async () => {
coreModuleMocks.runBackupCommand.mockResolvedValueOnce({ success: false, output: "" });
const store = createMockStore() as unknown as TaskStore & { getFusionDir: () => string };
store.getFusionDir = vi.fn().mockReturnValue("/tmp/.fusion");
const schedule = createMockSchedule({ id: "db-bk-empty", command: "fn backup --create" });
runner = new CronRunner(store, createMockAutomationStore());
const runResult = await (runner as unknown as { executeLegacyCommand: (s: ScheduledTask, startedAt: string) => Promise<AutomationRunResult> })
.executeLegacyCommand(schedule, new Date().toISOString());
expect(runResult.success).toBe(false);
expect(runResult.error).toBe("project DB run backup command failed; source: /tmp/.fusion/fusion.db; cause: unknown error");
expect(runResult.output).toBe("");
});
it("normalizes empty thrown errors for backup command steps", async () => {
coreModuleMocks.runBackupCommand.mockRejectedValueOnce(new Error(""));
const store = createMockStore() as unknown as TaskStore & { getFusionDir: () => string };
store.getFusionDir = vi.fn().mockReturnValue("/tmp/.fusion");
runner = new CronRunner(store, createMockAutomationStore());
const step: AutomationStep = {
id: "step-db-bk",
name: "Database Backup",
type: "command",
command: "fn backup --create",
};
const stepResult = await (runner as unknown as { executeCommandStep: (s: AutomationStep, i: number, t: number, startedAt: string) => Promise<AutomationRunResult> })
.executeCommandStep(step, 0, 30_000, new Date().toISOString());
expect(stepResult.success).toBe(false);
expect(stepResult.error).toBe("project DB run backup command failed; source: /tmp/.fusion/fusion.db; cause: unknown error");
expect(stepResult.output).toBe("");
});
});
describe("memory backup in-process dispatch", () => {

45
packages/engine/src/__tests__/routine-runner.test.ts
View File

@@ -9,6 +9,10 @@ import type {
Settings,
} from "@fusion/core";
import type { HeartbeatMonitor } from "../agent-heartbeat.js";
import { mkdtempSync } from "node:fs";
import { mkdir, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
// Default settings inline to avoid @fusion/core build dependency during tests
const DEFAULT_SETTINGS: Settings = {
@@ -111,9 +115,10 @@ function createMockAgentStore(): AgentStore {
} as unknown as AgentStore;
}
function createMockTaskStore(): TaskStore {
function createMockTaskStore(overrides: { fusionDir?: string; settings?: Partial<Settings> } = {}): TaskStore {
return {
getSettings: vi.fn().mockResolvedValue(DEFAULT_SETTINGS),
getFusionDir: vi.fn().mockReturnValue(overrides.fusionDir ?? "/tmp/.fusion"),
getSettings: vi.fn().mockResolvedValue({ ...DEFAULT_SETTINGS, ...overrides.settings }),
on: vi.fn(),
off: vi.fn(),
} as unknown as TaskStore;
@@ -238,6 +243,42 @@ describe("RoutineRunner", () => {
);
});
it("persists an actionable error for in-process Database Backup failures", async () => {
const tempDir = mkdtempSync(join(tmpdir(), "routine-backup-detail-"));
const fusionDir = join(tempDir, ".fusion");
await mkdir(fusionDir, { recursive: true });
const routine = createMockRoutine({
id: "routine-backup-missing-db",
command: "fn backup --create",
agentId: "",
});
const routineStore = createMockRoutineStore([routine]);
const runner = createRoutineRunner({
routineStore,
taskStore: createMockTaskStore({ fusionDir }),
});
try {
const result = await runner.executeRoutine("routine-backup-missing-db", "cron");
expect(result.success).toBe(false);
expect(result.error).toContain("project DB");
expect(result.error).toContain(`source: ${join(fusionDir, "fusion.db")}`);
expect(result.error).toContain("cause:");
expect(result.error).not.toBe("");
expect(routineStore.completeRoutineExecution).toHaveBeenCalledWith(
"routine-backup-missing-db",
expect.objectContaining({
success: false,
error: result.error,
output: expect.stringContaining("project DB"),
}),
);
} finally {
await rm(tempDir, { recursive: true, force: true });
}
});
it("marks execution as failed when executeHeartbeat rejects", async () => {
const routine = createMockRoutine({ id: "routine-fail" });
const routineStore = createMockRoutineStore([routine]);

20
packages/engine/src/cron-runner.ts
View File

@@ -608,13 +608,14 @@ export class CronRunner {
const fusionDir = this.store.getFusionDir();
const settings = await this.store.getSettings();
const result = await runBackupCommand(fusionDir, settings);
const output = truncateOutput(result.output ?? "", "");
return {
success: result.success,
output: truncateOutput(result.output ?? "", ""),
error: result.success ? undefined : result.output,
output,
error: result.success ? undefined : formatInProcessBackupError(output, fusionDir),
};
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
const message = formatInProcessBackupError(err, this.store.getFusionDir());
return { success: false, output: "", error: message };
}
}
@@ -1056,6 +1057,19 @@ export async function createAiPromptExecutor(cwd: string, store?: TaskStore): Pr
};
}
/*
FNXC:DatabaseBackup 2026-06-26-12:00:
Cron-runner in-process backups feed automation run history and step errors. Normalize empty thrown values and empty command output before they become operator-visible Database Backup failures.
*/
function formatInProcessBackupError(err: unknown, fusionDir: string): string {
const message = err instanceof Error ? err.message.trim() : String(err ?? "").trim();
const cause = message || "unknown error";
if (cause.includes("project DB") || cause.includes("central DB")) {
return cause;
}
return `project DB run backup command failed; source: ${fusionDir}/fusion.db; cause: ${cause}`;
}
/** Combine and truncate stdout/stderr to stay within storage limits. */
function truncateOutput(stdout: string | null | undefined, stderr: string | null | undefined): string {
const out = stdout ?? "";

20
packages/engine/src/routine-runner.ts
View File

@@ -306,15 +306,16 @@ export class RoutineRunner {
const fusionDir = this.options.taskStore.getFusionDir();
const settings = await this.options.taskStore.getSettings();
const result = await runBackupCommand(fusionDir, settings);
const output = truncateOutput(result.output ?? "", "");
return {
success: result.success,
output: truncateOutput(result.output ?? "", ""),
error: result.success ? undefined : result.output,
output,
error: result.success ? undefined : formatInProcessBackupError(output, fusionDir),
startedAt,
completedAt: new Date().toISOString(),
};
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
const message = formatInProcessBackupError(err, this.options.taskStore.getFusionDir());
return {
success: false,
output: "",
@@ -633,6 +634,19 @@ export class RoutineRunner {
}
}
/*
FNXC:DatabaseBackup 2026-06-26-12:00:
Routine-runner in-process backups persist AutomationRunResult.error directly to lastRunResult. Normalize empty or opaque failures here so Database Backup cards always show a DB-qualified cause.
*/
function formatInProcessBackupError(err: unknown, fusionDir: string): string {
const message = err instanceof Error ? err.message.trim() : String(err ?? "").trim();
const cause = message || "unknown error";
if (cause.includes("project DB") || cause.includes("central DB")) {
return cause;
}
return `project DB run backup command failed; source: ${fusionDir}/fusion.db; cause: ${cause}`;
}
function truncateOutput(stdout: string, stderr: string): string {
let output = stdout;
if (stderr) {