FN-7095: add actionable backup failure details
Database Backup automation failures now expose the affected database, paths, and root cause. - Add DB-qualified error formatting for project and central backup creation, verification, schedule validation, and command failures. - Normalize routine and cron in-process backup failures so empty or opaque errors become actionable AutomationRunResult errors. - Cover missing database files, backup directory failures, central copy failures, corrupt backup quarantine, and runner normalization with regression tests. - Document backup failure detail behavior and add a patch changeset. Files changed: .changeset/fn-7095-backup-detail.md | 7 ++ docs/storage.md | 2 + packages/core/src/__tests__/backup.test.ts | 114 ++++++++++++++++++++- packages/core/src/backup.ts | 112 +++++++++++++++----- packages/engine/src/__tests__/cron-runner.test.ts | 35 +++++++ .../engine/src/__tests__/routine-runner.test.ts | 45 ++++++++- packages/engine/src/cron-runner.ts | 20 +++- packages/engine/src/routine-runner.ts | 20 +++- 8 files changed, 324 insertions(+), 31 deletions(-) Fusion-Task-Id: FN-7095 Fusion-Task-Lineage: 368e3edf-20d3-4756-97be-75734dbff703 Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
7
.changeset/fn-7095-backup-detail.md
Normal file
7
.changeset/fn-7095-backup-detail.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
summary: Database backup automation failures now report which database and the underlying cause.
|
||||||
|
category: fix
|
||||||
|
dev: Hardens runBackupCommand + routine/cron in-process backup branches so AutomationRunResult.error is always actionable.
|
||||||
@@ -396,6 +396,8 @@ Backups in `.fusion/backups/` now capture the project DB and (when present) the
|
|||||||
|
|
||||||
`BackupManager` supports `includeCentralDb` (default `true`). If central DB is missing or disabled, project backup still succeeds and records a skip reason. Retention (`autoBackupRetention`) is still computed from project backups; when an old project backup is pruned, its matching `fusion-central-*` sibling is pruned too. Restoring a project backup also restores the paired central backup when available; restoring a `fusion-central-*` file restores the central DB only. Pre-restore snapshots use `fusion-pre-restore-<timestamp>.db` and `fusion-central-pre-restore-<timestamp>.db`.
|
`BackupManager` supports `includeCentralDb` (default `true`). If central DB is missing or disabled, project backup still succeeds and records a skip reason. Retention (`autoBackupRetention`) is still computed from project backups; when an old project backup is pruned, its matching `fusion-central-*` sibling is pruned too. Restoring a project backup also restores the paired central backup when available; restoring a `fusion-central-*` file restores the central DB only. Pre-restore snapshots use `fusion-pre-restore-<timestamp>.db` and `fusion-central-pre-restore-<timestamp>.db`.
|
||||||
|
|
||||||
|
Database Backup automation failures are surfaced with DB-qualified detail. Project backup failures include the project DB source path, backup target or backup directory when available, and the underlying cause; central DB sub-failures keep the project backup run successful but include `Central DB backup failed` plus central source/target/cause detail in the run output.
|
||||||
|
|
||||||
## 4) SQLite Tables Inventory (`packages/core/src/db.ts`)
|
## 4) SQLite Tables Inventory (`packages/core/src/db.ts`)
|
||||||
|
|
||||||
| Table | Purpose |
|
| Table | Purpose |
|
||||||
|
|||||||
@@ -878,7 +878,24 @@ describe("runBackupCommand", () => {
|
|||||||
expect(result.output).toContain("Backup created");
|
expect(result.output).toContain("Backup created");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("should return failure for invalid schedule", async () => {
|
it("reports central DB missing as an explicit successful skip", async () => {
|
||||||
|
const settings: ProjectSettings = {
|
||||||
|
maxConcurrent: 2,
|
||||||
|
maxWorktrees: 4,
|
||||||
|
pollIntervalMs: 15000,
|
||||||
|
groupOverlappingFiles: false,
|
||||||
|
autoMerge: true,
|
||||||
|
autoBackupEnabled: true,
|
||||||
|
autoBackupRetention: 7,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await runBackupCommand(fusionDir, settings);
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.output).toContain("Central DB skipped: missing");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns DB-qualified failure for invalid schedule", async () => {
|
||||||
const settings: ProjectSettings = {
|
const settings: ProjectSettings = {
|
||||||
maxConcurrent: 2,
|
maxConcurrent: 2,
|
||||||
maxWorktrees: 4,
|
maxWorktrees: 4,
|
||||||
@@ -892,7 +909,9 @@ describe("runBackupCommand", () => {
|
|||||||
const result = await runBackupCommand(fusionDir, settings);
|
const result = await runBackupCommand(fusionDir, settings);
|
||||||
|
|
||||||
expect(result.success).toBe(false);
|
expect(result.success).toBe(false);
|
||||||
expect(result.output).toContain("Invalid backup schedule");
|
expect(result.output).toContain("project DB");
|
||||||
|
expect(result.output).toContain(join(fusionDir, "fusion.db"));
|
||||||
|
expect(result.output).toContain("invalid cron expression: invalid-cron");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("should cleanup old backups after creation", async () => {
|
it("should cleanup old backups after creation", async () => {
|
||||||
@@ -921,7 +940,7 @@ describe("runBackupCommand", () => {
|
|||||||
expect(result.deletedCount).toBeGreaterThanOrEqual(1);
|
expect(result.deletedCount).toBeGreaterThanOrEqual(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("should report central copy failure while keeping success true", async () => {
|
it("reports central copy failure with DB and path detail while keeping success true", async () => {
|
||||||
const settings: ProjectSettings = {
|
const settings: ProjectSettings = {
|
||||||
maxConcurrent: 2,
|
maxConcurrent: 2,
|
||||||
maxWorktrees: 4,
|
maxWorktrees: 4,
|
||||||
@@ -938,9 +957,13 @@ describe("runBackupCommand", () => {
|
|||||||
|
|
||||||
expect(result.success).toBe(true);
|
expect(result.success).toBe(true);
|
||||||
expect(result.output).toContain("Central DB backup failed");
|
expect(result.output).toContain("Central DB backup failed");
|
||||||
|
expect(result.output).toContain("central DB");
|
||||||
|
expect(result.output).toContain("source:");
|
||||||
|
expect(result.output).toContain("target:");
|
||||||
|
expect(result.output).toContain("cause:");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("should return failure when database file is missing", async () => {
|
it("returns DB-qualified failure when the project database file is missing", async () => {
|
||||||
// Remove the database
|
// Remove the database
|
||||||
await rm(join(fusionDir, "fusion.db"));
|
await rm(join(fusionDir, "fusion.db"));
|
||||||
|
|
||||||
@@ -956,6 +979,87 @@ describe("runBackupCommand", () => {
|
|||||||
const result = await runBackupCommand(fusionDir, settings);
|
const result = await runBackupCommand(fusionDir, settings);
|
||||||
|
|
||||||
expect(result.success).toBe(false);
|
expect(result.success).toBe(false);
|
||||||
expect(result.output).toContain("failed");
|
expect(result.output).toContain("project DB");
|
||||||
|
expect(result.output).toContain(`source: ${join(fusionDir, "fusion.db")}`);
|
||||||
|
expect(result.output).toContain("target:");
|
||||||
|
expect(result.output).toContain("cause:");
|
||||||
|
expect(result.output).not.toMatch(/Backup failed:\s*$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns DB-qualified failure when the backup directory cannot be created", async () => {
|
||||||
|
const blockedBackupDir = join(tempDir, "blocked-backups");
|
||||||
|
writeFileSync(blockedBackupDir, "not a directory");
|
||||||
|
const settings: ProjectSettings = {
|
||||||
|
maxConcurrent: 2,
|
||||||
|
maxWorktrees: 4,
|
||||||
|
pollIntervalMs: 15000,
|
||||||
|
groupOverlappingFiles: false,
|
||||||
|
autoMerge: true,
|
||||||
|
autoBackupEnabled: true,
|
||||||
|
autoBackupDir: "blocked-backups",
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await runBackupCommand(fusionDir, settings);
|
||||||
|
|
||||||
|
expect(result.success).toBe(false);
|
||||||
|
expect(result.output).toContain("project DB");
|
||||||
|
expect(result.output).toContain(`source: ${join(fusionDir, "fusion.db")}`);
|
||||||
|
expect(result.output).toContain(`backup directory: ${blockedBackupDir}`);
|
||||||
|
expect(result.output).toContain("cause:");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns DB-qualified failure when project backup verification quarantines a corrupt copy", async () => {
|
||||||
|
const probe = spawnSync("sqlite3", ["--version"], { encoding: "utf-8" });
|
||||||
|
if (probe.error || probe.status !== 0) return;
|
||||||
|
writeFileSync(join(fusionDir, "fusion.db"), "not sqlite");
|
||||||
|
const settings: ProjectSettings = {
|
||||||
|
maxConcurrent: 2,
|
||||||
|
maxWorktrees: 4,
|
||||||
|
pollIntervalMs: 15000,
|
||||||
|
groupOverlappingFiles: false,
|
||||||
|
autoMerge: true,
|
||||||
|
autoBackupEnabled: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await runBackupCommand(fusionDir, settings);
|
||||||
|
|
||||||
|
expect(result.success).toBe(false);
|
||||||
|
expect(result.output).toContain("project DB");
|
||||||
|
expect(result.output).toContain(`source: ${join(fusionDir, "fusion.db")}`);
|
||||||
|
expect(result.output).toContain("quarantined as *.corrupt");
|
||||||
|
expect(result.output).toContain("cause:");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not report sqlite3-unavailable verification degradation as a backup failure", async () => {
|
||||||
|
vi.resetModules();
|
||||||
|
vi.doMock("node:child_process", () => ({
|
||||||
|
spawnSync: vi.fn(() => ({
|
||||||
|
error: Object.assign(new Error("spawn sqlite3 ENOENT"), { code: "ENOENT" }),
|
||||||
|
stdout: "",
|
||||||
|
stderr: "",
|
||||||
|
status: null,
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
try {
|
||||||
|
const { runBackupCommand: runBackupCommandWithMissingSqlite } = await import("../backup.js");
|
||||||
|
writeFileSync(join(fusionDir, "fusion.db"), "not sqlite but sqlite3 is unavailable");
|
||||||
|
const settings: ProjectSettings = {
|
||||||
|
maxConcurrent: 2,
|
||||||
|
maxWorktrees: 4,
|
||||||
|
pollIntervalMs: 15000,
|
||||||
|
groupOverlappingFiles: false,
|
||||||
|
autoMerge: true,
|
||||||
|
autoBackupEnabled: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await runBackupCommandWithMissingSqlite(fusionDir, settings);
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.output).toContain("Backup created");
|
||||||
|
expect(result.output).not.toContain("failed");
|
||||||
|
} finally {
|
||||||
|
vi.doUnmock("node:child_process");
|
||||||
|
vi.resetModules();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -67,7 +67,16 @@ export class BackupManager {
|
|||||||
async createBackup(): Promise<BackupInfo> {
|
async createBackup(): Promise<BackupInfo> {
|
||||||
const sourcePath = join(this.fusionDir, "fusion.db");
|
const sourcePath = join(this.fusionDir, "fusion.db");
|
||||||
const backupDirPath = this.getBackupDirPath();
|
const backupDirPath = this.getBackupDirPath();
|
||||||
await mkdir(backupDirPath, { recursive: true });
|
try {
|
||||||
|
await mkdir(backupDirPath, { recursive: true });
|
||||||
|
} catch (err) {
|
||||||
|
throw new Error(formatBackupError({
|
||||||
|
dbLabel: "project DB",
|
||||||
|
action: "prepare backup directory",
|
||||||
|
backupDirPath,
|
||||||
|
cause: err,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
const timestamp = currentBackupTimestamp();
|
const timestamp = currentBackupTimestamp();
|
||||||
let counter = 0;
|
let counter = 0;
|
||||||
@@ -96,21 +105,31 @@ export class BackupManager {
|
|||||||
const filename = generateBackupFilename(timestamp, counter);
|
const filename = generateBackupFilename(timestamp, counter);
|
||||||
const targetPath = join(backupDirPath, filename);
|
const targetPath = join(backupDirPath, filename);
|
||||||
|
|
||||||
await copyLiveDatabase(sourcePath, targetPath);
|
try {
|
||||||
|
await copyLiveDatabase(sourcePath, targetPath);
|
||||||
|
|
||||||
// Verify the freshly-written copy. A copy of a live WAL db can capture a
|
// Verify the freshly-written copy. A copy of a live WAL db can capture a
|
||||||
// torn/corrupt main file; refusing to keep a corrupt backup guarantees
|
// torn/corrupt main file; refusing to keep a corrupt backup guarantees
|
||||||
// that every retained `fusion-*.db` is restorable and that a corrupt copy
|
// that every retained `fusion-*.db` is restorable and that a corrupt copy
|
||||||
// is never counted as the "last known-good" by cleanupOldBackups().
|
// is never counted as the "last known-good" by cleanupOldBackups().
|
||||||
if (this.verifyIntegrity) {
|
if (this.verifyIntegrity) {
|
||||||
const integrity = verifyDatabaseIntegrity(targetPath);
|
const integrity = verifyDatabaseIntegrity(targetPath);
|
||||||
if (!integrity.ok) {
|
if (!integrity.ok) {
|
||||||
await quarantineCorruptBackup(targetPath);
|
await quarantineCorruptBackup(targetPath);
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Backup verification failed for ${filename}: ${integrity.error ?? "database disk image is malformed"}. ` +
|
`verification failed: ${integrity.error ?? "database disk image is malformed"}. ` +
|
||||||
"The source database may be corrupt; the unusable copy was quarantined as *.corrupt.",
|
"The source database may be corrupt; the unusable copy was quarantined as *.corrupt.",
|
||||||
);
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
} catch (err) {
|
||||||
|
throw new Error(formatBackupError({
|
||||||
|
dbLabel: "project DB",
|
||||||
|
action: "create backup",
|
||||||
|
sourcePath,
|
||||||
|
targetPath,
|
||||||
|
cause: err,
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
const stats = await stat(targetPath);
|
const stats = await stat(targetPath);
|
||||||
@@ -142,7 +161,8 @@ export class BackupManager {
|
|||||||
if (!centralIntegrity.ok) {
|
if (!centralIntegrity.ok) {
|
||||||
await quarantineCorruptBackup(centralTargetPath);
|
await quarantineCorruptBackup(centralTargetPath);
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`central DB verification failed: ${centralIntegrity.error ?? "database disk image is malformed"}`,
|
`verification failed: ${centralIntegrity.error ?? "database disk image is malformed"}. ` +
|
||||||
|
"The source database may be corrupt; the unusable copy was quarantined as *.corrupt.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -156,7 +176,13 @@ export class BackupManager {
|
|||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
backup.centralBackup = {
|
backup.centralBackup = {
|
||||||
failed: (err as Error).message,
|
failed: formatBackupError({
|
||||||
|
dbLabel: "central DB",
|
||||||
|
action: "create backup",
|
||||||
|
sourcePath: this.centralDbPath,
|
||||||
|
targetPath: centralTargetPath,
|
||||||
|
cause: err,
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -573,10 +599,16 @@ export async function runBackupCommand(
|
|||||||
fusionDir: string,
|
fusionDir: string,
|
||||||
settings: ProjectSettings
|
settings: ProjectSettings
|
||||||
): Promise<{ success: boolean; output: string; backupPath?: string; deletedCount?: number }> {
|
): Promise<{ success: boolean; output: string; backupPath?: string; deletedCount?: number }> {
|
||||||
|
const projectDbPath = join(fusionDir, "fusion.db");
|
||||||
if (settings.autoBackupSchedule && !validateBackupSchedule(settings.autoBackupSchedule)) {
|
if (settings.autoBackupSchedule && !validateBackupSchedule(settings.autoBackupSchedule)) {
|
||||||
return {
|
return {
|
||||||
success: false,
|
success: false,
|
||||||
output: `Invalid backup schedule: ${settings.autoBackupSchedule}`,
|
output: formatBackupError({
|
||||||
|
dbLabel: "project DB",
|
||||||
|
action: "validate backup schedule",
|
||||||
|
sourcePath: projectDbPath,
|
||||||
|
cause: `invalid cron expression: ${settings.autoBackupSchedule}`,
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -613,11 +645,55 @@ export async function runBackupCommand(
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
success: false,
|
success: false,
|
||||||
output: `Backup failed: ${(err as Error).message}`,
|
output: formatBackupError({
|
||||||
|
dbLabel: "project DB",
|
||||||
|
action: "run backup command",
|
||||||
|
sourcePath: projectDbPath,
|
||||||
|
cause: err,
|
||||||
|
}),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:DatabaseBackup 2026-06-26-12:00:
|
||||||
|
Database Backup automations are operator-facing data-safety signals. Every failure must name the affected DB, relevant path, and cause so CLI, dashboard, routine, and cron surfaces never persist a detail-less "Backup failed" result.
|
||||||
|
*/
|
||||||
|
function formatBackupError(input: {
|
||||||
|
dbLabel: "project DB" | "central DB";
|
||||||
|
action: string;
|
||||||
|
sourcePath?: string;
|
||||||
|
targetPath?: string;
|
||||||
|
backupDirPath?: string;
|
||||||
|
cause: unknown;
|
||||||
|
}): string {
|
||||||
|
const parts = [`${input.dbLabel} ${input.action} failed`];
|
||||||
|
if (input.sourcePath) parts.push(`source: ${input.sourcePath}`);
|
||||||
|
if (input.targetPath) parts.push(`target: ${input.targetPath}`);
|
||||||
|
if (input.backupDirPath) parts.push(`backup directory: ${input.backupDirPath}`);
|
||||||
|
parts.push(`cause: ${describeError(input.cause)}`);
|
||||||
|
return parts.join("; ");
|
||||||
|
}
|
||||||
|
|
||||||
|
function describeError(err: unknown): string {
|
||||||
|
if (err instanceof Error) {
|
||||||
|
return err.message.trim() || err.name || "unknown error";
|
||||||
|
}
|
||||||
|
if (typeof err === "string") {
|
||||||
|
return err.trim() || "unknown error";
|
||||||
|
}
|
||||||
|
if (err === null || err === undefined) {
|
||||||
|
return "unknown error";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const serialized = JSON.stringify(err);
|
||||||
|
if (serialized && serialized !== "{}") return serialized;
|
||||||
|
} catch {
|
||||||
|
// Fall through to String().
|
||||||
|
}
|
||||||
|
return String(err).trim() || "unknown error";
|
||||||
|
}
|
||||||
|
|
||||||
function formatBytes(bytes: number): string {
|
function formatBytes(bytes: number): string {
|
||||||
if (bytes === 0) return "0 B";
|
if (bytes === 0) return "0 B";
|
||||||
const k = 1024;
|
const k = 1024;
|
||||||
|
|||||||
@@ -2038,6 +2038,41 @@ describe("CronRunner", () => {
|
|||||||
expect(runResult.success).toBe(true);
|
expect(runResult.success).toBe(true);
|
||||||
expect(runResult.output).toContain("fusion-central-");
|
expect(runResult.output).toContain("fusion-central-");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("normalizes empty runBackupCommand failure output for legacy schedules", async () => {
|
||||||
|
coreModuleMocks.runBackupCommand.mockResolvedValueOnce({ success: false, output: "" });
|
||||||
|
const store = createMockStore() as unknown as TaskStore & { getFusionDir: () => string };
|
||||||
|
store.getFusionDir = vi.fn().mockReturnValue("/tmp/.fusion");
|
||||||
|
const schedule = createMockSchedule({ id: "db-bk-empty", command: "fn backup --create" });
|
||||||
|
runner = new CronRunner(store, createMockAutomationStore());
|
||||||
|
|
||||||
|
const runResult = await (runner as unknown as { executeLegacyCommand: (s: ScheduledTask, startedAt: string) => Promise<AutomationRunResult> })
|
||||||
|
.executeLegacyCommand(schedule, new Date().toISOString());
|
||||||
|
|
||||||
|
expect(runResult.success).toBe(false);
|
||||||
|
expect(runResult.error).toBe("project DB run backup command failed; source: /tmp/.fusion/fusion.db; cause: unknown error");
|
||||||
|
expect(runResult.output).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("normalizes empty thrown errors for backup command steps", async () => {
|
||||||
|
coreModuleMocks.runBackupCommand.mockRejectedValueOnce(new Error(""));
|
||||||
|
const store = createMockStore() as unknown as TaskStore & { getFusionDir: () => string };
|
||||||
|
store.getFusionDir = vi.fn().mockReturnValue("/tmp/.fusion");
|
||||||
|
runner = new CronRunner(store, createMockAutomationStore());
|
||||||
|
const step: AutomationStep = {
|
||||||
|
id: "step-db-bk",
|
||||||
|
name: "Database Backup",
|
||||||
|
type: "command",
|
||||||
|
command: "fn backup --create",
|
||||||
|
};
|
||||||
|
|
||||||
|
const stepResult = await (runner as unknown as { executeCommandStep: (s: AutomationStep, i: number, t: number, startedAt: string) => Promise<AutomationRunResult> })
|
||||||
|
.executeCommandStep(step, 0, 30_000, new Date().toISOString());
|
||||||
|
|
||||||
|
expect(stepResult.success).toBe(false);
|
||||||
|
expect(stepResult.error).toBe("project DB run backup command failed; source: /tmp/.fusion/fusion.db; cause: unknown error");
|
||||||
|
expect(stepResult.output).toBe("");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("memory backup in-process dispatch", () => {
|
describe("memory backup in-process dispatch", () => {
|
||||||
|
|||||||
@@ -9,6 +9,10 @@ import type {
|
|||||||
Settings,
|
Settings,
|
||||||
} from "@fusion/core";
|
} from "@fusion/core";
|
||||||
import type { HeartbeatMonitor } from "../agent-heartbeat.js";
|
import type { HeartbeatMonitor } from "../agent-heartbeat.js";
|
||||||
|
import { mkdtempSync } from "node:fs";
|
||||||
|
import { mkdir, rm } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
// Default settings inline to avoid @fusion/core build dependency during tests
|
// Default settings inline to avoid @fusion/core build dependency during tests
|
||||||
const DEFAULT_SETTINGS: Settings = {
|
const DEFAULT_SETTINGS: Settings = {
|
||||||
@@ -111,9 +115,10 @@ function createMockAgentStore(): AgentStore {
|
|||||||
} as unknown as AgentStore;
|
} as unknown as AgentStore;
|
||||||
}
|
}
|
||||||
|
|
||||||
function createMockTaskStore(): TaskStore {
|
function createMockTaskStore(overrides: { fusionDir?: string; settings?: Partial<Settings> } = {}): TaskStore {
|
||||||
return {
|
return {
|
||||||
getSettings: vi.fn().mockResolvedValue(DEFAULT_SETTINGS),
|
getFusionDir: vi.fn().mockReturnValue(overrides.fusionDir ?? "/tmp/.fusion"),
|
||||||
|
getSettings: vi.fn().mockResolvedValue({ ...DEFAULT_SETTINGS, ...overrides.settings }),
|
||||||
on: vi.fn(),
|
on: vi.fn(),
|
||||||
off: vi.fn(),
|
off: vi.fn(),
|
||||||
} as unknown as TaskStore;
|
} as unknown as TaskStore;
|
||||||
@@ -238,6 +243,42 @@ describe("RoutineRunner", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("persists an actionable error for in-process Database Backup failures", async () => {
|
||||||
|
const tempDir = mkdtempSync(join(tmpdir(), "routine-backup-detail-"));
|
||||||
|
const fusionDir = join(tempDir, ".fusion");
|
||||||
|
await mkdir(fusionDir, { recursive: true });
|
||||||
|
const routine = createMockRoutine({
|
||||||
|
id: "routine-backup-missing-db",
|
||||||
|
command: "fn backup --create",
|
||||||
|
agentId: "",
|
||||||
|
});
|
||||||
|
const routineStore = createMockRoutineStore([routine]);
|
||||||
|
const runner = createRoutineRunner({
|
||||||
|
routineStore,
|
||||||
|
taskStore: createMockTaskStore({ fusionDir }),
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await runner.executeRoutine("routine-backup-missing-db", "cron");
|
||||||
|
|
||||||
|
expect(result.success).toBe(false);
|
||||||
|
expect(result.error).toContain("project DB");
|
||||||
|
expect(result.error).toContain(`source: ${join(fusionDir, "fusion.db")}`);
|
||||||
|
expect(result.error).toContain("cause:");
|
||||||
|
expect(result.error).not.toBe("");
|
||||||
|
expect(routineStore.completeRoutineExecution).toHaveBeenCalledWith(
|
||||||
|
"routine-backup-missing-db",
|
||||||
|
expect.objectContaining({
|
||||||
|
success: false,
|
||||||
|
error: result.error,
|
||||||
|
output: expect.stringContaining("project DB"),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await rm(tempDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("marks execution as failed when executeHeartbeat rejects", async () => {
|
it("marks execution as failed when executeHeartbeat rejects", async () => {
|
||||||
const routine = createMockRoutine({ id: "routine-fail" });
|
const routine = createMockRoutine({ id: "routine-fail" });
|
||||||
const routineStore = createMockRoutineStore([routine]);
|
const routineStore = createMockRoutineStore([routine]);
|
||||||
|
|||||||
@@ -608,13 +608,14 @@ export class CronRunner {
|
|||||||
const fusionDir = this.store.getFusionDir();
|
const fusionDir = this.store.getFusionDir();
|
||||||
const settings = await this.store.getSettings();
|
const settings = await this.store.getSettings();
|
||||||
const result = await runBackupCommand(fusionDir, settings);
|
const result = await runBackupCommand(fusionDir, settings);
|
||||||
|
const output = truncateOutput(result.output ?? "", "");
|
||||||
return {
|
return {
|
||||||
success: result.success,
|
success: result.success,
|
||||||
output: truncateOutput(result.output ?? "", ""),
|
output,
|
||||||
error: result.success ? undefined : result.output,
|
error: result.success ? undefined : formatInProcessBackupError(output, fusionDir),
|
||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
const message = formatInProcessBackupError(err, this.store.getFusionDir());
|
||||||
return { success: false, output: "", error: message };
|
return { success: false, output: "", error: message };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1056,6 +1057,19 @@ export async function createAiPromptExecutor(cwd: string, store?: TaskStore): Pr
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:DatabaseBackup 2026-06-26-12:00:
|
||||||
|
Cron-runner in-process backups feed automation run history and step errors. Normalize empty thrown values and empty command output before they become operator-visible Database Backup failures.
|
||||||
|
*/
|
||||||
|
function formatInProcessBackupError(err: unknown, fusionDir: string): string {
|
||||||
|
const message = err instanceof Error ? err.message.trim() : String(err ?? "").trim();
|
||||||
|
const cause = message || "unknown error";
|
||||||
|
if (cause.includes("project DB") || cause.includes("central DB")) {
|
||||||
|
return cause;
|
||||||
|
}
|
||||||
|
return `project DB run backup command failed; source: ${fusionDir}/fusion.db; cause: ${cause}`;
|
||||||
|
}
|
||||||
|
|
||||||
/** Combine and truncate stdout/stderr to stay within storage limits. */
|
/** Combine and truncate stdout/stderr to stay within storage limits. */
|
||||||
function truncateOutput(stdout: string | null | undefined, stderr: string | null | undefined): string {
|
function truncateOutput(stdout: string | null | undefined, stderr: string | null | undefined): string {
|
||||||
const out = stdout ?? "";
|
const out = stdout ?? "";
|
||||||
|
|||||||
@@ -306,15 +306,16 @@ export class RoutineRunner {
|
|||||||
const fusionDir = this.options.taskStore.getFusionDir();
|
const fusionDir = this.options.taskStore.getFusionDir();
|
||||||
const settings = await this.options.taskStore.getSettings();
|
const settings = await this.options.taskStore.getSettings();
|
||||||
const result = await runBackupCommand(fusionDir, settings);
|
const result = await runBackupCommand(fusionDir, settings);
|
||||||
|
const output = truncateOutput(result.output ?? "", "");
|
||||||
return {
|
return {
|
||||||
success: result.success,
|
success: result.success,
|
||||||
output: truncateOutput(result.output ?? "", ""),
|
output,
|
||||||
error: result.success ? undefined : result.output,
|
error: result.success ? undefined : formatInProcessBackupError(output, fusionDir),
|
||||||
startedAt,
|
startedAt,
|
||||||
completedAt: new Date().toISOString(),
|
completedAt: new Date().toISOString(),
|
||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
const message = formatInProcessBackupError(err, this.options.taskStore.getFusionDir());
|
||||||
return {
|
return {
|
||||||
success: false,
|
success: false,
|
||||||
output: "",
|
output: "",
|
||||||
@@ -633,6 +634,19 @@ export class RoutineRunner {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:DatabaseBackup 2026-06-26-12:00:
|
||||||
|
Routine-runner in-process backups persist AutomationRunResult.error directly to lastRunResult. Normalize empty or opaque failures here so Database Backup cards always show a DB-qualified cause.
|
||||||
|
*/
|
||||||
|
function formatInProcessBackupError(err: unknown, fusionDir: string): string {
|
||||||
|
const message = err instanceof Error ? err.message.trim() : String(err ?? "").trim();
|
||||||
|
const cause = message || "unknown error";
|
||||||
|
if (cause.includes("project DB") || cause.includes("central DB")) {
|
||||||
|
return cause;
|
||||||
|
}
|
||||||
|
return `project DB run backup command failed; source: ${fusionDir}/fusion.db; cause: ${cause}`;
|
||||||
|
}
|
||||||
|
|
||||||
function truncateOutput(stdout: string, stderr: string): string {
|
function truncateOutput(stdout: string, stderr: string): string {
|
||||||
let output = stdout;
|
let output = stdout;
|
||||||
if (stderr) {
|
if (stderr) {
|
||||||
|
|||||||
Reference in New Issue
Block a user