From ad744aa28a684ec246b0143af70f1aaf7d5cf836 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 4 Jul 2026 16:25:34 -0700 Subject: [PATCH] FN-7537: make manual backup automation runs match cron in-process behavior Manual 'Run now' automation runs previously always shelled out via exec(), diverging from the scheduled cron path which runs backup commands in-process; this unifies both paths and adds live-run output coverage. - Export formatInProcessBackupError, isInProcessBackupCommand, and isInProcessMemoryBackupCommand from @fusion/engine for reuse - Have the dashboard's single-command/command-step manual run path (executeSingleCommand in routes.ts) intercept in-process backup/memory-backup commands via the scoped TaskStore, mirroring RoutineRunner.executeCommand/CronRunner - Add regression coverage confirming onStep/onText live-run callbacks stream incremental output for the new interception branch - Add changeset and doc note for the fix Files changed: .changeset/fn-7537-backup-automation-manual-run.md | 7 + docs/dashboard-guide.md | 3 + .../src/__tests__/routes-automation.test.ts | 171 +++++++++++++++++++++ packages/dashboard/src/routes.ts | 73 ++++++++- .../engine/src/__tests__/routine-runner.test.ts | 96 +++++++++++- packages/engine/src/cron-runner.ts | 7 +- packages/engine/src/index.ts | 2 +- 7 files changed, 354 insertions(+), 5 deletions(-) Fusion-Task-Id: FN-7537 Fusion-Task-Lineage: 47824270-c0d4-471b-a5c9-f5350176df29 Co-authored-by: Fusion (runfusion.ai) --- .../fn-7537-backup-automation-manual-run.md | 7 + docs/dashboard-guide.md | 3 + .../src/__tests__/routes-automation.test.ts | 171 ++++++++++++++++++ packages/dashboard/src/routes.ts | 73 +++++++- .../src/__tests__/routine-runner.test.ts | 96 +++++++++- packages/engine/src/cron-runner.ts | 7 +- packages/engine/src/index.ts | 2 +- 7 files changed, 354 insertions(+), 5 deletions(-) create mode 100644 .changeset/fn-7537-backup-automation-manual-run.md diff --git a/.changeset/fn-7537-backup-automation-manual-run.md b/.changeset/fn-7537-backup-automation-manual-run.md new file mode 100644 index 0000000000..d367f5d451 --- /dev/null +++ b/.changeset/fn-7537-backup-automation-manual-run.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Manual "Run now" for the Database Backup automation now runs in-process like the scheduler, matching cron behavior. +category: fix +dev: The legacy single-command and command-step manual automation run path (`executeSingleCommand` in packages/dashboard/src/routes.ts) now intercepts `isInProcessBackupCommand`/`isInProcessMemoryBackupCommand` via the scoped TaskStore, mirroring `RoutineRunner.executeCommand`/`CronRunner`, instead of always shelling out via `exec()`. `formatInProcessBackupError`, `isInProcessBackupCommand`, and `isInProcessMemoryBackupCommand` are now exported from `@fusion/engine` for reuse. Existing onStep/onText live-run callbacks already stream incremental output for command/backup runs; added regression coverage confirming this holds for the new interception branch. diff --git a/docs/dashboard-guide.md b/docs/dashboard-guide.md index c03740ec54..025a6cfe84 100644 --- a/docs/dashboard-guide.md +++ b/docs/dashboard-guide.md @@ -129,6 +129,9 @@ Open **Automations** from the left sidebar (or the mobile More surfaces) to crea When you choose **Run now**, the routine card opens a **Live output** panel while the manual run is active. The panel appends step status, AI text deltas, and tool start/finish activity as the run executes, then the card falls back to the persisted final run output and run history once the server records the result. The same `RoutineCard` surface is used by the floating modal and embedded Automations view, so live output appears in both presentations and collapses into a single-column card layout on mobile. + +The built-in **Database Backup** automation runs the backup in-process (via the engine's already-open task store) on both its scheduled cron trigger and a manual **Run now**, matching behavior identically between the two triggers — it never shells out to a separately-installed `fn`/`runfusion.ai` binary, which could be missing or out of date on the host. + ## Deep Links Use deep links to open a specific task directly from notifications, chat, or external tools. diff --git a/packages/dashboard/src/__tests__/routes-automation.test.ts b/packages/dashboard/src/__tests__/routes-automation.test.ts index 006aa2a18b..a5164fcbe5 100644 --- a/packages/dashboard/src/__tests__/routes-automation.test.ts +++ b/packages/dashboard/src/__tests__/routes-automation.test.ts @@ -110,7 +110,17 @@ vi.mock("@fusion/core", async (importOriginal) => { vi.mock("@fusion/engine", async () => { const { createEngineMock } = await import("../test/mockCoreEngine.js"); + // FNXC:DatabaseBackup 2026-07-04-00:00: + // FN-7537: route.ts's manual-run in-process backup interception statically imports + // isInProcessBackupCommand/isInProcessMemoryBackupCommand/formatInProcessBackupError from + // @fusion/engine. This module mock has no real `actual` behind it (see mockCoreEngine.js), so those + // three must be the REAL implementations (via vi.importActual) rather than the default vi.fn() + // fallback, or the interception would never match any command in these route tests. + const actualEngine = await vi.importActual("@fusion/engine"); return createEngineMock({ + isInProcessBackupCommand: actualEngine.isInProcessBackupCommand, + isInProcessMemoryBackupCommand: actualEngine.isInProcessMemoryBackupCommand, + formatInProcessBackupError: actualEngine.formatInProcessBackupError, createFnAgent: vi.fn(async (options?: { onText?: (delta: string) => void; onToolStart?: (name: string, args?: Record) => void; onToolEnd?: (name: string, isError: boolean, result?: unknown) => void }) => ({ session: { state: { @@ -1176,6 +1186,167 @@ describe("Automation routes", () => { const res = await REQUEST(app, "POST", "/api/automations/missing/run"); expect(res.status).toBe(404); }); + + /* + FNXC:DatabaseBackup 2026-07-04-00:00: + FN-7537 Symptom Verification: the reported symptom was the "Database Backup" automation failing on a + manual dashboard run while succeeding via cron. These tests drive the exact manual endpoint + (POST /automations/:id/run) that previously always shelled out via exec() for every command — + including the persisted `fn backup --create` command — and assert it now intercepts the in-process + backup (matching the cron path's CronRunner.executeBackupInProcess result) instead of failing on hosts + without a global `fn`/`runfusion.ai` binary. + */ + describe("manual backup run parity (FN-7537)", () => { + function writeTestDb(path: string): void { + try { + execFileSync("sqlite3", [path, "CREATE TABLE IF NOT EXISTS t(x); INSERT INTO t VALUES (1);"]); + } catch { + writeFileSync(path, "dummy database content"); + } + } + + it("intercepts the in-process backup for a legacy single-command schedule and does not shell out", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "routes-automation-backup-")); + const fusionDir = join(tempDir, ".fusion"); + mkdirSync(fusionDir, { recursive: true }); + writeTestDb(join(fusionDir, "fusion.db")); + + try { + const mockStore = createMockAutomationStore(); + mockStore.getSchedule.mockResolvedValue({ + ...FAKE_SCHEDULE, + command: "fn backup --create", + }); + const store = createMockStore({ getFusionDir: vi.fn().mockReturnValue(fusionDir) } as any); + const app = express(); + app.use(express.json()); + app.use("/api", createApiRoutes(store, { automationStore: mockStore as any })); + mockExecFile.mockClear(); + + const res = await REQUEST(app, "POST", "/api/automations/sched-001/run"); + + expect(res.status).toBe(200); + expect(res.body.result.success).toBe(true); + expect(res.body.result.output).toContain("Backup created"); + // A shelled-out command would have gone through node:child_process exec — assert it did not. + expect(mockExecFile).not.toHaveBeenCalledWith( + expect.anything(), + expect.arrayContaining([expect.stringContaining("backup")]), + expect.anything(), + expect.anything(), + ); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("intercepts the in-process backup for a command-type schedule step", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "routes-automation-backup-step-")); + const fusionDir = join(tempDir, ".fusion"); + mkdirSync(fusionDir, { recursive: true }); + writeTestDb(join(fusionDir, "fusion.db")); + + try { + const mockStore = createMockAutomationStore(); + mockStore.getSchedule.mockResolvedValue({ + ...FAKE_SCHEDULE, + command: "", + steps: [ + { + id: "step-backup", + type: "command", + name: "Run backup", + command: "fn backup --create", + }, + ], + }); + const store = createMockStore({ getFusionDir: vi.fn().mockReturnValue(fusionDir) } as any); + const app = express(); + app.use(express.json()); + app.use("/api", createApiRoutes(store, { automationStore: mockStore as any })); + + const res = await REQUEST(app, "POST", "/api/automations/sched-001/run"); + + expect(res.status).toBe(200); + expect(res.body.result.stepResults[0]).toEqual( + expect.objectContaining({ stepName: "Run backup", success: true, output: expect.stringContaining("Backup created") }), + ); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("streams a step-start live event before the terminal complete event for a manual backup run", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "routes-automation-backup-live-")); + const fusionDir = join(tempDir, ".fusion"); + mkdirSync(fusionDir, { recursive: true }); + writeTestDb(join(fusionDir, "fusion.db")); + + try { + const mockStore = createMockAutomationStore(); + mockStore.getSchedule.mockResolvedValue({ + ...FAKE_SCHEDULE, + command: "fn backup --create", + }); + const store = createMockStore({ getFusionDir: vi.fn().mockReturnValue(fusionDir) } as any); + const app = express(); + app.use(express.json()); + app.use("/api", createApiRoutes(store, { automationStore: mockStore as any })); + + const runRes = await REQUEST(app, "POST", "/api/automations/sched-001/run"); + expect(runRes.status).toBe(200); + expect(runRes.body.result.success).toBe(true); + + const streamRes = await performRequest(app, "GET", `/api/automations/sched-001/run/stream?runId=${runRes.body.liveRunId}`); + expect(streamRes.status).toBe(200); + const body = String(streamRes.body); + const stepIndex = body.indexOf("event: step"); + const completeIndex = body.indexOf("event: complete"); + expect(stepIndex).toBeGreaterThanOrEqual(0); + expect(completeIndex).toBeGreaterThan(stepIndex); + expect(body).toContain("\"status\":\"started\""); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("keeps memory-backup command parity on the manual run path", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "routes-automation-membackup-")); + const fusionDir = join(tempDir, ".fusion"); + mkdirSync(fusionDir, { recursive: true }); + + try { + const mockStore = createMockAutomationStore(); + mockStore.getSchedule.mockResolvedValue({ + ...FAKE_SCHEDULE, + command: "fn memory-backup --create", + }); + const store = createMockStore({ + getFusionDir: vi.fn().mockReturnValue(fusionDir), + getSettings: vi.fn().mockResolvedValue({ memoryEnabled: false }), + } as any); + const app = express(); + app.use(express.json()); + app.use("/api", createApiRoutes(store, { automationStore: mockStore as any })); + mockExecFile.mockClear(); + + const res = await REQUEST(app, "POST", "/api/automations/sched-001/run"); + + expect(res.status).toBe(200); + // Memory backups are a no-op success when memory is disabled; the important assertion is that + // the in-process path ran (not a shell-out to a missing global binary). + expect(typeof res.body.result.success).toBe("boolean"); + expect(mockExecFile).not.toHaveBeenCalledWith( + expect.anything(), + expect.arrayContaining([expect.stringContaining("memory-backup")]), + expect.anything(), + expect.anything(), + ); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + }); }); describe("POST /automations/:id/toggle", () => { diff --git a/packages/dashboard/src/routes.ts b/packages/dashboard/src/routes.ts index 442ad265c5..262adb7a69 100644 --- a/packages/dashboard/src/routes.ts +++ b/packages/dashboard/src/routes.ts @@ -372,6 +372,9 @@ import { resolveMcpServersForRuntime, resolveMcpServersForStore, validateMcpServer, + isInProcessBackupCommand, + isInProcessMemoryBackupCommand, + formatInProcessBackupError, } from "@fusion/engine"; interface McpValidateRequestBody { @@ -2411,8 +2414,13 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout result = await executeScheduleSteps(schedule, startedAt, scopedStore, liveCallbacks); } else { // Legacy single-command execution + // FNXC:Automations 2026-07-04-00:00: + // FN-7537: command/backup runs (including the new in-process backup branch inside + // executeSingleCommand) stream through the same onStep/onText live-run callbacks as every other + // step type, so the live-output panel populates during the run (step-start immediately, output once + // available) rather than only at the terminal `complete` event. liveCallbacks.onStep?.({ stepIndex: 0, stepId: "command", stepName: schedule.name, stepType: "command", status: "started" }); - result = await executeSingleCommand(schedule.command, schedule.timeoutMs, startedAt); + result = await executeSingleCommand(schedule.command, schedule.timeoutMs, startedAt, scopedStore); liveCallbacks.onStep?.({ stepIndex: 0, stepId: "command", stepName: schedule.name, stepType: "command", status: "completed", success: result.success, error: result.error }); if (result.output) liveCallbacks.onText?.(result.output); } @@ -5151,12 +5159,73 @@ function createAutomationLiveRunCallbacks(runId: string): AutomationLiveRunCallb /** * Execute a single shell command (used by manual run endpoint). + * + * FNXC:DatabaseBackup 2026-07-04-00:00: + * FN-7537: the dashboard's manual automation/schedule run path (legacy single-command schedules and + * `command`-type steps in `executeScheduleSteps`) previously always shelled the command out via `exec()`, + * unlike the scheduler (`CronRunner`) and routine runner (`RoutineRunner.executeCommand`), which both + * intercept the auto-backup command and run it in-process via the engine's already-open `TaskStore`. On + * hosts without a global `fn`/`runfusion.ai` binary on PATH this made a manual "Database Backup" run fail + * while the identical cron-triggered run succeeded. Mirror the cron/routine-runner interception here so a + * manual run behaves identically: when a `taskStore` is available and the command matches + * `isInProcessBackupCommand`/`isInProcessMemoryBackupCommand`, run the backup in-process instead of + * shelling out, using the same `formatInProcessBackupError` message shape on failure (parity with FN-7095). */ async function executeSingleCommand( command: string, timeoutMs: number | undefined, startedAt: string, + taskStore?: TaskStore, ): Promise { + if (taskStore && isInProcessBackupCommand(command)) { + const fusionDir = taskStore.getFusionDir(); + try { + const { runBackupCommand } = await import("@fusion/core"); + const settings = await taskStore.getSettings(); + const result = await runBackupCommand(fusionDir, settings); + const output = truncateAutomationOutput(result.output ?? "", ""); + return { + success: result.success, + output, + error: result.success ? undefined : formatInProcessBackupError(output, fusionDir), + startedAt, + completedAt: new Date().toISOString(), + }; + } catch (err) { + return { + success: false, + output: "", + error: formatInProcessBackupError(err, fusionDir), + startedAt, + completedAt: new Date().toISOString(), + }; + } + } + + if (taskStore && isInProcessMemoryBackupCommand(command)) { + const fusionDir = taskStore.getFusionDir(); + try { + const { runMemoryBackupCommand } = await import("@fusion/core"); + const settings = await taskStore.getSettings(); + const result = await runMemoryBackupCommand(fusionDir, settings); + return { + success: result.success, + output: truncateAutomationOutput(result.output ?? "", ""), + error: result.success ? undefined : result.output, + startedAt, + completedAt: new Date().toISOString(), + }; + } catch (err) { + return { + success: false, + output: "", + error: err instanceof Error ? err.message : String(err), + startedAt, + completedAt: new Date().toISOString(), + }; + } + } + const { exec } = await import("node:child_process"); const { promisify } = await import("node:util"); const execAsyncFn = promisify(exec); @@ -5377,7 +5446,7 @@ async function executeScheduleSteps( liveCallbacks?.onStep?.({ stepIndex: i, stepId: step.id, stepName: step.name, stepType: step.type, status: "started" }); if (step.type === "command") { - const cmdResult = await executeSingleCommand(step.command ?? "", timeoutMs, stepStartedAt); + const cmdResult = await executeSingleCommand(step.command ?? "", timeoutMs, stepStartedAt, taskStore); stepResult = { stepId: step.id, stepName: step.name, diff --git a/packages/engine/src/__tests__/routine-runner.test.ts b/packages/engine/src/__tests__/routine-runner.test.ts index b1c8fc792f..49531320b8 100644 --- a/packages/engine/src/__tests__/routine-runner.test.ts +++ b/packages/engine/src/__tests__/routine-runner.test.ts @@ -9,10 +9,26 @@ import type { Settings, } from "@fusion/core"; import type { HeartbeatMonitor } from "../agent-heartbeat.js"; -import { mkdtempSync } from "node:fs"; +import { mkdtempSync, writeFileSync } from "node:fs"; import { mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +/** + * Write a real SQLite database file so the production backup path's `PRAGMA quick_check` + * verification passes (mirrors packages/core/src/__tests__/backup.test.ts's fixture helper). + * Falls back to a placeholder file when the `sqlite3` CLI is unavailable — in that case + * verification also no-ops so the backup still succeeds. + */ +function writeTestDb(path: string): void { + const result = spawnSync("sqlite3", [path, "CREATE TABLE IF NOT EXISTS t(x); INSERT INTO t VALUES (1);"], { + encoding: "utf-8", + }); + if (result.error || result.status !== 0) { + writeFileSync(path, "dummy database content"); + } +} // Default settings inline to avoid @fusion/core build dependency during tests const DEFAULT_SETTINGS: Settings = { @@ -351,6 +367,84 @@ describe("RoutineRunner", () => { }); }); + /* + FNXC:DatabaseBackup 2026-07-04-00:00: + FN-7537 Symptom Verification: the reported bug was "Database Backup" succeeding on cron but failing on a + manual dashboard run. Both triggers share this exact RoutineRunner.executeCommand in-process backup + branch (guarded by `isInProcessBackupCommand(command) && this.options.taskStore`), so these tests assert + the invariant directly on the shared code path for both trigger kinds ("cron" and "api", the latter being + what `triggerManual` uses) rather than only the originally-reported reproduction. + */ + describe("manual/cron backup parity and live output (FN-7537)", () => { + it("runs the in-process backup for both cron and manual (api) triggers, never shelling out", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "routine-backup-parity-")); + const fusionDir = join(tempDir, ".fusion"); + await mkdir(fusionDir, { recursive: true }); + writeTestDb(join(fusionDir, "fusion.db")); + + try { + for (const triggerType of ["cron", "api"] as const) { + const routine = createMockRoutine({ + id: `routine-backup-${triggerType}`, + command: "fn backup --create", + agentId: "", + }); + const routineStore = createMockRoutineStore([routine]); + const runner = createRoutineRunner({ + routineStore, + taskStore: createMockTaskStore({ fusionDir }), + }); + + const result = triggerType === "api" + ? await runner.triggerManual(`routine-backup-${triggerType}`) + : await runner.executeRoutine(`routine-backup-${triggerType}`, "cron"); + + expect(result.success).toBe(true); + expect(result.output).toContain("Backup created"); + } + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); + + it("emits a step-start live event before the terminal output for a manual command/backup run", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "routine-backup-live-")); + const fusionDir = join(tempDir, ".fusion"); + await mkdir(fusionDir, { recursive: true }); + writeTestDb(join(fusionDir, "fusion.db")); + + try { + const routine = createMockRoutine({ + id: "routine-backup-live", + command: "fn backup --create", + agentId: "", + }); + const routineStore = createMockRoutineStore([routine]); + const runner = createRoutineRunner({ + routineStore, + taskStore: createMockTaskStore({ fusionDir }), + }); + + const events: Array<{ kind: string; data?: unknown }> = []; + const result = await runner.triggerManual("routine-backup-live", { + onStep: (data) => events.push({ kind: "step", data }), + onText: (delta) => events.push({ kind: "text", data: delta }), + }); + + expect(result.success).toBe(true); + expect(events[0]).toEqual(expect.objectContaining({ kind: "step", data: expect.objectContaining({ status: "started" }) })); + const completedStepIndex = events.findIndex((e) => e.kind === "step" && (e.data as { status?: string }).status === "completed"); + expect(completedStepIndex).toBeGreaterThan(0); + // A step-start (and, once available, output) event must be observed before any terminal signal; + // RoutineRunner itself has no "complete" event type, so the invariant here is simply that + // incremental events fired at all — not only the returned final result. + expect(events.some((e) => e.kind === "step" && (e.data as { status?: string }).status === "started")).toBe(true); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); + }); + describe("concurrency policies", () => { it("parallel policy: runs even when another execution is in-flight", async () => { const routine = createMockRoutine({ diff --git a/packages/engine/src/cron-runner.ts b/packages/engine/src/cron-runner.ts index 91083cebd3..4fed99556a 100644 --- a/packages/engine/src/cron-runner.ts +++ b/packages/engine/src/cron-runner.ts @@ -1074,8 +1074,13 @@ export async function createAiPromptExecutor(cwd: string, store?: TaskStore): Pr /* FNXC:DatabaseBackup 2026-06-26-12:00: Cron-runner in-process backups feed automation run history and step errors. Normalize empty thrown values and empty command output before they become operator-visible Database Backup failures. + +FNXC:DatabaseBackup 2026-07-04-00:00: +FN-7537: exported (was module-private) so the dashboard's manual automation/schedule run path +(packages/dashboard/src/routes.ts executeSingleCommand) can format in-process backup failures with the +same message shape cron/routine-runner already use, instead of diverging on manual runs. */ -function formatInProcessBackupError(err: unknown, fusionDir: string): string { +export function formatInProcessBackupError(err: unknown, fusionDir: string): string { const message = err instanceof Error ? err.message.trim() : String(err ?? "").trim(); const cause = message || "unknown error"; if (cause.includes("project DB") || cause.includes("central DB")) { diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index b202d6b4aa..c3ca808a06 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -712,7 +712,7 @@ export { // ── Notification Service ────────────────────────────────────── export { NtfyNotificationProvider, NotificationService, WebhookNotificationProvider } from "./notification/index.js"; export type { NtfyProviderConfig, NotificationServiceOptions, WebhookProviderConfig } from "./notification/index.js"; -export { CronRunner, type CronRunnerOptions, type AiPromptExecutor, createAiPromptExecutor } from "./cron-runner.js"; +export { CronRunner, type CronRunnerOptions, type AiPromptExecutor, createAiPromptExecutor, isInProcessBackupCommand, isInProcessMemoryBackupCommand, formatInProcessBackupError } from "./cron-runner.js"; export { RoutineRunner, type RoutineRunnerOptions } from "./routine-runner.js"; export { RoutineScheduler, type RoutineSchedulerOptions } from "./routine-scheduler.js"; export { StuckTaskDetector, type StuckTaskDetectorOptions, type DisposableSession } from "./stuck-task-detector.js";