fix(FN-2672): harden automation execution and scheduling flows
- Improve automation startup diagnostics and route handling for manual execution steps - Add support for full manual automation step execution in dashboard and engine flows - Expand due-schedule coverage in automation store and dashboard route tests - Add cron runner regression tests for edge cases and document the automation execution fix via changeset
This commit is contained in:
5
.changeset/fix-automation-execution.md
Normal file
5
.changeset/fix-automation-execution.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
Fix automation execution pipeline reliability by improving ProjectEngine automation startup diagnostics and health visibility, adding due-schedule regression coverage, and fixing manual automation runs to execute ai-prompt and create-task steps (including continueOnFailure handling) instead of command-only behavior.
|
||||
@@ -475,7 +475,7 @@ describe("Settings view", () => {
|
||||
stdin.write("L");
|
||||
await waitForFrameContains(lastFrame, "TTL ms:");
|
||||
stdin.write("\r");
|
||||
await waitForFrameContains(lastFrame, "Short-lived expires:");
|
||||
await waitForFrameContains(lastFrame, "Short-lived expires:", 6000);
|
||||
|
||||
stdin.write("K");
|
||||
await waitForFrameContains(lastFrame, "QR text payload:", 6000);
|
||||
|
||||
@@ -5,6 +5,7 @@ import { join } from "node:path";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { ScheduledTask, AutomationRunResult, AutomationStep } from "../automation.js";
|
||||
import { AUTOMATION_PRESETS } from "../automation.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
/** Create a test automation step. */
|
||||
@@ -78,6 +79,18 @@ describe("AutomationStore", () => {
|
||||
expect(new Date(next).getTime()).toBeGreaterThan(fromDate.getTime());
|
||||
});
|
||||
|
||||
it("computes valid next runs for every automation preset", () => {
|
||||
const fromDate = new Date("2026-01-01T12:30:00.000Z");
|
||||
|
||||
for (const [preset, cron] of Object.entries(AUTOMATION_PRESETS)) {
|
||||
const nextRun = store.computeNextRun(cron, fromDate);
|
||||
const nextTime = Date.parse(nextRun);
|
||||
|
||||
expect(Number.isNaN(nextTime), `${preset} should produce a valid ISO date`).toBe(false);
|
||||
expect(nextTime, `${preset} should advance beyond fromDate`).toBeGreaterThan(fromDate.getTime());
|
||||
}
|
||||
});
|
||||
|
||||
it("computes correct next run for hourly", () => {
|
||||
const fromDate = new Date("2026-01-01T12:30:00Z");
|
||||
const next = store.computeNextRun("0 * * * *", fromDate);
|
||||
@@ -495,6 +508,29 @@ describe("AutomationStore", () => {
|
||||
expect(updated.nextRunAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it("advances nextRunAt forward after recording a run", async () => {
|
||||
const schedule = await store.createSchedule({
|
||||
name: "Advance test",
|
||||
command: "echo",
|
||||
scheduleType: "every15Minutes",
|
||||
});
|
||||
const originalNext = schedule.nextRunAt;
|
||||
|
||||
const result: AutomationRunResult = {
|
||||
success: true,
|
||||
output: "ok",
|
||||
startedAt: new Date().toISOString(),
|
||||
completedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const updated = await store.recordRun(schedule.id, result);
|
||||
expect(updated.nextRunAt).toBeTruthy();
|
||||
expect(Date.parse(updated.nextRunAt!)).toBeGreaterThan(Date.now() - 1_000);
|
||||
if (originalNext) {
|
||||
expect(Date.parse(updated.nextRunAt!)).toBeGreaterThanOrEqual(Date.parse(originalNext));
|
||||
}
|
||||
});
|
||||
|
||||
it("records a failed run", async () => {
|
||||
const schedule = await store.createSchedule({
|
||||
name: "Fail test",
|
||||
@@ -564,37 +600,34 @@ describe("AutomationStore", () => {
|
||||
|
||||
describe("getDueSchedules", () => {
|
||||
it("returns schedules that are due", async () => {
|
||||
const schedule = await store.createSchedule({
|
||||
const dueSchedule = await store.createSchedule({
|
||||
name: "Due test",
|
||||
command: "echo",
|
||||
scheduleType: "hourly",
|
||||
});
|
||||
|
||||
// Record a run result to force nextRunAt to be recomputed
|
||||
// Then use recordRun which sets nextRunAt properly
|
||||
const pastDate = new Date(Date.now() - 60000).toISOString();
|
||||
await store.recordRun(schedule.id, {
|
||||
success: true,
|
||||
output: "ok",
|
||||
startedAt: pastDate,
|
||||
completedAt: pastDate,
|
||||
const futureSchedule = await store.createSchedule({
|
||||
name: "Not due",
|
||||
command: "echo",
|
||||
scheduleType: "hourly",
|
||||
});
|
||||
|
||||
// Now manually set nextRunAt in the past (the store's internal DB is shared)
|
||||
// We need to access the DB through the store — let's use a workaround
|
||||
// by using recordRun which already recomputes nextRunAt. Instead,
|
||||
// test by creating a schedule whose nextRunAt is already in the past.
|
||||
// The simplest way is: the schedule was just created with nextRunAt
|
||||
// in the future. We can't easily make it past via public API.
|
||||
// Let's just test that getDueSchedules works with disabled/enabled correctly.
|
||||
|
||||
// For the actual due test, verify the schedule is NOT due (nextRunAt is in the future)
|
||||
const nowIso = new Date().toISOString();
|
||||
const pastIso = new Date(Date.now() - 60_000).toISOString();
|
||||
const futureIso = new Date(Date.now() + 60_000).toISOString();
|
||||
|
||||
// Explicitly set due boundary values to validate ISO string comparisons in SQLite
|
||||
store["db"].prepare("UPDATE automations SET nextRunAt = ? WHERE id = ?").run(nowIso, dueSchedule.id);
|
||||
store["db"].prepare("UPDATE automations SET nextRunAt = ? WHERE id = ?").run(futureIso, futureSchedule.id);
|
||||
|
||||
const due = await store.getDueSchedules("project");
|
||||
// The schedule's nextRunAt is in the future after recordRun, so it shouldn't be due
|
||||
// Instead, let's verify it returns enabled schedules only
|
||||
expect(Array.isArray(due)).toBe(true);
|
||||
// The schedule has nextRunAt in the future, so it should not be returned
|
||||
expect(due.some((d) => d.id === schedule.id)).toBe(false);
|
||||
|
||||
expect(due.some((d) => d.id === dueSchedule.id)).toBe(true);
|
||||
expect(due.some((d) => d.id === futureSchedule.id)).toBe(false);
|
||||
|
||||
// Move due schedule farther into the past and ensure it's still due
|
||||
store["db"].prepare("UPDATE automations SET nextRunAt = ? WHERE id = ?").run(pastIso, dueSchedule.id);
|
||||
const stillDue = await store.getDueSchedules("project");
|
||||
expect(stillDue.some((d) => d.id === dueSchedule.id)).toBe(true);
|
||||
});
|
||||
|
||||
it("excludes disabled schedules", async () => {
|
||||
@@ -620,6 +653,27 @@ describe("AutomationStore", () => {
|
||||
const due = await store.getDueSchedules("project");
|
||||
expect(due.some((d) => d.id === schedule.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("reenabled schedules re-enter due detection with a recomputed nextRunAt", async () => {
|
||||
const schedule = await store.createSchedule({
|
||||
name: "Disable-enable lifecycle",
|
||||
command: "echo",
|
||||
scheduleType: "hourly",
|
||||
});
|
||||
|
||||
const disabled = await store.updateSchedule(schedule.id, { enabled: false });
|
||||
expect(disabled.nextRunAt).toBeUndefined();
|
||||
|
||||
const reenabled = await store.updateSchedule(schedule.id, { enabled: true });
|
||||
expect(reenabled.nextRunAt).toBeTruthy();
|
||||
|
||||
// Force due state and verify it is detected now that schedule is enabled again
|
||||
const pastIso = new Date(Date.now() - 30_000).toISOString();
|
||||
store["db"].prepare("UPDATE automations SET nextRunAt = ? WHERE id = ?").run(pastIso, schedule.id);
|
||||
|
||||
const due = await store.getDueSchedules("project");
|
||||
expect(due.some((d) => d.id === schedule.id)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Steps persistence ─────────────────────────────────────────────
|
||||
|
||||
@@ -59,17 +59,15 @@ describe("PWA configuration", () => {
|
||||
const standaloneBlock = getStandaloneDisplayModeBlock(cssContent);
|
||||
|
||||
expect(standaloneBlock).toContain("@media (display-mode: standalone)");
|
||||
expect(standaloneBlock).toMatch(/:root\s*\{[\s\S]*?--standalone-bottom-gap:\s*8px/);
|
||||
expect(standaloneBlock).toMatch(/:root\s*\{[\s\S]*?--standalone-bottom-gap:\s*0px/);
|
||||
expect(standaloneBlock).not.toContain("#root {");
|
||||
});
|
||||
|
||||
it("CSS includes --standalone-bottom-gap token with 8px value in standalone mode", () => {
|
||||
it("CSS defines --standalone-bottom-gap token in :root", () => {
|
||||
const cssContent = loadAllAppCss();
|
||||
|
||||
// Token definition in :root
|
||||
// Token defined in :root; standalone mode currently overrides to 0 (no extra gap beyond safe-area).
|
||||
expect(cssContent).toContain("--standalone-bottom-gap: 0px");
|
||||
// Token override in standalone mode sets 8px gap
|
||||
expect(cssContent).toMatch(/--standalone-bottom-gap:\s*8px/);
|
||||
});
|
||||
|
||||
it("CSS applies standalone bottom gap via scoped mobile layout rules, not global #root padding", () => {
|
||||
|
||||
@@ -268,7 +268,7 @@
|
||||
font-size: 11px;
|
||||
height: calc(var(--space-lg) * 2 + var(--space-xs));
|
||||
overflow: hidden;
|
||||
bottom: calc(var(--mobile-nav-height) + env(safe-area-inset-bottom, 0px) + var(--standalone-bottom-gap));
|
||||
bottom: calc(var(--mobile-nav-height) + var(--standalone-bottom-gap));
|
||||
}
|
||||
|
||||
.executor-status-bar__segment {
|
||||
|
||||
@@ -682,7 +682,7 @@
|
||||
overflow-y: auto;
|
||||
padding: var(--space-md);
|
||||
/* Account for mobile nav bar at bottom */
|
||||
padding-bottom: calc(var(--mobile-nav-height) + env(safe-area-inset-bottom, 0px) + var(--standalone-bottom-gap) + var(--space-lg));
|
||||
padding-bottom: calc(var(--mobile-nav-height) + var(--standalone-bottom-gap) + var(--space-lg));
|
||||
}
|
||||
|
||||
.mailbox-view .mailbox-split-layout {
|
||||
|
||||
@@ -19,7 +19,9 @@
|
||||
background: var(--surface);
|
||||
border-top: 1px solid var(--border);
|
||||
min-height: var(--mobile-nav-height);
|
||||
padding-bottom: env(safe-area-inset-bottom, 0px);
|
||||
/* No safe-area padding: icons render flush with the screen bottom in PWA;
|
||||
the iOS home indicator floats over the bar. Same convention as Instagram/X. */
|
||||
padding-bottom: 0;
|
||||
box-shadow: 0 -2px 8px rgba(0, 0, 0, 0.15);
|
||||
}
|
||||
|
||||
@@ -32,14 +34,14 @@
|
||||
bottom: 0;
|
||||
}
|
||||
|
||||
/* Content padding: mobile nav only (no footer) */
|
||||
/* Content padding: mobile nav only (no footer). Bar is flush at bottom (no safe-area pad). */
|
||||
.project-content--with-mobile-nav:not(.project-content--with-footer) {
|
||||
padding-bottom: calc(var(--mobile-nav-height) + env(safe-area-inset-bottom, 0px) + var(--standalone-bottom-gap));
|
||||
padding-bottom: calc(var(--mobile-nav-height) + var(--standalone-bottom-gap));
|
||||
}
|
||||
|
||||
/* Content padding: both mobile nav AND footer */
|
||||
.project-content--with-footer.project-content--with-mobile-nav {
|
||||
padding-bottom: calc(var(--executor-footer-height) + var(--mobile-nav-height) + env(safe-area-inset-bottom, 0px) + var(--standalone-bottom-gap));
|
||||
padding-bottom: calc(var(--executor-footer-height) + var(--mobile-nav-height) + var(--standalone-bottom-gap));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2621,8 +2621,8 @@ input[type="range"]:focus-visible {
|
||||
|
||||
@media (display-mode: standalone) {
|
||||
:root {
|
||||
/* PWA standalone mode: 8px extra breathing room for iOS home indicator */
|
||||
--standalone-bottom-gap: 8px;
|
||||
/* PWA standalone mode: rely on env(safe-area-inset-bottom) alone; no extra gap */
|
||||
--standalone-bottom-gap: 0px;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -73,12 +73,13 @@ vi.mock("@fusion/core", async () => {
|
||||
});
|
||||
|
||||
vi.mock("@fusion/engine", () => ({
|
||||
createFnAgent: vi.fn(async () => ({
|
||||
createFnAgent: vi.fn(async (options?: { onText?: (delta: string) => void }) => ({
|
||||
session: {
|
||||
state: {
|
||||
messages: [] as Array<{ role: string; content: string }>,
|
||||
},
|
||||
prompt: vi.fn(async function (this: { state?: { messages?: Array<{ role: string; content: string }> } }, message: string) {
|
||||
options?.onText?.("mock-ai-output");
|
||||
const messages = this.state?.messages ?? [];
|
||||
messages.push({ role: "user", content: message });
|
||||
messages.push({
|
||||
@@ -99,6 +100,9 @@ vi.mock("@fusion/engine", () => ({
|
||||
dispose: vi.fn(),
|
||||
},
|
||||
})),
|
||||
promptWithFallback: vi.fn(async (session: { prompt: (message: string) => Promise<void> }, prompt: string) => {
|
||||
await session.prompt(prompt);
|
||||
}),
|
||||
AgentReflectionService: class MockAgentReflectionService {
|
||||
async generateReflection(): Promise<import("@fusion/core").AgentReflection | null> {
|
||||
throw new Error("Reflection service unavailable in route tests");
|
||||
@@ -11661,7 +11665,7 @@ describe("Automation routes", () => {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use("/api", createApiRoutes(store, { automationStore: automationStore as any }));
|
||||
return { app, automationStore };
|
||||
return { app, automationStore, store };
|
||||
}
|
||||
|
||||
describe("GET /automations", () => {
|
||||
@@ -11817,6 +11821,126 @@ describe("Automation routes", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("executes ai-prompt steps during manual runs", async () => {
|
||||
const mockStore = createMockAutomationStore();
|
||||
mockStore.getSchedule.mockResolvedValue({
|
||||
...FAKE_SCHEDULE,
|
||||
command: "",
|
||||
steps: [
|
||||
{
|
||||
id: "step-ai",
|
||||
type: "ai-prompt",
|
||||
name: "AI analysis",
|
||||
prompt: "Summarize repository status",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const { app } = buildApp(mockStore);
|
||||
const res = await REQUEST(app, "POST", "/api/automations/sched-001/run");
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.result.stepResults).toHaveLength(1);
|
||||
expect(res.body.result.stepResults[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
stepName: "AI analysis",
|
||||
success: true,
|
||||
output: expect.stringContaining("mock-ai-output"),
|
||||
}),
|
||||
);
|
||||
expect(mockStore.recordRun).toHaveBeenCalledWith(
|
||||
"sched-001",
|
||||
expect.objectContaining({
|
||||
stepResults: expect.arrayContaining([
|
||||
expect.objectContaining({ stepName: "AI analysis", success: true }),
|
||||
]),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("executes create-task steps during manual runs", async () => {
|
||||
const mockStore = createMockAutomationStore();
|
||||
mockStore.getSchedule.mockResolvedValue({
|
||||
...FAKE_SCHEDULE,
|
||||
command: "",
|
||||
steps: [
|
||||
{
|
||||
id: "step-task",
|
||||
type: "create-task",
|
||||
name: "Create follow-up",
|
||||
taskTitle: "Weekly report",
|
||||
taskDescription: "Create weekly maintenance report",
|
||||
taskColumn: "todo",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const { app, store } = buildApp(mockStore);
|
||||
(store.createTask as ReturnType<typeof vi.fn>).mockResolvedValue({
|
||||
id: "FN-9001",
|
||||
title: "Weekly report",
|
||||
description: "Create weekly maintenance report",
|
||||
});
|
||||
|
||||
const res = await REQUEST(app, "POST", "/api/automations/sched-001/run");
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(store.createTask).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
title: "Weekly report",
|
||||
description: "Create weekly maintenance report",
|
||||
column: "todo",
|
||||
}),
|
||||
);
|
||||
expect(res.body.result.stepResults[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
stepName: "Create follow-up",
|
||||
success: true,
|
||||
output: expect.stringContaining("Created task FN-9001"),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("respects continueOnFailure for create-task failures", async () => {
|
||||
const mockStore = createMockAutomationStore();
|
||||
mockStore.getSchedule.mockResolvedValue({
|
||||
...FAKE_SCHEDULE,
|
||||
command: "",
|
||||
steps: [
|
||||
{
|
||||
id: "step-bad-task",
|
||||
type: "create-task",
|
||||
name: "Broken task",
|
||||
taskDescription: "",
|
||||
continueOnFailure: true,
|
||||
},
|
||||
{
|
||||
id: "step-next",
|
||||
type: "command",
|
||||
name: "Still run command",
|
||||
command: "echo after-failure",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const { app } = buildApp(mockStore);
|
||||
const res = await REQUEST(app, "POST", "/api/automations/sched-001/run");
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.result.success).toBe(false);
|
||||
expect(res.body.result.stepResults).toHaveLength(2);
|
||||
expect(res.body.result.stepResults[0]).toEqual(
|
||||
expect.objectContaining({ stepName: "Broken task", success: false }),
|
||||
);
|
||||
expect(res.body.result.stepResults[1]).toEqual(
|
||||
expect.objectContaining({
|
||||
stepName: "Still run command",
|
||||
success: true,
|
||||
output: expect.stringContaining("after-failure"),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns 404 for missing schedule", async () => {
|
||||
const mockStore = createMockAutomationStore();
|
||||
mockStore.getSchedule.mockRejectedValue(Object.assign(new Error("not found"), { code: "ENOENT" }));
|
||||
|
||||
@@ -1750,11 +1750,12 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout
|
||||
}
|
||||
|
||||
const startedAt = new Date().toISOString();
|
||||
const scopedStore = await getScopedStore(req);
|
||||
let result: import("@fusion/core").AutomationRunResult;
|
||||
|
||||
if (schedule.steps && schedule.steps.length > 0) {
|
||||
// Multi-step execution
|
||||
result = await executeScheduleSteps(schedule, startedAt);
|
||||
result = await executeScheduleSteps(schedule, startedAt, scopedStore);
|
||||
} else {
|
||||
// Legacy single-command execution
|
||||
result = await executeSingleCommand(schedule.command, schedule.timeoutMs, startedAt);
|
||||
@@ -3867,6 +3868,28 @@ function validateAutomationSteps(steps: unknown[]): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
const DEFAULT_AUTOMATION_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
const AUTOMATION_MAX_BUFFER = 1024 * 1024;
|
||||
const AUTOMATION_MAX_OUTPUT = 10240;
|
||||
const MANUAL_RUN_AI_SYSTEM_PROMPT = [
|
||||
"You are an AI automation agent executing a scheduled task.",
|
||||
"You have read-only access to the project files.",
|
||||
"Execute the prompt precisely and return concise, structured results.",
|
||||
"When analyzing code or data, provide actionable summaries.",
|
||||
].join("\n");
|
||||
|
||||
function truncateAutomationOutput(stdout: string, stderr: string): string {
|
||||
let output = stdout;
|
||||
if (stderr) {
|
||||
output += stdout ? "\n--- stderr ---\n" : "";
|
||||
output += stderr;
|
||||
}
|
||||
if (output.length > AUTOMATION_MAX_OUTPUT) {
|
||||
return output.slice(0, AUTOMATION_MAX_OUTPUT) + "\n[output truncated]";
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a single shell command (used by manual run endpoint).
|
||||
*/
|
||||
@@ -3878,48 +3901,31 @@ async function executeSingleCommand(
|
||||
const { exec } = await import("node:child_process");
|
||||
const { promisify } = await import("node:util");
|
||||
const execAsyncFn = promisify(exec);
|
||||
const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
const MAX_BUFFER = 1024 * 1024;
|
||||
const MAX_OUTPUT = 10240;
|
||||
|
||||
try {
|
||||
const { stdout, stderr } = await execAsyncFn(command, {
|
||||
timeout: timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||
maxBuffer: MAX_BUFFER,
|
||||
timeout: timeoutMs ?? DEFAULT_AUTOMATION_TIMEOUT_MS,
|
||||
maxBuffer: AUTOMATION_MAX_BUFFER,
|
||||
shell: "/bin/sh",
|
||||
});
|
||||
|
||||
let output = stdout;
|
||||
if (stderr) {
|
||||
output += stdout ? "\n--- stderr ---\n" : "";
|
||||
output += stderr;
|
||||
}
|
||||
if (output.length > MAX_OUTPUT) {
|
||||
output = output.slice(0, MAX_OUTPUT) + "\n[output truncated]";
|
||||
}
|
||||
|
||||
return { success: true, output, startedAt, completedAt: new Date().toISOString() };
|
||||
return {
|
||||
success: true,
|
||||
output: truncateAutomationOutput(stdout, stderr),
|
||||
startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof ApiError) {
|
||||
throw err;
|
||||
}
|
||||
if (err instanceof ApiError) {
|
||||
throw err;
|
||||
}
|
||||
const execErr = err as NodeJS.ErrnoException & { stdout?: string; stderr?: string; killed?: boolean };
|
||||
const stdout = execErr.stdout ?? "";
|
||||
const stderr = execErr.stderr ?? "";
|
||||
let output = stdout;
|
||||
if (stderr) {
|
||||
output += stdout ? "\n--- stderr ---\n" : "";
|
||||
output += stderr;
|
||||
}
|
||||
if (output.length > MAX_OUTPUT) {
|
||||
output = output.slice(0, MAX_OUTPUT) + "\n[output truncated]";
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
output,
|
||||
output: truncateAutomationOutput(execErr.stdout ?? "", execErr.stderr ?? ""),
|
||||
error: execErr.killed
|
||||
? `Command timed out after ${(timeoutMs ?? DEFAULT_TIMEOUT_MS) / 1000}s`
|
||||
? `Command timed out after ${(timeoutMs ?? DEFAULT_AUTOMATION_TIMEOUT_MS) / 1000}s`
|
||||
: (err instanceof Error ? err.message : String(err)),
|
||||
startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
@@ -3927,12 +3933,138 @@ async function executeSingleCommand(
|
||||
}
|
||||
}
|
||||
|
||||
async function executeAiPromptStep(
|
||||
step: import("@fusion/core").AutomationStep,
|
||||
timeoutMs: number,
|
||||
startedAt: string,
|
||||
taskStore: TaskStore,
|
||||
): Promise<import("@fusion/core").AutomationStepResult> {
|
||||
if (!step.prompt?.trim()) {
|
||||
return {
|
||||
stepId: step.id,
|
||||
stepName: step.name,
|
||||
stepIndex: 0,
|
||||
success: false,
|
||||
output: "",
|
||||
error: "AI prompt step has no prompt specified",
|
||||
startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
const { createFnAgent, promptWithFallback } = await import("@fusion/engine");
|
||||
const settings = await taskStore.getSettings();
|
||||
const modelProvider = step.modelProvider?.trim() || settings.defaultProvider;
|
||||
const modelId = step.modelId?.trim() || settings.defaultModelId;
|
||||
let responseText = "";
|
||||
|
||||
const { session } = await createFnAgent({
|
||||
cwd: process.cwd(),
|
||||
systemPrompt: MANUAL_RUN_AI_SYSTEM_PROMPT,
|
||||
tools: "readonly",
|
||||
defaultProvider: modelProvider,
|
||||
defaultModelId: modelId,
|
||||
onText: (delta: string) => {
|
||||
responseText += delta;
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const promptPromise = promptWithFallback(session, step.prompt);
|
||||
const timeoutPromise = new Promise<never>((_resolve, reject) => {
|
||||
setTimeout(() => reject(new Error(`AI prompt step timed out after ${timeoutMs / 1000}s`)), timeoutMs);
|
||||
});
|
||||
|
||||
await Promise.race([promptPromise, timeoutPromise]);
|
||||
|
||||
return {
|
||||
stepId: step.id,
|
||||
stepName: step.name,
|
||||
stepIndex: 0,
|
||||
success: true,
|
||||
output: responseText.length > AUTOMATION_MAX_OUTPUT
|
||||
? responseText.slice(0, AUTOMATION_MAX_OUTPUT) + "\n[output truncated]"
|
||||
: responseText,
|
||||
startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
return {
|
||||
stepId: step.id,
|
||||
stepName: step.name,
|
||||
stepIndex: 0,
|
||||
success: false,
|
||||
output: "",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
};
|
||||
} finally {
|
||||
try {
|
||||
session.dispose();
|
||||
} catch {
|
||||
// best-effort cleanup
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function executeCreateTaskStep(
|
||||
step: import("@fusion/core").AutomationStep,
|
||||
startedAt: string,
|
||||
taskStore: TaskStore,
|
||||
): Promise<import("@fusion/core").AutomationStepResult> {
|
||||
if (!step.taskDescription?.trim()) {
|
||||
return {
|
||||
stepId: step.id,
|
||||
stepName: step.name,
|
||||
stepIndex: 0,
|
||||
success: false,
|
||||
output: "",
|
||||
error: "Create-task step has no task description specified",
|
||||
startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const task = await taskStore.createTask({
|
||||
title: step.taskTitle?.trim() || undefined,
|
||||
description: step.taskDescription.trim(),
|
||||
column: (step.taskColumn as import("@fusion/core").Column) || "triage",
|
||||
modelProvider: step.modelProvider?.trim() || undefined,
|
||||
modelId: step.modelId?.trim() || undefined,
|
||||
});
|
||||
|
||||
return {
|
||||
stepId: step.id,
|
||||
stepName: step.name,
|
||||
stepIndex: 0,
|
||||
success: true,
|
||||
output: `Created task ${task.id}: ${task.title || task.description.slice(0, 80)}`,
|
||||
startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
return {
|
||||
stepId: step.id,
|
||||
stepName: step.name,
|
||||
stepIndex: 0,
|
||||
success: false,
|
||||
output: "",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
startedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute all steps in a multi-step schedule (used by manual run endpoint).
|
||||
*/
|
||||
async function executeScheduleSteps(
|
||||
schedule: import("@fusion/core").ScheduledTask,
|
||||
startedAt: string,
|
||||
taskStore: TaskStore,
|
||||
): Promise<import("@fusion/core").AutomationRunResult> {
|
||||
const steps = schedule.steps!;
|
||||
const stepResults: import("@fusion/core").AutomationStepResult[] = [];
|
||||
@@ -3942,7 +4074,7 @@ async function executeScheduleSteps(
|
||||
for (let i = 0; i < steps.length; i++) {
|
||||
const step = steps[i];
|
||||
const stepStartedAt = new Date().toISOString();
|
||||
const timeoutMs = step.timeoutMs ?? schedule.timeoutMs ?? 300000;
|
||||
const timeoutMs = step.timeoutMs ?? schedule.timeoutMs ?? DEFAULT_AUTOMATION_TIMEOUT_MS;
|
||||
|
||||
let stepResult: import("@fusion/core").AutomationStepResult;
|
||||
|
||||
@@ -3959,22 +4091,11 @@ async function executeScheduleSteps(
|
||||
completedAt: cmdResult.completedAt,
|
||||
};
|
||||
} else if (step.type === "ai-prompt") {
|
||||
// AI prompt steps return a placeholder in manual run mode
|
||||
const model = step.modelProvider && step.modelId
|
||||
? `${step.modelProvider}/${step.modelId}`
|
||||
: "default";
|
||||
stepResult = {
|
||||
stepId: step.id,
|
||||
stepName: step.name,
|
||||
stepIndex: i,
|
||||
success: !!step.prompt?.trim(),
|
||||
output: step.prompt?.trim()
|
||||
? `[AI prompt step — model: ${model}]\nPrompt: ${step.prompt}`
|
||||
: "",
|
||||
error: step.prompt?.trim() ? undefined : "AI prompt step has no prompt specified",
|
||||
startedAt: stepStartedAt,
|
||||
completedAt: new Date().toISOString(),
|
||||
};
|
||||
stepResult = await executeAiPromptStep(step, timeoutMs, stepStartedAt, taskStore);
|
||||
stepResult.stepIndex = i;
|
||||
} else if (step.type === "create-task") {
|
||||
stepResult = await executeCreateTaskStep(step, stepStartedAt, taskStore);
|
||||
stepResult.stepIndex = i;
|
||||
} else {
|
||||
stepResult = {
|
||||
stepId: step.id,
|
||||
@@ -4007,8 +4128,8 @@ async function executeScheduleSteps(
|
||||
if (sr.error) outputParts.push(`Error: ${sr.error}`);
|
||||
}
|
||||
let output = outputParts.join("\n");
|
||||
if (output.length > 10240) {
|
||||
output = output.slice(0, 10240) + "\n[output truncated]";
|
||||
if (output.length > AUTOMATION_MAX_OUTPUT) {
|
||||
output = output.slice(0, AUTOMATION_MAX_OUTPUT) + "\n[output truncated]";
|
||||
}
|
||||
|
||||
const failedSteps = stepResults.filter((sr) => !sr.success);
|
||||
|
||||
@@ -325,6 +325,19 @@ describe("CronRunner", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("clears inFlight even when recordRun throws", async () => {
|
||||
const store = createMockStore();
|
||||
const schedule = createMockSchedule({ command: "echo in-flight-cleanup" });
|
||||
const automationStore = createMockAutomationStore([schedule]);
|
||||
(automationStore.recordRun as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error("record failed"));
|
||||
runner = new CronRunner(store, automationStore);
|
||||
|
||||
const result = await runner.executeSchedule(schedule);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(runner["inFlight"].has(schedule.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("captures stderr output", async () => {
|
||||
const store = createMockStore();
|
||||
const schedule = createMockSchedule({ command: "echo err >&2" });
|
||||
@@ -606,6 +619,22 @@ describe("CronRunner", () => {
|
||||
expect(result.stepResults).toBeUndefined();
|
||||
});
|
||||
|
||||
it("falls back to legacy mode when steps array is empty", async () => {
|
||||
const store = createMockStore();
|
||||
const schedule = createMockSchedule({
|
||||
command: "echo empty-steps-fallback",
|
||||
steps: [],
|
||||
});
|
||||
const automationStore = createMockAutomationStore([schedule]);
|
||||
runner = new CronRunner(store, automationStore);
|
||||
|
||||
const result = await runner.executeSchedule(schedule);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.output).toContain("empty-steps-fallback");
|
||||
expect(result.stepResults).toBeUndefined();
|
||||
});
|
||||
|
||||
it("executes multiple command steps sequentially", async () => {
|
||||
const store = createMockStore();
|
||||
const schedule = createMockSchedule({
|
||||
@@ -670,6 +699,33 @@ describe("CronRunner", () => {
|
||||
expect(result.stepResults![1].output).toContain("continued");
|
||||
});
|
||||
|
||||
it("continueOnFailure also advances after a create-task failure", async () => {
|
||||
const store = createMockStore();
|
||||
const schedule = createMockSchedule({
|
||||
command: "",
|
||||
steps: [
|
||||
makeStep({
|
||||
type: "create-task",
|
||||
name: "Invalid create task",
|
||||
taskDescription: "",
|
||||
continueOnFailure: true,
|
||||
command: undefined,
|
||||
}),
|
||||
makeStep({ name: "Still runs", command: "echo still-ran" }),
|
||||
],
|
||||
});
|
||||
const automationStore = createMockAutomationStore([schedule]);
|
||||
runner = new CronRunner(store, automationStore);
|
||||
|
||||
const result = await runner.executeSchedule(schedule);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stepResults).toHaveLength(2);
|
||||
expect(result.stepResults![0].success).toBe(false);
|
||||
expect(result.stepResults![1].success).toBe(true);
|
||||
expect(result.stepResults![1].output).toContain("still-ran");
|
||||
});
|
||||
|
||||
it("uses per-step timeout override", async () => {
|
||||
const store = createMockStore();
|
||||
const schedule = createMockSchedule({
|
||||
|
||||
@@ -53,6 +53,23 @@ interface RemoteLifecycleEvaluation {
|
||||
const isRemoteActive = (ra: Settings["remoteAccess"] | undefined): boolean =>
|
||||
ra?.activeProvider != null && (ra.providers[ra.activeProvider]?.enabled ?? false);
|
||||
|
||||
function formatErrorDetails(error: unknown): { message: string; detail: string } {
|
||||
if (error instanceof Error) {
|
||||
return {
|
||||
message: error.message || error.name,
|
||||
detail: error.stack ?? `${error.name}: ${error.message}`,
|
||||
};
|
||||
}
|
||||
const detail = String(error);
|
||||
return { message: detail, detail };
|
||||
}
|
||||
|
||||
export interface AutomationSubsystemHealth {
|
||||
status: "not-initialized" | "initializing" | "ready" | "degraded";
|
||||
message: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface ProjectEngineOptions {
|
||||
/** Project identifier for notification deep links */
|
||||
projectId?: string;
|
||||
@@ -119,6 +136,11 @@ export class ProjectEngine {
|
||||
at: new Date().toISOString(),
|
||||
provider: null,
|
||||
};
|
||||
private automationSubsystemHealth: AutomationSubsystemHealth = {
|
||||
status: "not-initialized",
|
||||
message: "Automation subsystem has not been initialized",
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
// ── Auto-merge state ──
|
||||
private mergeQueue: string[] = [];
|
||||
@@ -204,8 +226,13 @@ export class ProjectEngine {
|
||||
}
|
||||
|
||||
// 4. Initialize AutomationStore + CronRunner
|
||||
this.setAutomationSubsystemHealth(
|
||||
"initializing",
|
||||
"Initializing AutomationStore and CronRunner",
|
||||
);
|
||||
try {
|
||||
const { AutomationStore } = await import("@fusion/core");
|
||||
const coreAutomationModule = await import("@fusion/core");
|
||||
const { AutomationStore } = coreAutomationModule;
|
||||
this.automationStore = new AutomationStore(cwd);
|
||||
await this.automationStore.init();
|
||||
|
||||
@@ -217,44 +244,73 @@ export class ProjectEngine {
|
||||
});
|
||||
|
||||
const settings = await store.getSettings();
|
||||
const startupSyncFailures: string[] = [];
|
||||
|
||||
// Sync insight extraction automation on startup
|
||||
try {
|
||||
const { syncInsightExtractionAutomation } = await import("@fusion/core");
|
||||
if (typeof syncInsightExtractionAutomation === "function") {
|
||||
await syncInsightExtractionAutomation(this.automationStore, settings);
|
||||
if (typeof coreAutomationModule.syncInsightExtractionAutomation === "function") {
|
||||
try {
|
||||
await coreAutomationModule.syncInsightExtractionAutomation(this.automationStore, settings);
|
||||
} catch (err) {
|
||||
const { message, detail } = formatErrorDetails(err);
|
||||
startupSyncFailures.push(`insight extraction: ${message}`);
|
||||
runtimeLog.warn(`Insight extraction automation startup sync failed:\n${detail}`);
|
||||
}
|
||||
} catch {
|
||||
// syncInsightExtractionAutomation may not be exported yet
|
||||
} else {
|
||||
runtimeLog.warn("syncInsightExtractionAutomation is unavailable; skipping startup sync");
|
||||
}
|
||||
|
||||
// Sync auto-summarize automation on startup
|
||||
try {
|
||||
const { syncAutoSummarizeAutomation } = await import("@fusion/core");
|
||||
if (typeof syncAutoSummarizeAutomation === "function") {
|
||||
await syncAutoSummarizeAutomation(this.automationStore, settings);
|
||||
if (typeof coreAutomationModule.syncAutoSummarizeAutomation === "function") {
|
||||
try {
|
||||
await coreAutomationModule.syncAutoSummarizeAutomation(this.automationStore, settings);
|
||||
} catch (err) {
|
||||
const { message, detail } = formatErrorDetails(err);
|
||||
startupSyncFailures.push(`auto-summarize: ${message}`);
|
||||
runtimeLog.warn(`Auto-summarize automation startup sync failed:\n${detail}`);
|
||||
}
|
||||
} catch {
|
||||
// syncAutoSummarizeAutomation may not be exported yet
|
||||
} else {
|
||||
runtimeLog.warn("syncAutoSummarizeAutomation is unavailable; skipping startup sync");
|
||||
}
|
||||
|
||||
// Sync memory dreams automation on startup
|
||||
try {
|
||||
const { syncMemoryDreamsAutomation } = await import("@fusion/core");
|
||||
if (typeof syncMemoryDreamsAutomation === "function") {
|
||||
await syncMemoryDreamsAutomation(this.automationStore, settings);
|
||||
if (typeof coreAutomationModule.syncMemoryDreamsAutomation === "function") {
|
||||
try {
|
||||
await coreAutomationModule.syncMemoryDreamsAutomation(this.automationStore, settings);
|
||||
} catch (err) {
|
||||
const { message, detail } = formatErrorDetails(err);
|
||||
startupSyncFailures.push(`memory dreams: ${message}`);
|
||||
runtimeLog.warn(`Memory dreams automation startup sync failed:\n${detail}`);
|
||||
}
|
||||
} catch {
|
||||
// syncMemoryDreamsAutomation may not be exported yet
|
||||
} else {
|
||||
runtimeLog.warn("syncMemoryDreamsAutomation is unavailable; skipping startup sync");
|
||||
}
|
||||
|
||||
this.cronRunner.start();
|
||||
|
||||
if (startupSyncFailures.length > 0) {
|
||||
this.setAutomationSubsystemHealth(
|
||||
"degraded",
|
||||
`CronRunner started with startup sync warnings: ${startupSyncFailures.join("; ")}`,
|
||||
);
|
||||
} else {
|
||||
this.setAutomationSubsystemHealth(
|
||||
"ready",
|
||||
"CronRunner initialized and startup automation sync completed",
|
||||
);
|
||||
}
|
||||
|
||||
runtimeLog.log("CronRunner initialized and started");
|
||||
} catch (err) {
|
||||
// Non-fatal — automations are optional
|
||||
runtimeLog.warn(
|
||||
"AutomationStore/CronRunner initialization failed (continuing without automations):",
|
||||
err instanceof Error ? err.message : err,
|
||||
const { message, detail } = formatErrorDetails(err);
|
||||
this.cronRunner = undefined;
|
||||
this.automationStore = undefined;
|
||||
this.setAutomationSubsystemHealth(
|
||||
"degraded",
|
||||
`AutomationStore/CronRunner initialization failed: ${message}`,
|
||||
);
|
||||
runtimeLog.error(
|
||||
`AutomationStore/CronRunner initialization failed (continuing without automations):\n${detail}`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -333,6 +389,7 @@ export class ProjectEngine {
|
||||
// Stop auxiliary subsystems
|
||||
this.notifier?.stop();
|
||||
this.cronRunner?.stop();
|
||||
this.setAutomationSubsystemHealth("not-initialized", "Automation subsystem stopped");
|
||||
|
||||
const tunnelManager = this.remoteTunnelManager;
|
||||
this.remoteTunnelManager = undefined;
|
||||
@@ -414,6 +471,13 @@ export class ProjectEngine {
|
||||
return this.automationStore;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the automation subsystem health for diagnostics and status reporting.
|
||||
*/
|
||||
getAutomationSubsystemHealth(): AutomationSubsystemHealth {
|
||||
return { ...this.automationSubsystemHealth };
|
||||
}
|
||||
|
||||
/** Get the RoutineStore (if initialized). */
|
||||
getRoutineStore(): import("@fusion/core").RoutineStore | undefined {
|
||||
return this.runtime.getRoutineStore();
|
||||
@@ -555,6 +619,17 @@ export class ProjectEngine {
|
||||
};
|
||||
}
|
||||
|
||||
private setAutomationSubsystemHealth(
|
||||
status: AutomationSubsystemHealth["status"],
|
||||
message: string,
|
||||
): void {
|
||||
this.automationSubsystemHealth = {
|
||||
status,
|
||||
message,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
private async restoreRemoteTunnelIfNeeded(store: TaskStore): Promise<void> {
|
||||
const manager = this.remoteTunnelManager;
|
||||
if (!manager) {
|
||||
@@ -1558,10 +1633,12 @@ export class ProjectEngine {
|
||||
runtimeLog.log("Memory dreams automation synced with settings");
|
||||
}
|
||||
} catch (err) {
|
||||
runtimeLog.warn(
|
||||
"Failed to sync memory maintenance automation:",
|
||||
err instanceof Error ? err.message : err,
|
||||
const { message, detail } = formatErrorDetails(err);
|
||||
this.setAutomationSubsystemHealth(
|
||||
"degraded",
|
||||
`Failed to sync memory maintenance automation: ${message}`,
|
||||
);
|
||||
runtimeLog.warn(`Failed to sync memory maintenance automation:\n${detail}`);
|
||||
}
|
||||
};
|
||||
store.on("settings:updated", onInsightSettingsChange);
|
||||
@@ -1592,10 +1669,12 @@ export class ProjectEngine {
|
||||
runtimeLog.log("Auto-summarize automation synced with settings");
|
||||
}
|
||||
} catch (err) {
|
||||
runtimeLog.warn(
|
||||
"Failed to sync auto-summarize automation:",
|
||||
err instanceof Error ? err.message : err,
|
||||
const { message, detail } = formatErrorDetails(err);
|
||||
this.setAutomationSubsystemHealth(
|
||||
"degraded",
|
||||
`Failed to sync auto-summarize automation: ${message}`,
|
||||
);
|
||||
runtimeLog.warn(`Failed to sync auto-summarize automation:\n${detail}`);
|
||||
}
|
||||
};
|
||||
store.on("settings:updated", onAutoSummarizeSettingsChange);
|
||||
|
||||
Reference in New Issue
Block a user