feat(FN-1269): complete routine engine integration with RoutineRunner and RoutineScheduler

- Add RoutineRunner class for routine execution via heartbeat system
- Add RoutineScheduler class for cron-based routine polling
- Add triggerManual and triggerWebhook methods for API and webhook triggers
- Wire RoutineScheduler into InProcessRuntime lifecycle
- Add routine trigger and webhook API endpoints
- Fix type mismatches between PROMPT and actual FN-1519 types
This commit is contained in:
gsxdsm
2026-04-10 12:11:55 -07:00
parent 6600b0b97e
commit 5cb368d7cf
14 changed files with 1340 additions and 781 deletions

View File

@@ -1,253 +1,458 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import type { Routine, RoutineStore, ProjectSettings } from "@fusion/core";
import { RoutineScheduler } from "./routine-scheduler.js";
import { RoutineScheduler, type RoutineSchedulerOptions } from "./routine-scheduler.js";
import type { RoutineStore, Routine, TaskStore, RoutineExecutionResult, Settings } from "@fusion/core";
import type { RoutineRunner } from "./routine-runner.js";
// Mock the logger
vi.mock("./logger.js", () => ({
createLogger: () => ({
log: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
}),
}));
// Default settings inline to avoid @fusion/core build dependency during tests
const DEFAULT_SETTINGS: Settings = {
maxConcurrent: 2,
maxWorktrees: 4,
pollIntervalMs: 30000,
autoResolveConflicts: true,
requirePlanApproval: false,
recycleWorktrees: false,
worktreeNaming: "random",
globalPause: false,
enginePaused: false,
ntfyEnabled: false,
defaultProvider: "anthropic",
defaultModelId: "claude-sonnet-4-5",
planningProvider: "anthropic",
planningModelId: "claude-sonnet-4-5",
validatorProvider: "openai",
validatorModelId: "gpt-4o",
taskStuckTimeoutMs: undefined,
groupOverlappingFiles: false,
autoMerge: true,
};
function createMockRoutine(overrides: Partial<Routine> = {}): Routine {
return {
id: "test-routine-id",
agentId: "test-agent",
name: "Test Routine",
description: "A test routine",
trigger: { type: "cron", cronExpression: "0 * * * *" },
catchUpPolicy: "run_one",
executionPolicy: "parallel",
enabled: true,
runCount: 0,
runHistory: [],
cronExpression: "0 * * * *",
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
...overrides,
};
}
function createMockTaskStore(settingsOverrides: Partial<Settings> = {}): TaskStore {
return {
getSettings: vi.fn().mockResolvedValue({
...DEFAULT_SETTINGS,
...settingsOverrides,
}),
on: vi.fn(),
off: vi.fn(),
} as unknown as TaskStore;
}
function createMockRoutineStore(routines: Routine[] = []): RoutineStore {
const routineMap = new Map(routines.map((r) => [r.id, r]));
return {
getRoutine: vi.fn().mockImplementation((id: string) => {
const routine = routineMap.get(id);
if (!routine) {
throw Object.assign(new Error(`Routine '${id}' not found`), { code: "ENOENT" });
}
return routine;
}),
listRoutines: vi.fn().mockResolvedValue(routines),
updateRoutine: vi.fn().mockImplementation((id: string, _updates: any) => {
const routine = routineMap.get(id);
if (!routine) {
throw Object.assign(new Error(`Routine '${id}' not found`), { code: "ENOENT" });
}
return routine;
}),
getDueRoutines: vi.fn().mockResolvedValue(routines),
recordRun: vi.fn().mockImplementation((id: string, result: RoutineExecutionResult) => {
return createMockRoutine({ id, lastRunResult: result });
}),
startRoutineExecution: vi.fn().mockResolvedValue(undefined),
completeRoutineExecution: vi.fn().mockResolvedValue(undefined),
cancelRoutineExecution: vi.fn().mockResolvedValue(undefined),
init: vi.fn().mockResolvedValue(undefined),
on: vi.fn(),
off: vi.fn(),
} as unknown as RoutineStore;
}
function createMockRoutineRunner(): RoutineRunner {
return {
executeRoutine: vi.fn().mockResolvedValue({
routineId: "test-routine",
success: true,
output: "Success",
startedAt: new Date().toISOString(),
completedAt: new Date().toISOString(),
} as RoutineExecutionResult),
handleCatchUp: vi.fn().mockResolvedValue(undefined),
getInFlightCount: vi.fn().mockReturnValue(0),
isRoutineRunning: vi.fn().mockReturnValue(false),
} as unknown as RoutineRunner;
}
function createRoutineScheduler(
taskStore?: TaskStore,
routineStore?: RoutineStore,
routineRunner?: RoutineRunner,
options?: Partial<Pick<RoutineSchedulerOptions, "pollIntervalMs">>,
): RoutineScheduler {
return new RoutineScheduler({
taskStore: taskStore ?? createMockTaskStore(),
routineStore: routineStore ?? createMockRoutineStore(),
routineRunner: routineRunner ?? createMockRoutineRunner(),
pollIntervalMs: options?.pollIntervalMs,
});
}
describe("RoutineScheduler", () => {
let mockRoutineStore: RoutineStore;
let mockRoutineRunner: RoutineRunner;
let mockGetSettings: ReturnType<typeof vi.fn>;
let onStart: ReturnType<typeof vi.fn>;
let onStop: ReturnType<typeof vi.fn>;
let scheduler: RoutineScheduler;
const createMockRoutine = (overrides: Partial<Routine> = {}): Routine =>
({
id: "routine-1",
agentId: "agent-1",
name: "Test Routine",
description: "A test routine",
enabled: true,
trigger: {
type: "cron",
cronExpression: "*/5 * * * *",
},
executionPolicy: "reject",
catchUpPolicy: "skip",
catchUpLimit: 5,
lastRunAt: null,
nextRunAt: new Date().toISOString(),
runCount: 0,
runHistory: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
...overrides,
} as Routine);
beforeEach(() => {
mockRoutineStore = {
getRoutine: vi.fn(),
listRoutines: vi.fn(),
createRoutine: vi.fn(),
updateRoutine: vi.fn(),
deleteRoutine: vi.fn(),
getDueRoutines: vi.fn().mockResolvedValue([]),
startRoutineExecution: vi.fn().mockResolvedValue(undefined),
completeRoutineExecution: vi.fn().mockResolvedValue(undefined),
cancelRoutineExecution: vi.fn().mockResolvedValue(undefined),
recordRun: vi.fn(),
on: vi.fn(),
off: vi.fn(),
} as unknown as RoutineStore;
mockRoutineRunner = {
execute: vi.fn().mockResolvedValue({ success: true }),
isExecuting: vi.fn().mockReturnValue(false),
determineCatchUp: vi.fn().mockReturnValue({ shouldCatchUp: false }),
clearInFlight: vi.fn(),
} as unknown as RoutineRunner;
mockGetSettings = vi.fn().mockReturnValue({
globalPause: false,
enginePaused: false,
});
onStart = vi.fn();
onStop = vi.fn();
scheduler = new RoutineScheduler({
routineStore: mockRoutineStore,
routineRunner: mockRoutineRunner,
pollIntervalMs: 1000, // Fast for testing
getSettings: mockGetSettings as () => ProjectSettings,
onStart,
onStop,
});
});
afterEach(() => {
scheduler.stop();
if (scheduler) {
scheduler.stop();
}
vi.clearAllMocks();
});
describe("lifecycle", () => {
it("should start and stop correctly", () => {
scheduler.start();
expect(scheduler.getStatus()).toBe("running");
expect(onStart).toHaveBeenCalled();
describe("constructor", () => {
it("clamps poll interval to minimum 10000ms", () => {
scheduler = createRoutineScheduler(
undefined,
undefined,
undefined,
{ pollIntervalMs: 100 }, // Below minimum
);
scheduler.stop();
expect(scheduler.getStatus()).toBe("stopped");
expect(onStop).toHaveBeenCalled();
// The scheduler should have clamped to 10000ms
expect(scheduler["pollIntervalMs"]).toBe(10000);
});
it("should not start twice", () => {
scheduler.start();
scheduler.start(); // Second start should be no-op
expect(onStart).toHaveBeenCalledTimes(1);
scheduler.stop();
});
it("should not stop twice", () => {
scheduler.start();
scheduler.stop();
scheduler.stop(); // Second stop should be no-op
expect(onStop).toHaveBeenCalledTimes(1);
});
it("should subscribe to routine store events on start", () => {
scheduler.start();
expect(mockRoutineStore.on).toHaveBeenCalledWith(
"routine:created",
expect.any(Function)
);
expect(mockRoutineStore.on).toHaveBeenCalledWith(
"routine:updated",
expect.any(Function)
);
expect(mockRoutineStore.on).toHaveBeenCalledWith(
"routine:deleted",
expect.any(Function)
);
});
it("should unsubscribe from routine store events on stop", () => {
scheduler.start();
scheduler.stop();
expect(mockRoutineStore.off).toHaveBeenCalledWith(
"routine:created",
expect.any(Function)
);
expect(mockRoutineStore.off).toHaveBeenCalledWith(
"routine:updated",
expect.any(Function)
);
expect(mockRoutineStore.off).toHaveBeenCalledWith(
"routine:deleted",
expect.any(Function)
);
it("uses default poll interval of 60000ms when not specified", () => {
scheduler = createRoutineScheduler();
expect(scheduler["pollIntervalMs"]).toBe(60000);
});
});
describe("poll behavior", () => {
it("should skip poll when globalPause is true", async () => {
mockGetSettings.mockReturnValue({ globalPause: true });
describe("start/stop", () => {
it("sets running = true after start", () => {
scheduler = createRoutineScheduler();
scheduler.start();
await new Promise((resolve) => setTimeout(resolve, 1500));
expect(mockRoutineStore.getDueRoutines).not.toHaveBeenCalled();
expect(scheduler.isActive()).toBe(true);
});
it("should skip poll when enginePaused is true", async () => {
mockGetSettings.mockReturnValue({ enginePaused: true });
it("clears interval and sets running = false after stop", () => {
scheduler = createRoutineScheduler();
scheduler.start();
await new Promise((resolve) => setTimeout(resolve, 1500));
expect(mockRoutineStore.getDueRoutines).not.toHaveBeenCalled();
scheduler.stop();
expect(scheduler.isActive()).toBe(false);
});
it("should process due routines", async () => {
const routine = createMockRoutine();
vi.mocked(mockRoutineStore.getDueRoutines).mockResolvedValue([routine]);
it("runs first tick immediately on start", async () => {
const routineStore = createMockRoutineStore([createMockRoutine({ id: "due-routine" })]);
const routineRunner = createMockRoutineRunner();
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore, routineRunner);
scheduler.start();
// Wait for the tick to complete
await new Promise((r) => setTimeout(r, 10));
scheduler.stop();
await new Promise((resolve) => setTimeout(resolve, 1500));
expect(mockRoutineStore.getDueRoutines).toHaveBeenCalled();
expect(mockRoutineRunner.execute).toHaveBeenCalledWith(routine);
expect(routineRunner.handleCatchUp).toHaveBeenCalled();
expect(routineRunner.executeRoutine).toHaveBeenCalled();
});
it("should skip disabled routines", async () => {
const routine = createMockRoutine({ enabled: false });
vi.mocked(mockRoutineStore.getDueRoutines).mockResolvedValue([routine]);
it("is idempotent on start", () => {
scheduler = createRoutineScheduler();
scheduler.start();
await new Promise((resolve) => setTimeout(resolve, 1500));
expect(mockRoutineRunner.execute).not.toHaveBeenCalled();
scheduler.start(); // Should not throw
expect(scheduler.isActive()).toBe(true);
});
it("should skip routines that are already executing", async () => {
const routine = createMockRoutine();
vi.mocked(mockRoutineStore.getDueRoutines).mockResolvedValue([routine]);
vi.mocked(mockRoutineRunner.isExecuting).mockReturnValue(true);
scheduler.start();
it("is safe to stop when not started", () => {
scheduler = createRoutineScheduler();
expect(() => scheduler.stop()).not.toThrow();
});
});
await new Promise((resolve) => setTimeout(resolve, 1500));
describe("tick", () => {
it("skips when globalPause is true", async () => {
const routineStore = createMockRoutineStore([createMockRoutine()]);
const routineRunner = createMockRoutineRunner();
scheduler = createRoutineScheduler(
createMockTaskStore({ globalPause: true }),
routineStore,
routineRunner,
);
expect(mockRoutineRunner.execute).not.toHaveBeenCalled();
await scheduler.tick();
expect(routineStore.getDueRoutines).not.toHaveBeenCalled();
});
it("should handle catch-up execution", async () => {
const routine = createMockRoutine({ catchUpPolicy: "run_one" });
vi.mocked(mockRoutineStore.getDueRoutines).mockResolvedValue([routine]);
vi.mocked(mockRoutineRunner.determineCatchUp).mockReturnValue({
shouldCatchUp: true,
catchUpFrom: "2024-01-01T00:00:00.000Z",
});
scheduler.start();
it("skips when enginePaused is true", async () => {
const routineStore = createMockRoutineStore([createMockRoutine()]);
const routineRunner = createMockRoutineRunner();
scheduler = createRoutineScheduler(
createMockTaskStore({ enginePaused: true }),
routineStore,
routineRunner,
);
await new Promise((resolve) => setTimeout(resolve, 1500));
await scheduler.tick();
expect(mockRoutineRunner.execute).toHaveBeenCalledWith(routine, {
catchUpFrom: "2024-01-01T00:00:00.000Z",
});
expect(routineStore.getDueRoutines).not.toHaveBeenCalled();
});
it("should isolate per-routine failures", async () => {
it("skips when no routines are due", async () => {
const routineStore = createMockRoutineStore([]);
const routineRunner = createMockRoutineRunner();
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore, routineRunner);
await scheduler.tick();
expect(routineRunner.handleCatchUp).not.toHaveBeenCalled();
expect(routineRunner.executeRoutine).not.toHaveBeenCalled();
});
it("processes due routines in order", async () => {
const routine1 = createMockRoutine({ id: "routine-1" });
const routine2 = createMockRoutine({ id: "routine-2" });
vi.mocked(mockRoutineStore.getDueRoutines).mockResolvedValue([
routine1,
routine2,
]);
vi.mocked(mockRoutineRunner.execute)
.mockRejectedValueOnce(new Error("Failed"))
.mockResolvedValueOnce({ success: true, routineId: "routine-2", executedAt: new Date().toISOString() });
scheduler.start();
const routineStore = createMockRoutineStore([routine1, routine2]);
const routineRunner = createMockRoutineRunner();
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore, routineRunner);
await new Promise((resolve) => setTimeout(resolve, 1500));
await scheduler.tick();
// Both should have been attempted despite the first failure
expect(mockRoutineRunner.execute).toHaveBeenCalledTimes(2);
// Both routines should be processed
expect(routineRunner.handleCatchUp).toHaveBeenCalledTimes(2);
expect(routineRunner.executeRoutine).toHaveBeenCalledTimes(2);
});
it("calls handleCatchUp then executeRoutine for each routine", async () => {
const routine1 = createMockRoutine({ id: "routine-order-1" });
const routine2 = createMockRoutine({ id: "routine-order-2" });
const routineStore = createMockRoutineStore([routine1, routine2]);
const routineRunner = createMockRoutineRunner();
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore, routineRunner);
await scheduler.tick();
// Both routines should be processed
expect(routineRunner.handleCatchUp).toHaveBeenCalledTimes(2);
expect(routineRunner.executeRoutine).toHaveBeenCalledTimes(2);
});
it("handles errors in individual routine execution gracefully", async () => {
const routine = createMockRoutine({ id: "routine-error" });
const routineStore = createMockRoutineStore([routine]);
const routineRunner = createMockRoutineRunner();
vi.mocked(routineRunner.executeRoutine).mockRejectedValueOnce(
new Error("Execution failed"),
);
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore, routineRunner);
// Should not throw
await expect(scheduler.tick()).resolves.toBeUndefined();
});
it("re-entrance guard prevents overlapping ticks", async () => {
const routine = createMockRoutine({ id: "routine-reentrant" });
const routineStore = createMockRoutineStore([routine]);
const routineRunner = createMockRoutineRunner();
// Make execution slow
vi.mocked(routineRunner.executeRoutine).mockImplementation(async () => {
await new Promise((r) => setTimeout(r, 50));
return {
routineId: "routine-reentrant",
success: true,
output: "",
startedAt: new Date().toISOString(),
completedAt: new Date().toISOString(),
} as RoutineExecutionResult;
});
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore, routineRunner);
// Start two ticks concurrently
const [tick1, tick2] = [scheduler.tick(), scheduler.tick()];
await Promise.all([tick1, tick2]);
// Should only have executed once (second tick was blocked by re-entrance guard)
expect(routineRunner.executeRoutine).toHaveBeenCalledTimes(1);
});
});
describe("re-entrance guard", () => {
it("should skip concurrent polls", async () => {
// Make getDueRoutines slow
vi.mocked(mockRoutineStore.getDueRoutines).mockImplementation(
() => new Promise((resolve) => setTimeout(() => resolve([]), 500))
describe("triggerManual", () => {
it("delegates to routineRunner.executeRoutine with 'api' trigger", async () => {
const routineRunner = createMockRoutineRunner();
scheduler = createRoutineScheduler(createMockTaskStore(), createMockRoutineStore(), routineRunner);
await scheduler.triggerManual("test-routine");
expect(routineRunner.executeRoutine).toHaveBeenCalledWith("test-routine", "api");
});
it("passes through the result from routineRunner", async () => {
const mockResult: RoutineExecutionResult = {
routineId: "test-routine",
success: true,
output: "Success",
startedAt: new Date().toISOString(),
completedAt: new Date().toISOString(),
};
const routineRunner = createMockRoutineRunner();
vi.mocked(routineRunner.executeRoutine).mockResolvedValue(mockResult);
scheduler = createRoutineScheduler(createMockTaskStore(), createMockRoutineStore(), routineRunner);
const result = await scheduler.triggerManual("test-routine");
expect(result).toEqual(mockResult);
});
});
describe("triggerWebhook", () => {
it("delegates to routineRunner.executeRoutine with 'webhook' trigger", async () => {
const routine = createMockRoutine({ id: "webhook-routine", trigger: { type: "webhook", webhookPath: "/test" } });
const routineStore = createMockRoutineStore([routine]);
const routineRunner = createMockRoutineRunner();
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore, routineRunner);
await scheduler.triggerWebhook("webhook-routine", { data: "test" });
expect(routineRunner.executeRoutine).toHaveBeenCalledWith(
"webhook-routine",
"webhook",
expect.objectContaining({ webhookPayload: { data: "test" } }),
);
});
it("throws for routine with non-webhook trigger type", async () => {
const routine = createMockRoutine({ id: "cron-routine", trigger: { type: "cron", cronExpression: "0 * * * *" } });
const routineStore = createMockRoutineStore([routine]);
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore);
await expect(scheduler.triggerWebhook("cron-routine", {})).rejects.toThrow(
"does not have webhook trigger type",
);
});
it("throws for nonexistent routine", async () => {
const routineStore = createMockRoutineStore([]);
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore);
await expect(scheduler.triggerWebhook("nonexistent", {})).rejects.toThrow(
"not found",
);
});
it("proceeds without signature check when no secret is configured", async () => {
// Clear the env var if it exists
const originalSecret = process.env.FUSION_ROUTINE_WEBHOOK_SECRET;
delete process.env.FUSION_ROUTINE_WEBHOOK_SECRET;
try {
const routine = createMockRoutine({ id: "webhook-no-secret", trigger: { type: "webhook", webhookPath: "/test" } });
const routineStore = createMockRoutineStore([routine]);
const routineRunner = createMockRoutineRunner();
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore, routineRunner);
// Should not throw even without signature
await scheduler.triggerWebhook("webhook-no-secret", { data: "test" });
expect(routineRunner.executeRoutine).toHaveBeenCalled();
} finally {
// Restore env var
if (originalSecret !== undefined) {
process.env.FUSION_ROUTINE_WEBHOOK_SECRET = originalSecret;
}
}
});
it("throws for invalid HMAC signature when secret is configured", async () => {
process.env.FUSION_ROUTINE_WEBHOOK_SECRET = "test-secret";
try {
const routine = createMockRoutine({ id: "webhook-with-secret", trigger: { type: "webhook", webhookPath: "/test" } });
const routineStore = createMockRoutineStore([routine]);
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore);
await expect(
scheduler.triggerWebhook("webhook-with-secret", { data: "test" }, "sha256=invalidsignature"),
).rejects.toThrow("Invalid webhook signature");
} finally {
delete process.env.FUSION_ROUTINE_WEBHOOK_SECRET;
}
});
it("accepts valid HMAC signature", async () => {
const secret = "test-secret-123";
process.env.FUSION_ROUTINE_WEBHOOK_SECRET = secret;
try {
const routine = createMockRoutine({ id: "webhook-valid-sig", trigger: { type: "webhook", webhookPath: "/test" } });
const routineStore = createMockRoutineStore([routine]);
const routineRunner = createMockRoutineRunner();
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore, routineRunner);
const payload = { data: "test" };
const crypto = await import("node:crypto");
const signature = "sha256=" + crypto.createHmac("sha256", secret).update(JSON.stringify(payload)).digest("hex");
await scheduler.triggerWebhook("webhook-valid-sig", payload, signature);
expect(routineRunner.executeRoutine).toHaveBeenCalled();
} finally {
delete process.env.FUSION_ROUTINE_WEBHOOK_SECRET;
}
});
it("throws when signature is missing but secret is configured", async () => {
process.env.FUSION_ROUTINE_WEBHOOK_SECRET = "test-secret";
try {
const routine = createMockRoutine({ id: "webhook-missing-sig", trigger: { type: "webhook", webhookPath: "/test" } });
const routineStore = createMockRoutineStore([routine]);
scheduler = createRoutineScheduler(createMockTaskStore(), routineStore);
await expect(
scheduler.triggerWebhook("webhook-missing-sig", { data: "test" }),
).rejects.toThrow("Missing webhook signature");
} finally {
delete process.env.FUSION_ROUTINE_WEBHOOK_SECRET;
}
});
});
describe("isActive", () => {
it("returns false initially", () => {
scheduler = createRoutineScheduler();
expect(scheduler.isActive()).toBe(false);
});
it("returns true after start", () => {
scheduler = createRoutineScheduler();
scheduler.start();
expect(scheduler.isActive()).toBe(true);
scheduler.stop();
});
// Trigger immediate poll
await scheduler.triggerPoll();
// The slow poll should be running, triggerPoll should complete quickly
await new Promise((resolve) => setTimeout(resolve, 100));
expect(scheduler.getStatus()).toBe("running");
it("returns false after stop", () => {
scheduler = createRoutineScheduler();
scheduler.start();
scheduler.stop();
expect(scheduler.isActive()).toBe(false);
});
});
});