Files
fusion/packages/engine/src/restart.integration.test.ts
2026-04-12 06:50:53 -07:00

1338 lines
49 KiB
TypeScript

/**
* Integration tests for engine restart resilience.
*
* Verifies that after an engine crash/restart:
* - In-progress tasks resume from their current step (not from scratch)
* - Existing worktrees are reused rather than recreated
* - Step progress survives and is communicated to the agent
* - In-review tasks get re-queued for merge
* - Triage re-picks unspecified tasks
* - Crash scenarios are handled gracefully (semaphore release, status cleanup)
*/
import { describe, it, expect, vi, beforeEach } from "vitest";
import { AgentSemaphore } from "./concurrency.js";
// ── Module-level mocks (matching existing test patterns) ──────────────────
vi.mock("./pi.js", () => ({
createKbAgent: vi.fn(),
describeModel: vi.fn().mockReturnValue("mock-provider/mock-model"),
promptWithFallback: vi.fn(async (session, prompt, options) => {
if (options === undefined) {
await session.prompt(prompt);
} else {
await session.prompt(prompt, options);
}
}),
}));
vi.mock("./reviewer.js", () => ({
reviewStep: vi.fn(),
}));
vi.mock("node:child_process", () => {
const { promisify } = require("node:util");
const execSyncFn = vi.fn().mockReturnValue(Buffer.from(""));
const execFn: any = vi.fn((cmd: any, opts: any, cb: any) => {
const callback = typeof opts === "function" ? opts : cb;
const forwardedOpts = typeof opts === "function" ? undefined : opts;
try {
const out = execSyncFn(cmd, forwardedOpts);
const stdout = out === undefined ? "" : out.toString();
if (typeof callback === "function") callback(null, stdout, "");
} catch (err: any) {
if (typeof callback === "function") {
callback(err, err?.stdout?.toString?.() ?? "", err?.stderr?.toString?.() ?? "");
}
}
});
// Mirror real child_process.exec: promisify resolves to { stdout, stderr }.
execFn[promisify.custom] = (cmd: any, opts?: any) =>
new Promise((resolve, reject) => {
execFn(cmd, opts, (err: any, stdout: any, stderr: any) => {
if (err) {
err.stdout = stdout;
err.stderr = stderr;
reject(err);
} else {
resolve({ stdout, stderr });
}
});
});
return { execSync: execSyncFn, exec: execFn };
});
vi.mock("node:fs", () => ({
existsSync: vi.fn().mockReturnValue(true),
readdirSync: vi.fn().mockReturnValue([]),
}));
vi.mock("node:fs/promises", () => ({
readFile: vi.fn().mockResolvedValue("# Task prompt content"),
}));
vi.mock("@mariozechner/pi-ai", () => ({
Type: {
Object: (props: Record<string, unknown>) => ({ type: "object", properties: props }),
String: (opts?: unknown) => ({ type: "string", ...((opts as object) ?? {}) }),
Number: (opts?: unknown) => ({ type: "number", ...((opts as object) ?? {}) }),
Boolean: (opts?: unknown) => ({ type: "boolean", ...((opts as object) ?? {}) }),
Optional: (schema: unknown) => schema,
Array: (schema: unknown, opts?: unknown) => ({ type: "array", items: schema, ...((opts as object) ?? {}) }),
Union: (schemas: unknown[], opts?: unknown) => ({ anyOf: schemas, ...((opts as object) ?? {}) }),
Literal: (value: unknown) => ({ const: value }),
},
}));
vi.mock("@mariozechner/pi-coding-agent", () => {
const mockSessionManager = {};
return {
SessionManager: {
create: vi.fn().mockReturnValue(mockSessionManager),
open: vi.fn().mockReturnValue(mockSessionManager),
inMemory: vi.fn().mockReturnValue(mockSessionManager),
},
};
});
import { TaskExecutor } from "./executor.js";
import { TriageProcessor } from "./triage.js";
import { Scheduler } from "./scheduler.js";
import { aiMergeTask } from "./merger.js";
import { WorktreePool, scanIdleWorktrees, cleanupOrphanedWorktrees } from "./worktree-pool.js";
import { createKbAgent } from "./pi.js";
import { execSync } from "node:child_process";
import { existsSync, readdirSync } from "node:fs";
import type { Task, TaskDetail, TaskStep, Column, Settings, StepStatus } from "@fusion/core";
const mockedCreateHaiAgent = vi.mocked(createKbAgent);
const mockedExecSync = vi.mocked(execSync);
const mockedExistsSync = vi.mocked(existsSync);
const mockedReaddirSync = vi.mocked(readdirSync);
// ── Mock helpers ──────────────────────────────────────────────────────────
const DEFAULT_SETTINGS: Settings = {
maxConcurrent: 2,
maxWorktrees: 4,
pollIntervalMs: 15000,
groupOverlappingFiles: false,
autoMerge: false,
worktreeInitCommand: undefined,
};
function createMockStore(overrides: Record<string, any> = {}) {
const listeners = new Map<string, Function[]>();
return {
on: vi.fn((event: string, fn: Function) => {
const existing = listeners.get(event) || [];
existing.push(fn);
listeners.set(event, existing);
}),
emit: vi.fn(),
listTasks: vi.fn().mockResolvedValue([]),
getTask: vi.fn().mockResolvedValue(makeTaskDetail("FN-001", "in-progress")),
updateTask: vi.fn().mockResolvedValue({}),
moveTask: vi.fn().mockImplementation(async (id: string, col: Column) => {
return makeTask(id, col);
}),
logEntry: vi.fn().mockResolvedValue(undefined),
appendAgentLog: vi.fn().mockResolvedValue(undefined),
parseStepsFromPrompt: vi.fn().mockResolvedValue([]),
parseFileScopeFromPrompt: vi.fn().mockResolvedValue([]),
getSettings: vi.fn().mockResolvedValue({ ...DEFAULT_SETTINGS }),
getRootDir: vi.fn().mockReturnValue("/tmp/root"),
getTasksDir: vi.fn().mockReturnValue("/tmp/root/.fusion/tasks"),
updateStep: vi.fn().mockImplementation(async (id: string, step: number, status: StepStatus) => {
return makeTaskDetail(id, "in-progress");
}),
createTask: vi.fn().mockImplementation(async (input: any) => {
return makeTask("FN-NEW", "triage");
}),
deleteTask: vi.fn().mockResolvedValue(undefined),
_trigger(event: string, ...args: any[]) {
for (const fn of listeners.get(event) || []) fn(...args);
},
_listeners: listeners,
...overrides,
} as any;
}
function makeTask(id: string, column: Column, overrides: Partial<Task> = {}): Task {
return {
id,
title: `Task ${id}`,
description: `Description for ${id}`,
column,
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
...overrides,
};
}
function makeTaskDetail(id: string, column: Column, overrides: Partial<TaskDetail> = {}): TaskDetail {
return {
...makeTask(id, column, overrides),
prompt: overrides.prompt ?? "# test\n## Steps\n### Step 0: Preflight\n- [ ] check\n## Review Level: 0",
...overrides,
};
}
function makeSteps(...statuses: StepStatus[]): TaskStep[] {
return statuses.map((status, i) => ({
name: `Step ${i}`,
status,
}));
}
function mockAgentSuccess() {
mockedCreateHaiAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockResolvedValue(undefined),
dispose: vi.fn(),
},
} as any);
}
function mockAgentFailure(error = "agent crashed") {
mockedCreateHaiAgent.mockRejectedValue(new Error(error));
}
/**
* Create a mock agent that auto-triggers the task_done tool when prompt is called.
* This simulates a successful task execution where the agent calls task_done(),
* preventing the executor from entering the "finished without task_done — retrying
* with new session" branch. Follows the same pattern as executor.test.ts.
*
* Uses per-session local customTools capture to avoid race conditions when
* multiple tasks execute concurrently.
*/
function createAgentWithTaskDone() {
mockedCreateHaiAgent.mockImplementation((async (opts: any) => {
// Capture tools per-session to avoid race conditions with concurrent tasks
const localCustomTools = opts.customTools || [];
const session = {
prompt: vi.fn().mockImplementation(async () => {
// Find and execute task_done tool to set taskDone = true
const taskDoneTool = localCustomTools.find((t: any) => t.name === "task_done");
if (taskDoneTool) {
await taskDoneTool.execute("tool-1", {});
}
}),
dispose: vi.fn(),
};
return { session };
}) as any);
}
// ── Tests begin ───────────────────────────────────────────────────────────
beforeEach(() => {
vi.clearAllMocks();
mockedExistsSync.mockReturnValue(true); // Default: worktrees exist (resume scenario)
mockedExecSync.mockImplementation((cmd: any) => {
if (String(cmd) === "git worktree list --porcelain") {
return [
"worktree /tmp/test",
"HEAD abc123",
"branch refs/heads/main",
"",
"worktree /tmp/wt/KB-010",
"HEAD def456",
"branch refs/heads/fusion/fn-010",
"",
"worktree /tmp/wt/FN-963",
"HEAD def456",
"branch refs/heads/fusion/fn-963",
"",
"worktree /root/.worktrees/active-wt",
"HEAD def456",
"branch refs/heads/fusion/active-wt",
"",
"worktree /root/.worktrees/review-wt",
"HEAD def456",
"branch refs/heads/fusion/review-wt",
"",
"worktree /root/.worktrees/idle-wt",
"HEAD def456",
"branch refs/heads/fusion/idle-wt",
"",
].join("\n") as any;
}
return Buffer.from("");
});
});
// ── Step 2: In-progress task resume tests ─────────────────────────────────
describe("In-progress task resume after restart", () => {
it("resumeOrphaned() calls execute() for each in-progress task not already executing", async () => {
const store = createMockStore();
const task1 = makeTask("FN-001", "in-progress");
const task2 = makeTask("FN-002", "in-progress");
const taskDone = makeTask("FN-003", "done");
store.listTasks.mockResolvedValue([task1, task2, taskDone]);
// Use deterministic mock that calls task_done to prevent retry-with-new-session
createAgentWithTaskDone();
const executor = new TaskExecutor(store, "/tmp/test");
await executor.resumeOrphaned();
// Wait for async execute calls to complete
await new Promise((r) => setTimeout(r, 50));
// Exactly one agent session per in-progress task (no retry inflation)
expect(mockedCreateHaiAgent).toHaveBeenCalledTimes(2);
// Both tasks should have received resume log entries (behavioral guarantee)
expect(store.logEntry).toHaveBeenCalledWith("FN-001", "Resumed after engine restart");
expect(store.logEntry).toHaveBeenCalledWith("FN-002", "Resumed after engine restart");
});
it("resumed task reuses existing worktree — no git worktree add called", async () => {
const store = createMockStore();
const task = makeTask("FN-010", "in-progress", {
worktree: "/tmp/wt/KB-010",
});
store.listTasks.mockResolvedValue([task]);
store.getTask.mockResolvedValue(makeTaskDetail("FN-010", "in-progress", {
worktree: "/tmp/wt/KB-010",
}));
// Worktree exists on disk
mockedExistsSync.mockReturnValue(true);
mockAgentSuccess();
const executor = new TaskExecutor(store, "/tmp/test");
await executor.resumeOrphaned();
await new Promise((r) => setTimeout(r, 50));
// No git worktree add commands should have been called
const gitWorktreeAddCalls = mockedExecSync.mock.calls.filter(
(call) => typeof call[0] === "string" && call[0].includes("git worktree add"),
);
expect(gitWorktreeAddCalls).toHaveLength(0);
});
it("resumed task with step progress includes RESUMING section in agent prompt", async () => {
const store = createMockStore();
const steps = makeSteps("done", "done", "done", "in-progress", "pending");
const task = makeTask("FN-020", "in-progress", { steps, currentStep: 3 });
store.listTasks.mockResolvedValue([task]);
store.getTask.mockResolvedValue(makeTaskDetail("FN-020", "in-progress", {
steps,
currentStep: 3,
}));
let capturedPrompt = "";
mockedCreateHaiAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockImplementation(async (prompt: string) => {
capturedPrompt = prompt;
}),
dispose: vi.fn(),
},
} as any);
const executor = new TaskExecutor(store, "/tmp/test");
await executor.resumeOrphaned();
await new Promise((r) => setTimeout(r, 50));
expect(capturedPrompt).toContain("⚠️ RESUMING");
expect(capturedPrompt).toContain("Step 0 (Step 0): **done**");
expect(capturedPrompt).toContain("Step 3 (Step 3): **in-progress**");
expect(capturedPrompt).toContain("Resume from: Step 3");
});
it("resumed task does NOT re-run worktreeInitCommand", async () => {
const store = createMockStore();
store.getSettings.mockResolvedValue({
...DEFAULT_SETTINGS,
worktreeInitCommand: "pnpm install",
});
const task = makeTask("FN-030", "in-progress");
store.listTasks.mockResolvedValue([task]);
store.getTask.mockResolvedValue(makeTaskDetail("FN-030", "in-progress"));
mockedExistsSync.mockReturnValue(true); // worktree exists
mockAgentSuccess();
const executor = new TaskExecutor(store, "/tmp/test");
await executor.resumeOrphaned();
await new Promise((r) => setTimeout(r, 50));
// getSettings is called (for project commands in execution prompt) but init command should not run
expect(store.getSettings).toHaveBeenCalled();
// No init command calls
const initCalls = mockedExecSync.mock.calls.filter(
(call) => call[0] === "pnpm install",
);
expect(initCalls).toHaveLength(0);
});
it("resumeOrphaned() logs 'Resumed after engine restart' for each orphaned task", async () => {
const store = createMockStore();
const task1 = makeTask("FN-040", "in-progress");
const task2 = makeTask("FN-041", "in-progress");
store.listTasks.mockResolvedValue([task1, task2]);
store.getTask.mockImplementation(async (id: string) =>
makeTaskDetail(id, "in-progress"),
);
mockAgentSuccess();
const executor = new TaskExecutor(store, "/tmp/test");
await executor.resumeOrphaned();
await new Promise((r) => setTimeout(r, 50));
expect(store.logEntry).toHaveBeenCalledWith("FN-040", "Resumed after engine restart");
expect(store.logEntry).toHaveBeenCalledWith("FN-041", "Resumed after engine restart");
});
it("resumeOrphaned() fast-paths already-complete tasks to in-review", async () => {
const store = createMockStore();
const completedTask = makeTask("FN-963", "in-progress", {
worktree: "/tmp/wt/FN-963",
baseCommitSha: "base123",
steps: makeSteps("done", "done", "skipped"),
});
store.listTasks.mockResolvedValue([completedTask]);
store.getTask.mockResolvedValue(makeTaskDetail("FN-963", "in-progress", {
worktree: "/tmp/wt/FN-963",
baseCommitSha: "base123",
steps: makeSteps("done", "done", "skipped"),
}));
const executor = new TaskExecutor(store, "/tmp/test");
const executeSpy = vi.spyOn(executor, "execute");
mockedExecSync.mockImplementation((command) => {
const cmd = String(command);
if (cmd === 'git diff --name-only base123..HEAD') {
return "packages/dashboard/app/components/SettingsModal.tsx\n" as any;
}
return "" as any;
});
await executor.resumeOrphaned();
expect(executeSpy).not.toHaveBeenCalled();
expect(store.updateTask).toHaveBeenCalledWith("FN-963", {
modifiedFiles: ["packages/dashboard/app/components/SettingsModal.tsx"],
});
expect(store.moveTask).toHaveBeenCalledWith("FN-963", "in-review");
expect(store.logEntry).toHaveBeenCalledWith(
"FN-963",
"Auto-recovered: task work was complete but stuck in in-progress — moved to in-review",
);
});
it("recoverCompletedTask() marks task failed then moves to in-review when workflow fails", async () => {
const store = createMockStore({
getTask: vi.fn().mockResolvedValue(makeTaskDetail("FN-963", "in-progress", {
worktree: "/tmp/wt/FN-963",
steps: makeSteps("done"),
enabledWorkflowSteps: ["wf-1"],
})),
getWorkflowStep: vi.fn().mockResolvedValue({
id: "wf-1",
name: "Build",
mode: "script",
scriptName: "pnpm test",
phase: "pre-merge",
}),
});
const task = makeTask("FN-963", "in-progress", {
worktree: "/tmp/wt/FN-963",
steps: makeSteps("done"),
});
mockedExecSync.mockImplementation((command) => {
const cmd = String(command);
if (cmd === "pnpm test") {
throw new Error("tests failed");
}
return "" as any;
});
const executor = new TaskExecutor(store, "/tmp/test");
const recovered = await executor.recoverCompletedTask(task);
expect(recovered).toBe(true);
expect(store.updateTask).toHaveBeenCalledWith("FN-963", {
status: "failed",
error: "Workflow step failed during recovery",
});
expect(store.moveTask).toHaveBeenCalledWith("FN-963", "in-review");
});
});
// ── Step 3: In-review merge re-queue tests ────────────────────────────────
//
// The merge queue/enqueueMerge logic lives in dashboard.ts (CLI layer).
// These tests focus on what @fusion/engine owns: aiMergeTask() behaviour
// relevant to restart resilience — state validation, status lifecycle,
// and error handling with git reset --merge cleanup.
describe("In-review merge handling after restart", () => {
it("aiMergeTask validates task is in 'in-review' before merging", async () => {
const store = createMockStore();
store.getTask.mockResolvedValue(makeTaskDetail("FN-050", "in-progress"));
await expect(aiMergeTask(store, "/tmp/root", "FN-050")).rejects.toThrow(
"Cannot merge FN-050: task is in 'in-progress', must be in 'in-review'",
);
// No git commands should have been executed
expect(mockedExecSync).not.toHaveBeenCalled();
});
it("aiMergeTask sets status to 'merging' during execution and clears on success", async () => {
const store = createMockStore();
store.getTask.mockResolvedValue(makeTaskDetail("FN-051", "in-review"));
store.moveTask.mockResolvedValue(makeTask("FN-051", "done"));
// Branch exists, merge succeeds, no conflicts
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
// Post-squash check: squash staged changes → "1"
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
// Post-agent check: agent committed → "0"
if (cmdStr.includes("diff --cached")) return "0" as any;
return Buffer.from("");
});
mockAgentSuccess();
await aiMergeTask(store, "/tmp/root", "FN-051");
// Should have set status to "merging"
expect(store.updateTask).toHaveBeenCalledWith("FN-051", { status: "merging" });
// Should have cleared status via completeTask (status: null before moveTask)
expect(store.updateTask).toHaveBeenCalledWith("FN-051", { status: null });
});
it("sequential aiMergeTask calls for multiple in-review tasks all succeed", async () => {
const taskIds = ["FN-052", "FN-053", "FN-054"];
for (const taskId of taskIds) {
const store = createMockStore();
store.getTask.mockResolvedValue(makeTaskDetail(taskId, "in-review"));
store.moveTask.mockResolvedValue(makeTask(taskId, "done"));
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
if (cmdStr.includes("diff --cached")) return "0" as any;
return Buffer.from("");
});
mockAgentSuccess();
const result = await aiMergeTask(store, "/tmp/root", taskId);
expect(result.merged).toBe(true);
expect(store.moveTask).toHaveBeenCalledWith(taskId, "done");
}
});
it("aiMergeTask throws on agent failure during session.prompt and calls git reset --merge", async () => {
const store = createMockStore();
store.getTask.mockResolvedValue(makeTaskDetail("FN-055", "in-review"));
// Branch exists, merge starts, agent creates but prompt fails
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
if (cmdStr.includes("diff --cached")) return "0" as any;
return Buffer.from("");
});
mockedCreateHaiAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockRejectedValue(new Error("merge agent crashed")),
dispose: vi.fn(),
},
} as any);
await expect(aiMergeTask(store, "/tmp/root", "FN-055")).rejects.toThrow(
"AI merge failed for FN-055: all 3 attempts exhausted",
);
// Should have attempted git reset --merge cleanup
const resetCalls = mockedExecSync.mock.calls.filter(
(call) => typeof call[0] === "string" && call[0].includes("git reset --merge"),
);
expect(resetCalls.length).toBeGreaterThan(0);
// Status was set to "merging" but NOT cleared by aiMergeTask (that's the dashboard's job)
expect(store.updateTask).toHaveBeenCalledWith("FN-055", { status: "merging" });
});
it("aiMergeTask moves task to done when branch does not exist", async () => {
const store = createMockStore();
store.getTask.mockResolvedValue(makeTaskDetail("FN-056", "in-review"));
store.moveTask.mockResolvedValue(makeTask("FN-056", "done"));
// git rev-parse --verify throws (branch not found)
mockedExecSync.mockImplementation((cmd: any) => {
if (typeof cmd === "string" && cmd.includes("git rev-parse --verify")) {
throw new Error("branch not found");
}
return Buffer.from("");
});
const result = await aiMergeTask(store, "/tmp/root", "FN-056");
expect(result.merged).toBe(false);
expect(result.error).toContain("Branch");
expect(store.moveTask).toHaveBeenCalledWith("FN-056", "done");
});
});
// ── Step 4: Triage re-pick and scheduler tests ────────────────────────────
describe("Triage re-pick after restart", () => {
it("TriageProcessor.start() after restart picks up triage tasks (processing set is fresh)", async () => {
const store = createMockStore();
const triageTask1 = makeTask("FN-060", "triage");
const triageTask2 = makeTask("FN-061", "triage");
store.listTasks.mockResolvedValue([triageTask1, triageTask2]);
store.getTask.mockImplementation(async (id: string) =>
makeTaskDetail(id, "triage"),
);
mockAgentSuccess();
const triage = new TriageProcessor(store, "/tmp/root", {
pollIntervalMs: 100000, // large interval to avoid re-poll
});
triage.start();
// Wait for the immediate poll() to fire
await new Promise((r) => setTimeout(r, 100));
triage.stop();
// Both triage tasks should have been picked up for specification
expect(store.updateTask).toHaveBeenCalledWith("FN-060", { status: "specifying" });
expect(store.updateTask).toHaveBeenCalledWith("FN-061", { status: "specifying" });
expect(mockedCreateHaiAgent).toHaveBeenCalledTimes(2);
});
it("specifyTask() skips task already in processing set (no double-specification)", async () => {
const store = createMockStore();
const task = makeTask("FN-062", "triage");
store.getTask.mockResolvedValue(makeTaskDetail("FN-062", "triage"));
// Slow agent to keep task in processing
let resolvePrompt: Function;
mockedCreateHaiAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockImplementation(() => new Promise((r) => { resolvePrompt = r; })),
dispose: vi.fn(),
},
} as any);
const triage = new TriageProcessor(store, "/tmp/root");
// Start first specification (will block on prompt)
const first = triage.specifyTask(task);
// Give it time to enter processing set
await new Promise((r) => setTimeout(r, 20));
// Second call should be a no-op (already processing)
await triage.specifyTask(task);
// Only one agent created
expect(mockedCreateHaiAgent).toHaveBeenCalledTimes(1);
// Resolve the blocked prompt to clean up
resolvePrompt!();
await first;
});
});
describe("Scheduler after restart", () => {
it("schedule() moves todo tasks to in-progress when deps are satisfied", async () => {
const store = createMockStore();
const todoTask = makeTask("FN-070", "todo");
store.listTasks.mockResolvedValue([todoTask]);
store.getSettings.mockResolvedValue({ ...DEFAULT_SETTINGS });
// Needs parseFileScopeFromPrompt for overlap checks
store.parseFileScopeFromPrompt.mockResolvedValue([]);
const onSchedule = vi.fn();
const scheduler = new Scheduler(store, {
maxConcurrent: 2,
maxWorktrees: 4,
onSchedule,
});
// Use start/stop to trigger schedule() then clean up
scheduler.start();
await new Promise((r) => setTimeout(r, 50));
scheduler.stop();
expect(store.moveTask).toHaveBeenCalledWith("FN-070", "in-progress");
expect(store.updateTask).toHaveBeenCalledWith("FN-070", expect.objectContaining({ status: null, blockedBy: null }));
expect(onSchedule).toHaveBeenCalledWith(todoTask);
});
it("schedule() respects dependency ordering — blocked tasks stay in todo", async () => {
const store = createMockStore();
const depTask = makeTask("FN-071", "in-progress");
const blockedTask = makeTask("FN-072", "todo", {
dependencies: ["FN-071"],
});
store.listTasks.mockResolvedValue([depTask, blockedTask]);
store.getSettings.mockResolvedValue({ ...DEFAULT_SETTINGS });
const onBlocked = vi.fn();
const scheduler = new Scheduler(store, {
maxConcurrent: 2,
maxWorktrees: 4,
onBlocked,
});
scheduler.start();
await new Promise((r) => setTimeout(r, 50));
scheduler.stop();
// Task should NOT have been moved
expect(store.moveTask).not.toHaveBeenCalledWith("FN-072", "in-progress");
expect(onBlocked).toHaveBeenCalledWith(blockedTask, ["FN-071"]);
});
it("full column coverage: restart with tasks in every column", async () => {
const store = createMockStore();
// Tasks across all columns
const triageTask = makeTask("FN-080", "triage");
const todoTask = makeTask("FN-081", "todo");
const inProgressTask = makeTask("FN-082", "in-progress");
const inReviewTask = makeTask("FN-083", "in-review");
const doneTask = makeTask("FN-084", "done");
const allTasks = [triageTask, todoTask, inProgressTask, inReviewTask, doneTask];
store.listTasks.mockResolvedValue(allTasks);
store.getTask.mockImplementation(async (id: string) => {
const t = allTasks.find((t) => t.id === id)!;
return makeTaskDetail(id, t.column);
});
store.getSettings.mockResolvedValue({ ...DEFAULT_SETTINGS, autoMerge: true });
mockAgentSuccess();
// 1. Triage picks up triage tasks
const triage = new TriageProcessor(store, "/tmp/root", {
pollIntervalMs: 100000,
});
triage.start();
await new Promise((r) => setTimeout(r, 100));
triage.stop();
expect(store.updateTask).toHaveBeenCalledWith("FN-080", { status: "specifying" });
// 2. Scheduler moves todo → in-progress
vi.clearAllMocks();
store.listTasks.mockResolvedValue(allTasks);
store.getSettings.mockResolvedValue({ ...DEFAULT_SETTINGS });
const scheduler = new Scheduler(store, { maxConcurrent: 2, maxWorktrees: 4 });
scheduler.start();
await new Promise((r) => setTimeout(r, 50));
scheduler.stop();
expect(store.moveTask).toHaveBeenCalledWith("FN-081", "in-progress");
// 3. Executor resumes in-progress tasks
vi.clearAllMocks();
store.listTasks.mockResolvedValue(allTasks);
store.getTask.mockImplementation(async (id: string) => {
const t = allTasks.find((t) => t.id === id)!;
return makeTaskDetail(id, t.column);
});
mockAgentSuccess();
const executor = new TaskExecutor(store, "/tmp/root");
await executor.resumeOrphaned();
await new Promise((r) => setTimeout(r, 50));
expect(store.logEntry).toHaveBeenCalledWith("FN-082", "Resumed after engine restart");
// 4. Done tasks are untouched (no operations on KB-084)
const doneCalls = [
...store.updateTask.mock.calls,
...store.moveTask.mock.calls,
].filter((call) => call[0] === "FN-084");
expect(doneCalls).toHaveLength(0);
});
});
// ── Step 5: Crash scenario edge case tests ────────────────────────────────
describe("Crash scenario edge cases", () => {
it("agent dies mid-step — onError is called, semaphore slot released, task eligible for resume", async () => {
const sem = new AgentSemaphore(2);
const store = createMockStore();
const task = makeTask("FN-090", "in-progress");
store.listTasks.mockResolvedValue([task]);
store.getTask.mockResolvedValue(makeTaskDetail("FN-090", "in-progress"));
// Agent session.prompt rejects (simulating crash mid-step)
mockedCreateHaiAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockRejectedValue(new Error("agent died mid-step")),
dispose: vi.fn(),
},
} as any);
const onError = vi.fn();
const executor = new TaskExecutor(store, "/tmp/test", {
semaphore: sem,
onError,
});
await executor.resumeOrphaned();
await new Promise((r) => setTimeout(r, 50));
// onError should have been called
expect(onError).toHaveBeenCalledWith(task, expect.any(Error));
// Semaphore slot should be released
expect(sem.activeCount).toBe(0);
// Task should be eligible for resume (not in executing set)
// Verify by calling resumeOrphaned again — it should try to execute again
vi.clearAllMocks();
store.listTasks.mockResolvedValue([task]);
store.getTask.mockResolvedValue(makeTaskDetail("FN-090", "in-progress"));
// Use deterministic mock that calls task_done to prevent retry-with-new-session
createAgentWithTaskDone();
await executor.resumeOrphaned();
await new Promise((r) => setTimeout(r, 50));
// Exactly one agent created for the re-resume, proving the task was eligible
// and completed without retry inflation.
expect(mockedCreateHaiAgent).toHaveBeenCalledTimes(1);
// Re-resume should have logged again (behavioral guarantee)
expect(store.logEntry).toHaveBeenCalledWith("FN-090", "Resumed after engine restart");
});
it("engine killed during merge — git reset --merge cleanup, task stays in-review", async () => {
const store = createMockStore();
store.getTask.mockResolvedValue(makeTaskDetail("FN-091", "in-review"));
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
// Post-squash check: squash staged changes → "1"
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
return Buffer.from("");
});
// Agent prompt rejects (simulating kill during merge)
mockedCreateHaiAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockRejectedValue(new Error("killed")),
dispose: vi.fn(),
},
} as any);
await expect(aiMergeTask(store, "/tmp/root", "FN-091")).rejects.toThrow();
// git reset --merge should have been called
const resetCalls = mockedExecSync.mock.calls.filter(
(call) => typeof call[0] === "string" && call[0].includes("git reset --merge"),
);
expect(resetCalls.length).toBeGreaterThan(0);
// Task should NOT have been moved to done
expect(store.moveTask).not.toHaveBeenCalledWith("FN-091", "done");
// Status was set to "merging" during execution
expect(store.updateTask).toHaveBeenCalledWith("FN-091", { status: "merging" });
});
it("concurrent resumeOrphaned() calls don't double-execute the same task", async () => {
const store = createMockStore();
const task = makeTask("FN-092", "in-progress");
store.listTasks.mockResolvedValue([task]);
store.getTask.mockResolvedValue(makeTaskDetail("FN-092", "in-progress"));
let resolvePrompt: Function;
mockedCreateHaiAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockImplementation(() => new Promise((r) => { resolvePrompt = r; })),
dispose: vi.fn(),
},
} as any);
const executor = new TaskExecutor(store, "/tmp/test");
// First call starts execution
const first = executor.resumeOrphaned();
await new Promise((r) => setTimeout(r, 20));
// Second call while first is still executing
const second = executor.resumeOrphaned();
await new Promise((r) => setTimeout(r, 20));
// Only one agent should have been created (the executing set guards against double-exec)
expect(mockedCreateHaiAgent).toHaveBeenCalledTimes(1);
// Clean up
resolvePrompt!();
await first;
await second;
});
it("semaphore integrity after crash — activeCount returns to pre-execution value", async () => {
const sem = new AgentSemaphore(3);
const store = createMockStore();
// Pre-acquire one slot to simulate other work
await sem.acquire();
expect(sem.activeCount).toBe(1);
const task = makeTask("FN-093", "in-progress");
store.listTasks.mockResolvedValue([task]);
store.getTask.mockResolvedValue(makeTaskDetail("FN-093", "in-progress"));
// Agent creation itself fails
mockedCreateHaiAgent.mockRejectedValue(new Error("cannot create agent"));
const onError = vi.fn();
const executor = new TaskExecutor(store, "/tmp/test", {
semaphore: sem,
onError,
});
await executor.resumeOrphaned();
await new Promise((r) => setTimeout(r, 50));
// Semaphore should return to pre-execution count (1 from our manual acquire)
expect(sem.activeCount).toBe(1);
expect(onError).toHaveBeenCalled();
// Release our manual slot
sem.release();
expect(sem.activeCount).toBe(0);
});
});
// ── Worktree pool restart resilience tests ────────────────────────────────
function makeDirEntry(name: string) {
return { name, isDirectory: () => true } as any;
}
function mockRegisteredWorktrees(rootDir: string, names: string[]) {
mockedExecSync.mockImplementation((cmd: any) => {
if (String(cmd) === "git worktree list --porcelain") {
return [
`worktree ${rootDir}`,
"HEAD abc123",
"branch refs/heads/main",
"",
...names.flatMap((name) => [
`worktree ${rootDir}/.worktrees/${name}`,
"HEAD def456",
`branch refs/heads/fusion/${name}`,
"",
]),
].join("\n") as any;
}
return Buffer.from("");
});
}
describe("Worktree pool restart with recycleWorktrees=true", () => {
it("pool is rehydrated with idle worktrees from disk", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("swift-falcon"),
makeDirEntry("calm-river"),
makeDirEntry("bold-eagle"),
] as any);
mockedExistsSync.mockReturnValue(true);
mockRegisteredWorktrees("/root", ["swift-falcon", "calm-river", "bold-eagle"]);
const store = createMockStore();
store.listTasks.mockResolvedValue([
makeTask("FN-100", "in-progress", { worktree: "/root/.worktrees/swift-falcon" }),
makeTask("FN-101", "done", { worktree: "/root/.worktrees/calm-river" }),
]);
// Simulate startup rehydration
const pool = new WorktreePool();
const idlePaths = await scanIdleWorktrees("/root", store);
pool.rehydrate(idlePaths);
// swift-falcon → in-progress, not idle
// calm-river → done, idle
// bold-eagle → unassigned, idle
expect(pool.size).toBe(2);
expect(pool.has("/root/.worktrees/calm-river")).toBe(true);
expect(pool.has("/root/.worktrees/bold-eagle")).toBe(true);
expect(pool.has("/root/.worktrees/swift-falcon")).toBe(false);
});
it("executor acquires from rehydrated pool instead of creating new worktrees", async () => {
// Setup: rehydrate pool with one idle worktree
mockedReaddirSync.mockReturnValue([
makeDirEntry("idle-wt"),
] as any);
mockRegisteredWorktrees("/root", ["idle-wt"]);
const store = createMockStore();
store.listTasks.mockResolvedValue([]);
store.getSettings.mockResolvedValue({
...DEFAULT_SETTINGS,
recycleWorktrees: true,
});
store.getTask.mockResolvedValue(makeTaskDetail("FN-110", "in-progress"));
const pool = new WorktreePool();
const idlePaths = await scanIdleWorktrees("/root", store);
pool.rehydrate(idlePaths);
expect(pool.size).toBe(1);
// Now simulate executor acquiring from pool
// The pool path exists on disk, but the task's default path does not
mockedExistsSync.mockImplementation(
(p) => p === "/root/.worktrees/idle-wt",
);
mockAgentSuccess();
const executor = new TaskExecutor(store, "/root", { pool });
await executor.execute(makeTask("FN-110", "in-progress"));
await new Promise((r) => setTimeout(r, 50));
// Pool should be empty (worktree acquired)
expect(pool.size).toBe(0);
// No git worktree add calls (reused from pool)
const worktreeAddCalls = mockedExecSync.mock.calls.filter(
(c) => typeof c[0] === "string" && (c[0] as string).includes("worktree add"),
);
expect(worktreeAddCalls).toHaveLength(0);
// Should log pool acquisition
expect(store.logEntry).toHaveBeenCalledWith(
"FN-110",
expect.stringContaining("Acquired worktree from pool"),
undefined,
expect.objectContaining({ agentId: "executor" }),
);
});
it("worktrees assigned to in-progress tasks are preserved (not in pool)", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("active-wt"),
makeDirEntry("idle-wt"),
] as any);
mockedExistsSync.mockReturnValue(true);
mockRegisteredWorktrees("/root", ["active-wt", "idle-wt"]);
const store = createMockStore();
store.listTasks.mockResolvedValue([
makeTask("KB-120", "in-progress", { worktree: "/root/.worktrees/active-wt" }),
]);
const pool = new WorktreePool();
const idlePaths = await scanIdleWorktrees("/root", store);
pool.rehydrate(idlePaths);
// Only idle-wt should be in pool (active-wt is assigned to in-progress task)
expect(pool.size).toBe(1);
expect(pool.has("/root/.worktrees/idle-wt")).toBe(true);
expect(pool.has("/root/.worktrees/active-wt")).toBe(false);
});
it("worktrees assigned to in-review tasks are preserved (not in pool)", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("review-wt"),
] as any);
mockedExistsSync.mockReturnValue(true);
mockRegisteredWorktrees("/root", ["review-wt"]);
const store = createMockStore();
store.listTasks.mockResolvedValue([
makeTask("KB-121", "in-review", { worktree: "/root/.worktrees/review-wt" }),
]);
const pool = new WorktreePool();
const idlePaths = await scanIdleWorktrees("/root", store);
pool.rehydrate(idlePaths);
// review-wt is assigned to in-review task — NOT idle
expect(pool.size).toBe(0);
});
});
describe("Worktree cleanup on restart with recycleWorktrees=false", () => {
it("orphaned worktrees are cleaned up via git worktree remove", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("orphan-1"),
makeDirEntry("orphan-2"),
] as any);
mockedExistsSync.mockReturnValue(true);
mockRegisteredWorktrees("/root", ["orphan-1", "orphan-2"]);
const store = createMockStore();
store.listTasks.mockResolvedValue([]);
const cleaned = await cleanupOrphanedWorktrees("/root", store);
expect(cleaned).toBe(2);
const removeCalls = mockedExecSync.mock.calls.filter(
(c) => typeof c[0] === "string" && (c[0] as string).includes("worktree remove"),
);
expect(removeCalls).toHaveLength(2);
});
it("worktrees assigned to in-progress tasks are preserved during cleanup", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("active-wt"),
makeDirEntry("orphan-wt"),
] as any);
mockedExistsSync.mockReturnValue(true);
mockRegisteredWorktrees("/root", ["active-wt", "orphan-wt"]);
const store = createMockStore();
store.listTasks.mockResolvedValue([
makeTask("KB-130", "in-progress", { worktree: "/root/.worktrees/active-wt" }),
]);
const cleaned = await cleanupOrphanedWorktrees("/root", store);
expect(cleaned).toBe(1);
const removeCalls = mockedExecSync.mock.calls.filter(
(c) => typeof c[0] === "string" && (c[0] as string).includes("worktree remove"),
);
expect(removeCalls).toHaveLength(1);
expect(removeCalls[0][0]).toContain("orphan-wt");
expect(removeCalls[0][0]).not.toContain("active-wt");
});
it("worktrees assigned to in-review tasks are preserved during cleanup", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("review-wt"),
] as any);
mockedExistsSync.mockReturnValue(true);
mockRegisteredWorktrees("/root", ["review-wt"]);
const store = createMockStore();
store.listTasks.mockResolvedValue([
makeTask("KB-131", "in-review", { worktree: "/root/.worktrees/review-wt" }),
]);
const cleaned = await cleanupOrphanedWorktrees("/root", store);
// review-wt is assigned to in-review task — should NOT be removed
expect(cleaned).toBe(0);
});
});
describe("Edge case: worktree deleted between scan and acquire", () => {
it("acquire returns null when rehydrated worktree was deleted from disk", async () => {
const pool = new WorktreePool();
// Rehydrate succeeds (path exists at scan time)
mockedExistsSync.mockReturnValue(true);
pool.rehydrate(["/root/.worktrees/vanished-wt"]);
expect(pool.size).toBe(1);
// Between rehydrate and acquire, the directory is deleted
mockedExistsSync.mockReturnValue(false);
// acquire() checks existsSync and prunes the stale entry
const result = pool.acquire();
expect(result).toBeNull();
expect(pool.size).toBe(0);
});
});
// ── Engine pause/unpause cycle integration tests ──────────────────────────
describe("Engine pause/unpause cycle", () => {
it("executor: agents continue running on enginePaused (soft pause), complete normally", async () => {
const store = createMockStore();
const task = makeTask("FN-EP1", "in-progress");
store.getTask.mockResolvedValue(makeTaskDetail("FN-EP1", "in-progress"));
// Agent triggers engine pause mid-flight but continues normally (soft pause)
mockedCreateHaiAgent.mockImplementation(async () => ({
session: {
prompt: vi.fn().mockImplementation(async () => {
// Trigger engine pause — session should NOT be terminated
store._trigger("settings:updated", {
settings: { enginePaused: true },
previous: { enginePaused: false },
});
// Session continues normally
}),
dispose: vi.fn(),
},
} as any));
const executor = new TaskExecutor(store, "/tmp/test");
await executor.execute(task);
// Task should complete normally (in-review), NOT moved to todo
expect(store.moveTask).toHaveBeenCalledWith("FN-EP1", "in-review");
expect(store.moveTask).not.toHaveBeenCalledWith("FN-EP1", "todo");
expect(store.updateTask).not.toHaveBeenCalledWith("FN-EP1", { status: "failed" });
});
it("triage: agents NOT terminated on enginePaused (soft pause), session continues", async () => {
const store = createMockStore();
const disposeFn = vi.fn();
let sessionContinued = false;
// Agent triggers engine pause mid-flight; session should NOT be disposed
mockedCreateHaiAgent.mockImplementation(async () => ({
session: {
prompt: vi.fn().mockImplementation(async () => {
store._trigger("settings:updated", {
settings: { enginePaused: true },
previous: { enginePaused: false },
});
// If the session was disposed by the listener, we'd get an error.
// Instead, the session continues normally (soft pause).
sessionContinued = true;
// Throw to exit without needing file system (PROMPT.md read).
// The key assertion is that dispose was NOT called by the listener.
throw new Error("simulated completion");
}),
dispose: disposeFn,
},
} as any));
const triage = new TriageProcessor(store, "/tmp/test");
await triage.specifyTask(makeTask("FN-EP2", "triage"));
// Session should have continued past the enginePaused event
expect(sessionContinued).toBe(true);
// dispose should only be called once in the finally block, not by the engine pause listener
expect(disposeFn).toHaveBeenCalledTimes(1);
});
it("scheduler resumes on unpause: schedule() runs when enginePaused goes true→false", async () => {
const store = createMockStore();
const todoTask = makeTask("FN-EP3", "todo");
store.listTasks.mockResolvedValue([todoTask]);
store.getSettings.mockResolvedValue({ ...DEFAULT_SETTINGS, enginePaused: false });
store.parseFileScopeFromPrompt.mockResolvedValue([]);
const onSchedule = vi.fn();
const scheduler = new Scheduler(store, {
maxConcurrent: 2,
maxWorktrees: 4,
onSchedule,
});
scheduler.start();
await new Promise((r) => setTimeout(r, 50));
// Scheduler should have moved todo task to in-progress
expect(store.moveTask).toHaveBeenCalledWith("FN-EP3", "in-progress");
// Now simulate engine pause then unpause
store.moveTask.mockClear();
onSchedule.mockClear();
// During pause, scheduler halts
store.getSettings.mockResolvedValue({ ...DEFAULT_SETTINGS, enginePaused: true });
// Add a new todo task
const newTask = makeTask("FN-EP4", "todo");
store.listTasks.mockResolvedValue([newTask]);
// Unpause — trigger settings:updated to wake the scheduler
store.getSettings.mockResolvedValue({ ...DEFAULT_SETTINGS, enginePaused: false });
store._trigger("settings:updated", {
settings: { ...DEFAULT_SETTINGS, enginePaused: false },
previous: { ...DEFAULT_SETTINGS, enginePaused: true },
});
await new Promise((r) => setTimeout(r, 100));
scheduler.stop();
// The new task should have been scheduled after unpause
expect(store.moveTask).toHaveBeenCalledWith("FN-EP4", "in-progress");
});
it("concurrency slots freed after agent completes during enginePaused (soft pause)", async () => {
const sem = new AgentSemaphore(1); // Only 1 concurrent slot
const store = createMockStore();
store.getTask.mockResolvedValue(makeTaskDetail("FN-EP5", "in-progress"));
mockedCreateHaiAgent.mockImplementation(async () => ({
session: {
prompt: vi.fn().mockImplementation(async () => {
// Trigger engine pause while the agent holds the semaphore slot
store._trigger("settings:updated", {
settings: { enginePaused: true },
previous: { enginePaused: false },
});
// Agent continues and finishes normally (soft pause)
}),
dispose: vi.fn(),
},
} as any));
const executor = new TaskExecutor(store, "/tmp/test", { semaphore: sem });
// Execute — agent runs to completion despite engine pause
await executor.execute(makeTask("FN-EP5", "in-progress"));
// After completion, the semaphore slot should be freed.
// Verify by running a new task through the semaphore — it should not block.
let secondSlotAcquired = false;
const runPromise = sem.run(async () => {
secondSlotAcquired = true;
}, 10);
// If the slot was properly released, this resolves immediately
await runPromise;
expect(secondSlotAcquired).toBe(true);
});
});
// ── Regression: getTaskMergeBlocker import contract ──────────────────────
describe("getTaskMergeBlocker import regression", () => {
it("getTaskMergeBlocker is importable from @fusion/core at runtime", async () => {
// Dynamic import to verify the runtime export contract — if this fails,
// packages/core/dist/index.js is stale or the export was removed.
const core = await import("@fusion/core");
expect(typeof core.getTaskMergeBlocker).toBe("function");
expect(typeof core.isTaskReadyForMerge).toBe("function");
});
it("getTaskMergeBlocker rejects non-in-review tasks", async () => {
const { getTaskMergeBlocker } = await import("@fusion/core");
const blocker = getTaskMergeBlocker({
column: "in-progress",
paused: false,
status: undefined,
error: undefined,
steps: [],
workflowStepResults: [],
});
expect(blocker).toContain("must be in 'in-review'");
});
it("aiMergeTask can be loaded from merger module (getTaskMergeBlocker reachable)", async () => {
// Verify the merger module itself loads without import errors.
// aiMergeTask internally uses getTaskMergeBlocker from @fusion/core —
// if the core export is broken, this dynamic import will fail.
const merger = await import("./merger.js");
expect(typeof merger.aiMergeTask).toBe("function");
expect(typeof merger.findWorktreeUser).toBe("function");
});
});