fix(engine): prevent worktree collisions on manual task moves

Two related bugs let two in-progress tasks share a single
.worktrees/<name> directory:

1. The dashboard POST /tasks/:id/move route promoted tasks to
   in-progress without allocating a fresh worktree path, so a queued
   task carrying a stale worktree field from a prior preserveResumeState
   requeue could land in-progress on a directory already held by another
   active task.

2. moveTask({preserveResumeState:true}) kept the worktree pointer on
   requeue. When the on-disk checkout was later removed or reassigned,
   the next dispatch collided with a worktree the scheduler had handed
   to another task.

moveTask now releases the worktree pointer on every reopen-to-todo hop
(branch is kept so committed progress survives via git worktree add
<path> <branch>). A new preserveWorktree option opts internal bounces
out of the release. moveTask also accepts an allocateWorktree callback
that runs under a new cross-task allocation lock in TaskStore, so two
concurrent moves cannot pick the same name from a stale snapshot. Both
the manual-move route and the scheduler dispatch path flow through the
allocator and share the lock.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-05-06 07:23:38 -07:00
parent a143bcc4f5
commit 1100b39cff
11 changed files with 308 additions and 73 deletions

View File

@@ -8649,7 +8649,7 @@ describe("Workflow Steps Execution", () => {
// todo must flag preserveResumeState so the workflow-rerun bounce keeps
// the worktree and accumulated step progress through the transient
// todo state on its way back to in-progress.
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveResumeState: true });
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveResumeState: true, preserveWorktree: true });
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress");
// onComplete should NOT be called (task is being retried, not completed)
@@ -8784,7 +8784,7 @@ describe("Workflow Steps Execution", () => {
// todo must flag preserveResumeState so the workflow-rerun bounce keeps
// the worktree and accumulated step progress through the transient
// todo state on its way back to in-progress.
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveResumeState: true });
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveResumeState: true, preserveWorktree: true });
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress");
// onComplete should NOT be called (task is being retried, not completed)
@@ -8925,7 +8925,7 @@ describe("Workflow Steps Execution", () => {
await new Promise<void>((resolve) => queueMicrotask(resolve));
// (2) bounce uses preserveResumeState so step progress + worktree survive
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveResumeState: true });
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "todo", { preserveResumeState: true, preserveWorktree: true });
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress");
expect(store.moveTask).not.toHaveBeenCalledWith("FN-001", "in-review");
expect(onError).not.toHaveBeenCalled();
@@ -11880,7 +11880,7 @@ describe("TaskExecutor watchdogs", () => {
executionStartedAt: originalExecutionStartedAt,
});
expect(store.moveTask.mock.calls).toEqual([
["FN-WD-4", "todo", { preserveResumeState: true }],
["FN-WD-4", "todo", { preserveResumeState: true, preserveWorktree: true }],
["FN-WD-4", "in-progress"],
]);
});
@@ -12716,7 +12716,7 @@ describe("StepSessionExecutor integration", () => {
// Task should move to todo then in-progress (not in-review). The
// workflow-rerun bounce flags preserveResumeState so the worktree and
// accumulated step progress survive the transient todo state.
expect(store.moveTask).toHaveBeenCalledWith("FN-200", "todo", { preserveResumeState: true });
expect(store.moveTask).toHaveBeenCalledWith("FN-200", "todo", { preserveResumeState: true, preserveWorktree: true });
expect(store.moveTask).toHaveBeenCalledWith("FN-200", "in-progress");
vi.useRealTimers();

View File

@@ -725,8 +725,14 @@ describe("In-progress task resume after restart", () => {
// Run any pending microtasks (the async code in setTimeout)
await vi.runAllTimersAsync();
// Task should move to todo then in-progress (not in-review)
expect(store.moveTask).toHaveBeenCalledWith("FN-963", "todo");
// Task should move to todo then in-progress (not in-review). The
// workflow-rerun bounce passes `preserveWorktree: true` so the
// checkout doesn't briefly disappear during the hop.
expect(store.moveTask).toHaveBeenCalledWith(
"FN-963",
"todo",
expect.objectContaining({ preserveWorktree: true }),
);
expect(store.moveTask).toHaveBeenCalledWith("FN-963", "in-progress");
vi.useRealTimers();
@@ -963,7 +969,7 @@ describe("Scheduler after restart", () => {
await new Promise((r) => setTimeout(r, 50));
scheduler.stop();
expect(store.moveTask).toHaveBeenCalledWith("FN-070", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-070", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
expect(store.updateTask).toHaveBeenCalledWith("FN-070", expect.objectContaining({ status: null, blockedBy: null }));
expect(onSchedule).toHaveBeenCalledWith(todoTask);
});
@@ -1033,7 +1039,7 @@ describe("Scheduler after restart", () => {
await new Promise((r) => setTimeout(r, 50));
scheduler.stop();
expect(store.moveTask).toHaveBeenCalledWith("FN-081", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-081", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
// 3. Executor resumes in-progress tasks
vi.clearAllMocks();
@@ -1549,7 +1555,7 @@ describe("Engine pause/unpause cycle", () => {
await new Promise((r) => setTimeout(r, 50));
// Scheduler should have moved todo task to in-progress
expect(store.moveTask).toHaveBeenCalledWith("FN-EP3", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-EP3", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
// Now simulate engine pause then unpause
store.moveTask.mockClear();
@@ -1573,7 +1579,7 @@ describe("Engine pause/unpause cycle", () => {
scheduler.stop();
// The new task should have been scheduled after unpause
expect(store.moveTask).toHaveBeenCalledWith("FN-EP4", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-EP4", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
it("concurrency slots freed after agent completes during enginePaused (soft pause)", async () => {

View File

@@ -244,7 +244,7 @@ describe("Scheduler", () => {
await flushAsyncWork();
// Verify schedule() was called (moveTask should be called since task can start)
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
it("resets mergeRetries when dispatching a task to in-progress", async () => {
@@ -282,7 +282,7 @@ describe("Scheduler", () => {
"FN-001",
expect.objectContaining({ mergeRetries: 0 }),
);
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
it("registers task:moved event listener", () => {
@@ -335,7 +335,7 @@ describe("Scheduler", () => {
await flushAsyncWork();
// Verify schedule() was called - FN-002 should now be able to start
expect(store.moveTask).toHaveBeenCalledWith("FN-002", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-002", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
it("does not trigger scheduling for non-done task:moved events", async () => {
@@ -406,7 +406,7 @@ describe("Scheduler", () => {
await flushAsyncWork();
// Verify schedule() was called — task in todo should be scheduled
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
});
@@ -450,7 +450,7 @@ describe("Scheduler", () => {
await flushAsyncWork();
// Should have triggered scheduling and moved the task to in-progress
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-001", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
it("does not trigger scheduling on unpause if scheduler is not running", async () => {
@@ -710,7 +710,7 @@ describe("Scheduler", () => {
expect(moveTask).not.toHaveBeenCalledWith("FN-102", "in-progress");
// Lower-priority ready task still runs.
expect(moveTask).toHaveBeenCalledWith("FN-104", "in-progress");
expect(moveTask).toHaveBeenCalledWith("FN-104", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
// Overlap-blocked urgent task must not run.
expect(moveTask).not.toHaveBeenCalledWith("FN-103", "in-progress");
});
@@ -751,7 +751,7 @@ describe("Scheduler", () => {
(scheduler as any).running = true;
await scheduler.schedule();
expect(moveTask).toHaveBeenCalledWith("FN-002", "in-progress");
expect(moveTask).toHaveBeenCalledWith("FN-002", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
expect(updateTask).not.toHaveBeenCalledWith("FN-002", { status: "queued", blockedBy: "FN-001" });
});
@@ -789,7 +789,7 @@ describe("Scheduler", () => {
(scheduler as any).running = true;
await scheduler.schedule();
expect(moveTask).toHaveBeenCalledWith("FN-002", "in-progress");
expect(moveTask).toHaveBeenCalledWith("FN-002", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
expect(updateTask).not.toHaveBeenCalledWith("FN-002", { status: "queued", blockedBy: "FN-001" });
});
@@ -855,12 +855,11 @@ describe("Scheduler", () => {
status: null,
blockedBy: null,
executionStartBranch: undefined,
worktree: "/test/project/.worktrees/fn-010",
effectiveNodeId: null,
effectiveNodeSource: "local",
mergeRetries: 0,
});
expect(moveTask).toHaveBeenCalledWith("FN-010", "in-progress");
expect(moveTask).toHaveBeenCalledWith("FN-010", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
expect(updateTask.mock.invocationCallOrder[0]).toBeLessThan(moveTask.mock.invocationCallOrder[0]);
});
@@ -894,7 +893,6 @@ describe("Scheduler", () => {
status: null,
blockedBy: null,
executionStartBranch: undefined,
worktree: "/test/project/.worktrees/amber-aspen",
effectiveNodeId: null,
effectiveNodeSource: "local",
mergeRetries: 0,
@@ -903,7 +901,6 @@ describe("Scheduler", () => {
status: null,
blockedBy: null,
executionStartBranch: undefined,
worktree: "/test/project/.worktrees/amber-aspen-2",
effectiveNodeId: null,
effectiveNodeSource: "local",
mergeRetries: 0,
@@ -1035,7 +1032,7 @@ describe("Scheduler", () => {
// Flush any remaining microtasks
await new Promise(resolve => setTimeout(resolve, 0));
expect(moveTask).toHaveBeenCalledWith("FN-010", "in-progress");
expect(moveTask).toHaveBeenCalledWith("FN-010", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
expect(moveTask).not.toHaveBeenCalledWith("FN-010", "triage");
});
@@ -1217,7 +1214,7 @@ describe("Scheduler", () => {
expect.any(String)
);
// Should move to in-progress (since deps are satisfied and concurrency allows)
expect(moveTask).toHaveBeenCalledWith("FN-004", "in-progress");
expect(moveTask).toHaveBeenCalledWith("FN-004", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
it("does not validate filesystem for tasks with unmet dependencies", async () => {
@@ -1956,7 +1953,7 @@ describe("Scheduler", () => {
(scheduler as any).running = true;
await scheduler.schedule();
expect(store.moveTask).toHaveBeenCalledWith("FN-100", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-100", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
it("schedules tasks without sliceId regardless of mission state", async () => {
@@ -1981,7 +1978,7 @@ describe("Scheduler", () => {
(scheduler as any).running = true;
await scheduler.schedule();
expect(store.moveTask).toHaveBeenCalledWith("FN-100", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-100", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
});
@@ -2037,7 +2034,7 @@ describe("Scheduler", () => {
(scheduler as any).running = true;
await scheduler.schedule();
expect(store.moveTask).toHaveBeenCalledWith("FN-011", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-011", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
it("picks up todo tasks without nextRecoveryAt normally", async () => {
@@ -2063,7 +2060,7 @@ describe("Scheduler", () => {
(scheduler as any).running = true;
await scheduler.schedule();
expect(store.moveTask).toHaveBeenCalledWith("FN-012", "in-progress");
expect(store.moveTask).toHaveBeenCalledWith("FN-012", "in-progress", expect.objectContaining({ allocateWorktree: expect.any(Function) }));
});
});

View File

@@ -1427,11 +1427,25 @@ export class TaskExecutor {
// moveTask's default reopen-to-todo path resets every step to
// pending and rewrites PROMPT.md checkboxes, which would discard
// the partial progress this bounce is supposed to retry on top of.
// `preserveWorktree` keeps the same checkout assigned across the
// hop so listeners never observe an interim `worktree=null` state
// — this bounce immediately re-promotes the task on the same
// directory, so releasing it would publish a misleading snapshot
// and could let self-healing reclaim the worktree as idle.
if (preserveResumeState) {
await this.store.moveTask(taskId, "todo", { preserveResumeState: true });
await this.store.moveTask(taskId, "todo", {
preserveResumeState: true,
preserveWorktree: true,
});
} else {
await this.store.moveTask(taskId, "todo");
await this.store.moveTask(taskId, "todo", { preserveWorktree: true });
}
// Restore worktree + executionStartedAt unconditionally to match
// the original bounce contract: even with preserveWorktree the
// worktree pointer could have been cleared by an in-flight
// updateTask, and executionStartedAt is reset by moveTask when
// preserveResumeState is false. Keep the writes so callers and
// tests can observe the restoration deterministically.
await this.store.updateTask(taskId, {
worktree: worktreePath,
executionStartedAt: originalExecutionStartedAt ?? null,

View File

@@ -66,6 +66,7 @@ export {
} from "./agent-instructions.js";
export { HEARTBEAT_PROCEDURE, HEARTBEAT_SYSTEM_PROMPT, HEARTBEAT_NO_TASK_SYSTEM_PROMPT } from "./agent-heartbeat.js";
export { WorktreePool, scanIdleWorktrees, cleanupOrphanedWorktrees, reapOrphanWorktrees } from "./worktree-pool.js";
export { generateReservedWorktreeName, generateWorktreeName, planTaskWorktreePath, slugify } from "./worktree-names.js";
export { createLogger, type Logger } from "./logger.js";
export { isUsageLimitError, UsageLimitPauser } from "./usage-limit-detector.js";
export { withRateLimitRetry } from "./rate-limit-retry.js";

View File

@@ -10,9 +10,9 @@ import {
} from "@fusion/core";
import { existsSync } from "node:fs";
import { readFile } from "node:fs/promises";
import { basename, join } from "node:path";
import { join } from "node:path";
import type { AgentSemaphore } from "./concurrency.js";
import { generateReservedWorktreeName, slugify } from "./worktree-names.js";
import { planTaskWorktreePath } from "./worktree-names.js";
import { schedulerLog } from "./logger.js";
import { type PrMonitor, type PrComment } from "./pr-monitor.js";
import { reconcileMissionFeatureState } from "./mission-feature-sync.js";
@@ -495,28 +495,7 @@ export class Scheduler {
naming: string | undefined,
reservedNames: Set<string>,
): string {
if (task.worktree) {
const existingName = basename(task.worktree);
if (existingName) reservedNames.add(existingName);
return task.worktree;
}
let worktreeName: string;
switch (naming || "random") {
case "task-id":
worktreeName = task.id.toLowerCase();
break;
case "task-title":
worktreeName = slugify(task.title || task.description.slice(0, 60));
break;
case "random":
default:
worktreeName = generateReservedWorktreeName(this.store.getRootDir(), reservedNames);
break;
}
reservedNames.add(worktreeName);
return join(this.store.getRootDir(), ".worktrees", worktreeName);
return planTaskWorktreePath(task, this.store.getRootDir(), naming, reservedNames);
}
/**
@@ -687,11 +666,6 @@ export class Scheduler {
// Resolve dependency order among todo tasks
const ordered = resolveDependencyOrder(todo);
let started = 0;
const reservedWorktreeNames = new Set(
tasks
.map((task) => (task.worktree ? basename(task.worktree) : undefined))
.filter((name): name is string => Boolean(name)),
);
for (const taskId of ordered) {
const task = tasks.find((t) => t.id === taskId)!;
@@ -760,13 +734,11 @@ export class Scheduler {
continue;
}
// Dependencies met — resolve base branch from in-review deps
// Dependencies met — resolve base branch from in-review deps.
// Worktree allocation is deferred to moveTask below, where it
// runs under TaskStore's cross-task allocation lock so it can't
// race against a concurrent manual-move.
const baseBranch = this.resolveBaseBranch(task, tasks);
const plannedWorktree = this.planWorktreePath(
task,
settings.worktreeNaming,
reservedWorktreeNames,
);
// Compare-and-swap: re-read the task to verify it's still in "todo" before dispatching.
// This prevents dispatching a task twice if another schedule() call or user action
@@ -836,12 +808,14 @@ export class Scheduler {
status: null,
blockedBy: null,
executionStartBranch: baseBranch ?? undefined,
worktree: plannedWorktree,
effectiveNodeId: effectiveNode.nodeId ?? null,
effectiveNodeSource: effectiveNode.source,
mergeRetries: 0,
});
await this.store.moveTask(task.id, "in-progress");
await this.store.moveTask(task.id, "in-progress", {
allocateWorktree: (reservedNames) =>
this.planWorktreePath(task, settings.worktreeNaming, reservedNames),
});
this.wasNodeBlocked.delete(task.id);
await this.store.logEntry(task.id, `Node routing resolved: ${effectiveNode.nodeId ?? "local"} (source: ${effectiveNode.source})`);
this.options.onSchedule?.(task);

View File

@@ -94,6 +94,48 @@ export function generateReservedWorktreeName(
return `${baseName}-${suffix}`;
}
/**
* Plan a worktree directory path for a task that is about to enter
* `in-progress`. Returns the absolute path under `<rootDir>/.worktrees/`.
*
* If the task already carries a `worktree` value, it is reused — the
* caller is responsible for ensuring it does not collide with another
* active task. Otherwise a name is generated according to `naming`,
* avoiding any names already in `reservedNames`.
*
* Shared by the scheduler dispatch path and the manual-move HTTP route
* so both allocate via the same collision rules.
*/
export function planTaskWorktreePath(
task: { id: string; title?: string | null; description: string; worktree?: string | null },
rootDir: string,
naming: string | undefined,
reservedNames: Set<string>,
): string {
if (task.worktree) {
const existingName = task.worktree.split("/").filter(Boolean).pop();
if (existingName) reservedNames.add(existingName);
return task.worktree;
}
let worktreeName: string;
switch (naming || "random") {
case "task-id":
worktreeName = task.id.toLowerCase();
break;
case "task-title":
worktreeName = slugify(task.title || task.description.slice(0, 60));
break;
case "random":
default:
worktreeName = generateReservedWorktreeName(rootDir, reservedNames);
break;
}
reservedNames.add(worktreeName);
return join(rootDir, ".worktrees", worktreeName);
}
function getExistingWorktreeNames(worktreesDir: string): Set<string> {
if (!existsSync(worktreesDir)) {
return new Set();