Fix dashboard startup blocking on recovery
This commit is contained in:
@@ -317,6 +317,8 @@ export interface TaskExecutorOptions {
|
||||
export class TaskExecutor {
|
||||
private activeWorktrees = new Map<string, string>();
|
||||
private executing = new Set<string>();
|
||||
/** Completed orphan recovery tasks currently running during startup. */
|
||||
private recoveringCompleted = new Set<string>();
|
||||
/** Active agent sessions per task, used to terminate on pause and inject steering. */
|
||||
private activeSessions = new Map<string, {
|
||||
session: AgentSession;
|
||||
@@ -381,7 +383,7 @@ export class TaskExecutor {
|
||||
|
||||
/** Returns the set of task IDs currently being executed. */
|
||||
getExecutingTaskIds(): Set<string> {
|
||||
return new Set(this.executing);
|
||||
return new Set([...this.executing, ...this.recoveringCompleted]);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -748,8 +750,19 @@ export class TaskExecutor {
|
||||
// Fast-path: if the task already completed its work (all steps done),
|
||||
// move it directly to in-review instead of re-executing from scratch.
|
||||
if (this.isTaskWorkComplete(task)) {
|
||||
if (this.recoveringCompleted.has(task.id)) {
|
||||
executorLog.log(`${task.id} completed-task recovery already running - skipping duplicate startup recovery`);
|
||||
continue;
|
||||
}
|
||||
executorLog.log(`${task.id} is already complete — fast-pathing to in-review`);
|
||||
await this.recoverCompletedTask(task);
|
||||
this.recoveringCompleted.add(task.id);
|
||||
void this.recoverCompletedTask(task)
|
||||
.catch((err) =>
|
||||
executorLog.error(`Failed to recover completed orphan ${task.id}:`, err),
|
||||
)
|
||||
.finally(() => {
|
||||
this.recoveringCompleted.delete(task.id);
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import { CronRunner, createAiPromptExecutor } from "./cron-runner.js";
|
||||
import { aiMergeTask } from "./merger.js";
|
||||
import { PRIORITY_MERGE } from "./concurrency.js";
|
||||
import { runtimeLog } from "./logger.js";
|
||||
import type { HeartbeatMonitor, HeartbeatTriggerScheduler } from "./agent-heartbeat.js";
|
||||
|
||||
/**
|
||||
* Callback for processing pull-request merge strategy.
|
||||
@@ -258,6 +259,21 @@ export class ProjectEngine {
|
||||
return this.automationStore;
|
||||
}
|
||||
|
||||
/** Get the project's working directory. */
|
||||
getWorkingDirectory(): string {
|
||||
return this.config.workingDirectory;
|
||||
}
|
||||
|
||||
/** Get the HeartbeatMonitor from the underlying runtime, if initialized. */
|
||||
getHeartbeatMonitor(): HeartbeatMonitor | undefined {
|
||||
return this.runtime.getHeartbeatMonitor();
|
||||
}
|
||||
|
||||
/** Get the HeartbeatTriggerScheduler from the underlying runtime, if initialized. */
|
||||
getHeartbeatTriggerScheduler(): HeartbeatTriggerScheduler | undefined {
|
||||
return this.runtime.getTriggerScheduler();
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a task ID for auto-merge if it is not already queued or active.
|
||||
* Exposed publicly so callers can integrate the engine's merge queue with
|
||||
|
||||
@@ -418,14 +418,34 @@ describe("In-progress task resume after restart", () => {
|
||||
await executor.resumeOrphaned();
|
||||
|
||||
expect(executeSpy).not.toHaveBeenCalled();
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-963", {
|
||||
modifiedFiles: ["packages/dashboard/app/components/SettingsModal.tsx"],
|
||||
await vi.waitFor(() => {
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-963", {
|
||||
modifiedFiles: ["packages/dashboard/app/components/SettingsModal.tsx"],
|
||||
});
|
||||
expect(store.moveTask).toHaveBeenCalledWith("FN-963", "in-review");
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-963",
|
||||
"Auto-recovered: task work was complete but stuck in in-progress — moved to in-review",
|
||||
);
|
||||
});
|
||||
expect(store.moveTask).toHaveBeenCalledWith("FN-963", "in-review");
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
"FN-963",
|
||||
"Auto-recovered: task work was complete but stuck in in-progress — moved to in-review",
|
||||
});
|
||||
|
||||
it("resumeOrphaned() does not block startup on completed-task recovery", async () => {
|
||||
const store = createMockStore();
|
||||
const completedTask = makeTask("FN-964", "in-progress", {
|
||||
worktree: "/tmp/wt/FN-964",
|
||||
baseCommitSha: "base123",
|
||||
steps: makeSteps("done", "done", "skipped"),
|
||||
});
|
||||
store.listTasks.mockResolvedValue([completedTask]);
|
||||
|
||||
const executor = new TaskExecutor(store, "/tmp/test");
|
||||
const recoverSpy = vi.spyOn(executor, "recoverCompletedTask").mockImplementation(
|
||||
() => new Promise(() => {}),
|
||||
);
|
||||
|
||||
await expect(executor.resumeOrphaned()).resolves.toBeUndefined();
|
||||
expect(recoverSpy).toHaveBeenCalledWith(completedTask);
|
||||
});
|
||||
|
||||
it("resumeOrphaned() leaves no-progress no-task_done failures for self-healing", async () => {
|
||||
|
||||
@@ -466,7 +466,9 @@ export class InProcessRuntime
|
||||
// they no longer have a tracked session/worktree, so the stuck detector
|
||||
// cannot recover them. Delegate the startup recovery pass to
|
||||
// SelfHealingManager so the policy lives in one place.
|
||||
await this.selfHealingManager.runStartupRecovery();
|
||||
void this.selfHealingManager.runStartupRecovery().catch((err) => {
|
||||
runtimeLog.error("Self-healing startup recovery failed:", err);
|
||||
});
|
||||
|
||||
// 11. Start scheduler and triage processor
|
||||
this.scheduler.start();
|
||||
|
||||
Reference in New Issue
Block a user