fix(engine): recover orphaned tasks at startup
This commit is contained in:
@@ -8,11 +8,13 @@ const {
|
||||
mockSelfHealingStart,
|
||||
mockSelfHealingStop,
|
||||
mockSelfHealingCtor,
|
||||
mockRunStartupRecovery,
|
||||
mockExecutorCtor,
|
||||
} = vi.hoisted(() => ({
|
||||
mockSelfHealingStart: vi.fn(),
|
||||
mockSelfHealingStop: vi.fn(),
|
||||
mockSelfHealingCtor: vi.fn(),
|
||||
mockRunStartupRecovery: vi.fn().mockResolvedValue(undefined),
|
||||
mockExecutorCtor: vi.fn(),
|
||||
}));
|
||||
|
||||
@@ -92,6 +94,7 @@ vi.mock("../self-healing.js", async () => {
|
||||
return {
|
||||
start: mockSelfHealingStart,
|
||||
stop: mockSelfHealingStop,
|
||||
runStartupRecovery: mockRunStartupRecovery,
|
||||
};
|
||||
}),
|
||||
};
|
||||
@@ -184,6 +187,12 @@ describe("InProcessRuntime", () => {
|
||||
expect(mockSelfHealingStart).toHaveBeenCalled();
|
||||
}, 30000);
|
||||
|
||||
it("runs self-healing startup recovery immediately after orphan resume on startup", async () => {
|
||||
await runtime.start();
|
||||
|
||||
expect(mockRunStartupRecovery).toHaveBeenCalledTimes(1);
|
||||
}, 30000);
|
||||
|
||||
it("creates a stuck task detector and passes it to the executor", async () => {
|
||||
await runtime.start();
|
||||
|
||||
|
||||
@@ -360,6 +360,12 @@ export class InProcessRuntime
|
||||
// 9. Resume orphaned in-progress tasks
|
||||
await this.executor.resumeOrphaned();
|
||||
|
||||
// Some "stuck" tasks are already orphaned by the time the runtime boots:
|
||||
// they no longer have a tracked session/worktree, so the stuck detector
|
||||
// cannot recover them. Delegate the startup recovery pass to
|
||||
// SelfHealingManager so the policy lives in one place.
|
||||
await this.selfHealingManager.runStartupRecovery();
|
||||
|
||||
// 10. Start scheduler
|
||||
this.scheduler.start();
|
||||
|
||||
|
||||
@@ -320,6 +320,20 @@ describe("SelfHealingManager", () => {
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
expect(store.updateSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("runStartupRecovery invokes the startup recovery subset", async () => {
|
||||
const recoverCompletedTasks = vi.spyOn(manager, "recoverCompletedTasks").mockResolvedValue(1);
|
||||
const recoverMisclassifiedFailures = vi.spyOn(manager, "recoverMisclassifiedFailures").mockResolvedValue(1);
|
||||
const recoverOrphanedExecutions = vi.spyOn(manager, "recoverOrphanedExecutions").mockResolvedValue(1);
|
||||
const recoverApprovedTriageTasks = vi.spyOn(manager, "recoverApprovedTriageTasks").mockResolvedValue(1);
|
||||
|
||||
await manager.runStartupRecovery();
|
||||
|
||||
expect(recoverCompletedTasks).toHaveBeenCalledTimes(1);
|
||||
expect(recoverMisclassifiedFailures).toHaveBeenCalledTimes(1);
|
||||
expect(recoverOrphanedExecutions).toHaveBeenCalledTimes(1);
|
||||
expect(recoverApprovedTriageTasks).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── cleanupOrphanedBranches ────────────────────────────────────────
|
||||
|
||||
@@ -95,6 +95,20 @@ export class SelfHealingManager {
|
||||
log.log("Started");
|
||||
}
|
||||
|
||||
/**
|
||||
* Run only the recovery subset needed at runtime startup, after the executor
|
||||
* has had a chance to resume orphaned sessions.
|
||||
*
|
||||
* This avoids waiting for the periodic maintenance interval before fixing
|
||||
* stale in-progress/specifying tasks that no longer have a live worker.
|
||||
*/
|
||||
async runStartupRecovery(): Promise<void> {
|
||||
await this.recoverCompletedTasks();
|
||||
await this.recoverMisclassifiedFailures();
|
||||
await this.recoverOrphanedExecutions();
|
||||
await this.recoverApprovedTriageTasks();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
// Remove settings listener
|
||||
if (this.settingsListener) {
|
||||
|
||||
Reference in New Issue
Block a user