fix(FN-1290): recover orphaned stuck tasks

This commit is contained in:
gsxdsm
2026-04-08 20:22:58 -07:00
parent a9bbb25fd0
commit f2829c7634
5 changed files with 229 additions and 4 deletions

View File

@@ -8,10 +8,12 @@ const {
mockSelfHealingStart,
mockSelfHealingStop,
mockSelfHealingCtor,
mockExecutorCtor,
} = vi.hoisted(() => ({
mockSelfHealingStart: vi.fn(),
mockSelfHealingStop: vi.fn(),
mockSelfHealingCtor: vi.fn(),
mockExecutorCtor: vi.fn(),
}));
// Mock the TaskStore class
@@ -79,9 +81,14 @@ vi.mock("../self-healing.js", async () => {
// Mock the executor
vi.mock("../executor.js", async () => {
return {
TaskExecutor: vi.fn().mockImplementation(() => {
TaskExecutor: vi.fn().mockImplementation((_store, _rootDir, options) => {
mockExecutorCtor(options);
const self = {} as Record<string, unknown>;
self.resumeOrphaned = vi.fn().mockResolvedValue(undefined);
self.recoverCompletedTask = vi.fn().mockResolvedValue(true);
self.getExecutingTaskIds = vi.fn().mockReturnValue(new Set());
self.handleLoopDetected = vi.fn().mockResolvedValue(false);
self.markStuckAborted = vi.fn();
self.activeWorktrees = new Map();
return self;
}),
@@ -146,6 +153,17 @@ describe("InProcessRuntime", () => {
expect(mockSelfHealingStart).toHaveBeenCalled();
});
it("creates a stuck task detector and passes it to the executor", async () => {
await runtime.start();
expect(mockExecutorCtor).toHaveBeenCalledWith(
expect.objectContaining({
stuckTaskDetector: expect.any(Object),
}),
);
expect((runtime as any).stuckTaskDetector).toBeDefined();
});
it("should transition to 'stopped' after stop", async () => {
await runtime.start();
await runtime.stop();

View File

@@ -20,7 +20,7 @@ import type {
ProjectRuntimeEvents,
} from "../project-runtime.js";
import { runtimeLog } from "../logger.js";
import type { StuckTaskDetector } from "../stuck-task-detector.js";
import { StuckTaskDetector } from "../stuck-task-detector.js";
import type { UsageLimitPauser } from "../usage-limit-detector.js";
import { SelfHealingManager } from "../self-healing.js";
import { MissionAutopilot } from "../mission-autopilot.js";
@@ -163,6 +163,18 @@ export class InProcessRuntime
});
// 5. Initialize TaskExecutor
this.stuckTaskDetector = new StuckTaskDetector(this.taskStore, {
beforeRequeue: (taskId) => this.selfHealingManager?.checkStuckBudget(taskId) ?? Promise.resolve(true),
onLoopDetected: (event) => this.executor?.handleLoopDetected(event) ?? Promise.resolve(false),
onStuck: (event) => {
this.executor?.markStuckAborted(event.taskId, event.shouldRequeue);
runtimeLog.warn(
`Task ${event.taskId} stuck (${event.reason}) — ` +
`${event.shouldRequeue ? "will retry" : "budget exhausted"}`,
);
},
});
const executorOptions: TaskExecutorOptions = {
semaphore: this.globalSemaphore,
pool: this.worktreePool,
@@ -315,6 +327,7 @@ export class InProcessRuntime
getExecutingTaskIds: () => this.executor.getExecutingTaskIds(),
});
this.selfHealingManager.start();
this.stuckTaskDetector.start();
// 8. Set up event forwarding from TaskStore
this.setupEventForwarding();
@@ -376,13 +389,19 @@ export class InProcessRuntime
runtimeLog.log("TriggerScheduler stopped");
}
// 3. Stop heartbeat monitor
// 3. Stop stuck task detector
if (this.stuckTaskDetector) {
this.stuckTaskDetector.stop();
runtimeLog.log("StuckTaskDetector stopped");
}
// 4. Stop heartbeat monitor
if (this.heartbeatMonitor) {
this.heartbeatMonitor.stop();
runtimeLog.log("HeartbeatMonitor stopped");
}
// 4. Stop scheduler (prevents new task scheduling)
// 5. Stop scheduler (prevents new task scheduling)
if (this.scheduler) {
this.scheduler.stop();
runtimeLog.log("Scheduler stopped");