fix(FN-1473): recover no-progress task_done failures

This commit is contained in:
gsxdsm
2026-04-12 12:13:06 -07:00
parent f915764e80
commit b2e25c08b2
8 changed files with 395 additions and 4 deletions

View File

@@ -11,12 +11,14 @@ const {
mockSelfHealingStart,
mockSelfHealingStop,
mockSelfHealingCtor,
mockRecoverNoProgressNoTaskDoneFailures,
mockRunStartupRecovery,
mockExecutorCtor,
} = vi.hoisted(() => ({
mockSelfHealingStart: vi.fn(),
mockSelfHealingStop: vi.fn(),
mockSelfHealingCtor: vi.fn(),
mockRecoverNoProgressNoTaskDoneFailures: vi.fn().mockResolvedValue(0),
mockRunStartupRecovery: vi.fn().mockResolvedValue(undefined),
mockExecutorCtor: vi.fn(),
}));
@@ -97,6 +99,7 @@ vi.mock("../self-healing.js", async () => {
return {
start: mockSelfHealingStart,
stop: mockSelfHealingStop,
recoverNoProgressNoTaskDoneFailures: mockRecoverNoProgressNoTaskDoneFailures,
runStartupRecovery: mockRunStartupRecovery,
};
}),
@@ -207,6 +210,7 @@ describe("InProcessRuntime", () => {
it("runs self-healing startup recovery immediately after orphan resume on startup", async () => {
await runtime.start();
expect(mockRecoverNoProgressNoTaskDoneFailures).toHaveBeenCalledTimes(1);
expect(mockRunStartupRecovery).toHaveBeenCalledTimes(1);
}, 30000);

View File

@@ -417,7 +417,11 @@ export class InProcessRuntime
// 8. Set up event forwarding from TaskStore
this.setupEventForwarding();
// 9. Resume orphaned in-progress tasks
// 9. Requeue no-progress no-task_done failures before resumeOrphaned
// can restart them.
await this.selfHealingManager.recoverNoProgressNoTaskDoneFailures();
// 10. Resume orphaned in-progress tasks
await this.executor.resumeOrphaned();
// Some "stuck" tasks are already orphaned by the time the runtime boots:
@@ -426,10 +430,10 @@ export class InProcessRuntime
// SelfHealingManager so the policy lives in one place.
await this.selfHealingManager.runStartupRecovery();
// 10. Start scheduler
// 11. Start scheduler
this.scheduler.start();
// 11. Start MissionExecutionLoop for validation cycle handling
// 12. Start MissionExecutionLoop for validation cycle handling
this.missionExecutionLoop = missionExecutionLoop;
if (missionExecutionLoop) {
missionExecutionLoop.start();
@@ -446,7 +450,7 @@ export class InProcessRuntime
void activeMissionAutopilot.recoverMissions(activeMissionStore);
}
// 12. Reconcile feature status for all active missions (not just autopilot)
// 13. Reconcile feature status for all active missions (not just autopilot)
if (activeMissionStore) {
void this.scheduler.reconcileAllMissionFeatures();
}