root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-1218): harden mission autopilot with self-healing recovery

Browse Source 
- Add mission self-healing project settings for stale activation thresholds, task retry budgets, and mission health-check intervals
- Extend MissionAutopilot with failed-task retry handling, blocked feature escalation, stale activating mission recovery, and periodic feature/task consistency reconciliation
- Wire autopilot failure and recovery flows through Scheduler and InProcessRuntime, including startup recovery for watched missions after crashes
- Add blocked feature status support in core mission types and CLI mission status labels
- Expand mission-autopilot coverage for retry flows, stale/startup recovery, health checks, and regression scenarios, and include a patch changeset for @gsxdsm/fusion
This commit is contained in:
gsxdsm
2026-04-08 10:01:06 -07:00
parent a321b89011
commit dfa109742a
8 changed files with 904 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
packages/engine/src/scheduler.ts
View File

@@ -59,6 +59,8 @@ export interface SchedulerOptions {
onSchedule?: (task: Task) => void;
/** Called when a task is blocked by deps */
onBlocked?: (task: Task, blockedBy: string[]) => void;
/** Called when a mission-linked task fails and is queued for retry handling. */
onTaskFailed?: (taskId: string) => void | Promise<void>;
/** Optional PR monitor for tracking in-review PRs */
prMonitor?: PrMonitor;
/** Optional MissionStore for slice activation and auto-advance */
@@ -105,6 +107,8 @@ export class Scheduler {
private activePollMs: number | null = null;
/** Tracks which task IDs are currently paused, to detect unpause transitions. */
private pausedTaskIds = new Set<string>();
/** Tracks mission-linked tasks observed with status=failed before moveTask clears status/error. */
private failedTaskIds = new Set<string>();
constructor(
private store: TaskStore,
@@ -194,6 +198,17 @@ export class Scheduler {
void this.handleMissionTaskCompletion(task.id, task.sliceId);
}
// Mission failure tracking: status/error are cleared during moveTask(in-progress → todo),
// so we pair this with failedTaskIds captured from task:updated events.
if (task.sliceId && to === "todo" && this.options.onTaskFailed) {
if (task.status === "failed" || this.failedTaskIds.has(task.id)) {
this.failedTaskIds.delete(task.id);
void Promise.resolve(this.options.onTaskFailed(task.id)).catch((err) => {
schedulerLog.error(`Error in onTaskFailed for ${task.id}:`, err);
});
}
}
// Event-driven scheduling: when a task moves to "done" (completion) or "todo" (retry/manual move),
// trigger scheduling immediately so waiting tasks can start without waiting
// for the next poll interval (up to 15 seconds).
@@ -208,6 +223,13 @@ export class Scheduler {
* Also detects task-level unpause transitions and triggers immediate scheduling.
*/
this.store.on("task:updated", (task) => {
// Track mission failure signals before moveTask clears failure metadata.
if (task.sliceId && task.column === "in-progress" && task.status === "failed") {
this.failedTaskIds.add(task.id);
} else if (task.status !== "failed") {
this.failedTaskIds.delete(task.id);
}
// Track pause state transitions for event-driven scheduling on unpause.
// When a previously-paused task is unpaused in a schedulable column,
// trigger a scheduling pass immediately instead of waiting for the next
@@ -313,6 +335,7 @@ export class Scheduler {
if (this.options.missionAutopilot) {
this.options.missionAutopilot.stop();
}
this.failedTaskIds.clear();
schedulerLog.log("Stopped");
}
@@ -332,6 +355,10 @@ export class Scheduler {
schedulerLog.log(`Poll interval updated to ${newIntervalMs}ms`);
}
getMissionAutopilot(): import("./mission-autopilot.js").MissionAutopilot | undefined {
return this.options.missionAutopilot;
}
/**
* Resolve the base branch for a task being started.
*