feat(FN-1218): harden mission autopilot with self-healing recovery

- Add mission self-healing project settings for stale activation thresholds, task retry budgets, and mission health-check intervals
- Extend MissionAutopilot with failed-task retry handling, blocked feature escalation, stale activating mission recovery, and periodic feature/task consistency reconciliation
- Wire autopilot failure and recovery flows through Scheduler and InProcessRuntime, including startup recovery for watched missions after crashes
- Add blocked feature status support in core mission types and CLI mission status labels
- Expand mission-autopilot coverage for retry flows, stale/startup recovery, health checks, and regression scenarios, and include a patch changeset for @gsxdsm/fusion
This commit is contained in:
gsxdsm
2026-04-08 10:01:06 -07:00
parent a321b89011
commit dfa109742a
8 changed files with 904 additions and 34 deletions

View File

@@ -23,6 +23,7 @@ import { runtimeLog } from "../logger.js";
import type { StuckTaskDetector } from "../stuck-task-detector.js";
import type { UsageLimitPauser } from "../usage-limit-detector.js";
import { SelfHealingManager } from "../self-healing.js";
import { MissionAutopilot } from "../mission-autopilot.js";
/**
* InProcessRuntime runs a project within the main process.
@@ -137,11 +138,21 @@ export class InProcessRuntime
// 4. Initialize Scheduler
const missionStore = this.taskStore.getMissionStore();
const missionAutopilot = missionStore
? new MissionAutopilot(this.taskStore, missionStore)
: undefined;
this.scheduler = new Scheduler(this.taskStore, {
maxConcurrent: this.config.maxConcurrent,
maxWorktrees: this.config.maxWorktrees,
semaphore: this.globalSemaphore,
missionStore,
missionAutopilot,
onTaskFailed: (taskId) => {
if (missionAutopilot) {
void missionAutopilot.handleTaskFailure(taskId);
}
},
onSchedule: (task) => {
this.recordActivity();
runtimeLog.log(`Scheduled task ${task.id}`);
@@ -193,6 +204,22 @@ export class InProcessRuntime
this.recordActivity();
runtimeLog.error(`Task ${task.id} failed:`, error.message);
this.recordTaskCompletion(task.id, false);
// Mission-linked failures should be re-queued to todo so autopilot retry
// policies can decide whether to retry or block the feature.
if (task.sliceId) {
void (async () => {
try {
const latest = await this.taskStore.getTask(task.id);
if (latest?.column === "in-progress") {
await this.taskStore.moveTask(task.id, "todo");
}
} catch (moveErr) {
runtimeLog.warn(`Failed to requeue mission task ${task.id} after error:`, moveErr);
}
})();
}
// Update agent state to terminated (failed)
const agentId = this.taskAgentMap.get(task.id);
if (agentId && this.agentStore) {
@@ -288,6 +315,13 @@ export class InProcessRuntime
// 10. Start scheduler
this.scheduler.start();
// Mission crash recovery: restore autopilot state for missions that were active before crash
const activeMissionStore = this.taskStore.getMissionStore();
const activeMissionAutopilot = this.scheduler.getMissionAutopilot?.();
if (activeMissionStore && activeMissionAutopilot) {
void activeMissionAutopilot.recoverMissions(activeMissionStore);
}
this.setStatus("active");
runtimeLog.log(`InProcessRuntime started for project ${this.config.projectId}`);
} catch (error) {