feat(FN-1218): harden mission autopilot with self-healing recovery
- Add mission self-healing project settings for stale activation thresholds, task retry budgets, and mission health-check intervals - Extend MissionAutopilot with failed-task retry handling, blocked feature escalation, stale activating mission recovery, and periodic feature/task consistency reconciliation - Wire autopilot failure and recovery flows through Scheduler and InProcessRuntime, including startup recovery for watched missions after crashes - Add blocked feature status support in core mission types and CLI mission status labels - Expand mission-autopilot coverage for retry flows, stale/startup recovery, health checks, and regression scenarios, and include a patch changeset for @gsxdsm/fusion
This commit is contained in:
5
.changeset/mission-self-healing.md
Normal file
5
.changeset/mission-self-healing.md
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
---
|
||||||
|
"@gsxdsm/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
Add mission self-healing resilience with stale activation recovery, failed-task retry budgets that can block exhausted features, periodic mission health consistency checks, and startup crash recovery of watched missions.
|
||||||
@@ -30,6 +30,7 @@ const FEATURE_STATUS_LABELS: Record<FeatureStatus, string> = {
|
|||||||
triaged: "Triaged",
|
triaged: "Triaged",
|
||||||
"in-progress": "In Progress",
|
"in-progress": "In Progress",
|
||||||
done: "Done",
|
done: "Done",
|
||||||
|
blocked: "Blocked",
|
||||||
};
|
};
|
||||||
|
|
||||||
async function promptForTitleAndDescription(
|
async function promptForTitleAndDescription(
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ export const SLICE_STATUSES = ["pending", "active", "complete"] as const;
|
|||||||
export type SliceStatus = (typeof SLICE_STATUSES)[number];
|
export type SliceStatus = (typeof SLICE_STATUSES)[number];
|
||||||
|
|
||||||
/** Status values for a Feature within a slice */
|
/** Status values for a Feature within a slice */
|
||||||
export const FEATURE_STATUSES = ["defined", "triaged", "in-progress", "done"] as const;
|
export const FEATURE_STATUSES = ["defined", "triaged", "in-progress", "done", "blocked"] as const;
|
||||||
export type FeatureStatus = (typeof FEATURE_STATUSES)[number];
|
export type FeatureStatus = (typeof FEATURE_STATUSES)[number];
|
||||||
|
|
||||||
/** Interview state for AI-assisted specification */
|
/** Interview state for AI-assisted specification */
|
||||||
|
|||||||
@@ -967,6 +967,18 @@ export interface ProjectSettings {
|
|||||||
* enabled and steps have non-overlapping file scopes. Range: 1–4.
|
* enabled and steps have non-overlapping file scopes. Range: 1–4.
|
||||||
* Default: 2. */
|
* Default: 2. */
|
||||||
maxParallelSteps?: number;
|
maxParallelSteps?: number;
|
||||||
|
/** Time in milliseconds after which a mission in `activating` state is
|
||||||
|
* considered stale and eligible for self-healing recovery.
|
||||||
|
* Default: 600000 (10 minutes). */
|
||||||
|
missionStaleThresholdMs?: number;
|
||||||
|
/** Maximum automatic retry attempts for a failed mission-linked task before
|
||||||
|
* its feature is marked as blocked for manual intervention.
|
||||||
|
* Default: 3. */
|
||||||
|
missionMaxTaskRetries?: number;
|
||||||
|
/** Interval in milliseconds between mission feature/task consistency checks.
|
||||||
|
* Set to 0 to disable periodic health checks.
|
||||||
|
* Default: 300000 (5 minutes). */
|
||||||
|
missionHealthCheckIntervalMs?: number;
|
||||||
/** Configurable agent role prompt templates and assignments.
|
/** Configurable agent role prompt templates and assignments.
|
||||||
* When set, allows per-project customization of system prompts
|
* When set, allows per-project customization of system prompts
|
||||||
* for different agent roles (executor, triage, reviewer, merger). */
|
* for different agent roles (executor, triage, reviewer, merger). */
|
||||||
@@ -1071,6 +1083,9 @@ export const DEFAULT_PROJECT_SETTINGS: ProjectSettings = {
|
|||||||
tokenCap: undefined,
|
tokenCap: undefined,
|
||||||
runStepsInNewSessions: false,
|
runStepsInNewSessions: false,
|
||||||
maxParallelSteps: 2,
|
maxParallelSteps: 2,
|
||||||
|
missionStaleThresholdMs: 600_000,
|
||||||
|
missionMaxTaskRetries: 3,
|
||||||
|
missionHealthCheckIntervalMs: 300_000,
|
||||||
agentPrompts: undefined,
|
agentPrompts: undefined,
|
||||||
reflectionEnabled: false,
|
reflectionEnabled: false,
|
||||||
reflectionIntervalMs: 3_600_000,
|
reflectionIntervalMs: 3_600_000,
|
||||||
@@ -1160,6 +1175,9 @@ export const PROJECT_SETTINGS_KEYS: ReadonlyArray<keyof ProjectSettings> = [
|
|||||||
"maxSpawnedAgentsGlobal",
|
"maxSpawnedAgentsGlobal",
|
||||||
"runStepsInNewSessions",
|
"runStepsInNewSessions",
|
||||||
"maxParallelSteps",
|
"maxParallelSteps",
|
||||||
|
"missionStaleThresholdMs",
|
||||||
|
"missionMaxTaskRetries",
|
||||||
|
"missionHealthCheckIntervalMs",
|
||||||
"agentPrompts",
|
"agentPrompts",
|
||||||
"reflectionEnabled",
|
"reflectionEnabled",
|
||||||
"reflectionIntervalMs",
|
"reflectionIntervalMs",
|
||||||
|
|||||||
@@ -92,6 +92,7 @@ function createMockMissionStore(missions: Mission[] = []) {
|
|||||||
listSlices: vi.fn(),
|
listSlices: vi.fn(),
|
||||||
getFeatureByTaskId: vi.fn(),
|
getFeatureByTaskId: vi.fn(),
|
||||||
listFeatures: vi.fn(),
|
listFeatures: vi.fn(),
|
||||||
|
updateFeatureStatus: vi.fn(),
|
||||||
getMissionWithHierarchy: vi.fn(),
|
getMissionWithHierarchy: vi.fn(),
|
||||||
on: vi.fn(),
|
on: vi.fn(),
|
||||||
off: vi.fn(),
|
off: vi.fn(),
|
||||||
@@ -103,6 +104,14 @@ function createMockTaskStore() {
|
|||||||
return {
|
return {
|
||||||
on: vi.fn(),
|
on: vi.fn(),
|
||||||
off: vi.fn(),
|
off: vi.fn(),
|
||||||
|
getSettings: vi.fn().mockResolvedValue({
|
||||||
|
missionStaleThresholdMs: 600_000,
|
||||||
|
missionMaxTaskRetries: 3,
|
||||||
|
missionHealthCheckIntervalMs: 300_000,
|
||||||
|
}),
|
||||||
|
getTask: vi.fn().mockResolvedValue({ id: "FN-001", column: "in-progress" }),
|
||||||
|
moveTask: vi.fn().mockResolvedValue({ id: "FN-001", column: "todo" }),
|
||||||
|
updateTask: vi.fn().mockResolvedValue({}),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -356,6 +365,106 @@ describe("MissionAutopilot", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("handleTaskFailure", () => {
|
||||||
|
function wireMissionTask(taskId = "FN-001") {
|
||||||
|
const feature = createMockFeature({ id: "F-001", taskId, sliceId: "SL-001", status: "in-progress" });
|
||||||
|
const slice = createMockSlice({ id: "SL-001", milestoneId: "MS-001" });
|
||||||
|
const milestone = createMockMilestone({ id: "MS-001", missionId: "M-TEST1" });
|
||||||
|
|
||||||
|
missionStore.getFeatureByTaskId.mockReturnValue(feature);
|
||||||
|
missionStore.getSlice.mockReturnValue(slice);
|
||||||
|
missionStore.getMilestone.mockReturnValue(milestone);
|
||||||
|
|
||||||
|
return { feature, slice, milestone };
|
||||||
|
}
|
||||||
|
|
||||||
|
it("increments retries and requeues failed tasks", async () => {
|
||||||
|
wireMissionTask();
|
||||||
|
autopilot.watchMission("M-TEST1");
|
||||||
|
|
||||||
|
await autopilot.handleTaskFailure("FN-001");
|
||||||
|
|
||||||
|
expect(taskStore.moveTask).toHaveBeenCalledWith("FN-001", "todo");
|
||||||
|
expect(taskStore.updateTask).toHaveBeenCalledWith(
|
||||||
|
"FN-001",
|
||||||
|
expect.objectContaining({ error: null, status: null, paused: false }),
|
||||||
|
);
|
||||||
|
expect(missionStore.updateFeatureStatus).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("marks feature blocked after max retries and does not retry again", async () => {
|
||||||
|
const { feature } = wireMissionTask();
|
||||||
|
autopilot.watchMission("M-TEST1");
|
||||||
|
taskStore.getSettings.mockResolvedValue({
|
||||||
|
missionStaleThresholdMs: 600_000,
|
||||||
|
missionMaxTaskRetries: 1,
|
||||||
|
missionHealthCheckIntervalMs: 300_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
await autopilot.handleTaskFailure("FN-001");
|
||||||
|
await autopilot.handleTaskFailure("FN-001");
|
||||||
|
|
||||||
|
expect(missionStore.updateFeatureStatus).toHaveBeenCalledWith(feature.id, "blocked");
|
||||||
|
expect(taskStore.moveTask).toHaveBeenCalledTimes(1);
|
||||||
|
expect(taskStore.updateTask).toHaveBeenCalledWith(
|
||||||
|
"FN-001",
|
||||||
|
expect.objectContaining({ status: "failed", paused: true }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clears retry budget for a task after successful completion", async () => {
|
||||||
|
wireMissionTask();
|
||||||
|
autopilot.watchMission("M-TEST1");
|
||||||
|
taskStore.getSettings.mockResolvedValue({
|
||||||
|
missionStaleThresholdMs: 600_000,
|
||||||
|
missionMaxTaskRetries: 1,
|
||||||
|
missionHealthCheckIntervalMs: 300_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
await autopilot.handleTaskFailure("FN-001");
|
||||||
|
|
||||||
|
missionStore.listFeatures.mockReturnValue([
|
||||||
|
createMockFeature({ id: "F-001", taskId: "FN-001", sliceId: "SL-001", status: "in-progress" }),
|
||||||
|
]);
|
||||||
|
await autopilot.handleTaskCompletion("FN-001");
|
||||||
|
await autopilot.handleTaskFailure("FN-001");
|
||||||
|
|
||||||
|
expect(missionStore.updateFeatureStatus).not.toHaveBeenCalledWith("F-001", "blocked");
|
||||||
|
expect(taskStore.moveTask).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is a no-op when task is not linked to a feature", async () => {
|
||||||
|
missionStore.getFeatureByTaskId.mockReturnValue(undefined);
|
||||||
|
|
||||||
|
await autopilot.handleTaskFailure("FN-001");
|
||||||
|
|
||||||
|
expect(taskStore.moveTask).not.toHaveBeenCalled();
|
||||||
|
expect(taskStore.updateTask).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is a no-op for missions that are not being watched", async () => {
|
||||||
|
wireMissionTask();
|
||||||
|
|
||||||
|
await autopilot.handleTaskFailure("FN-001");
|
||||||
|
|
||||||
|
expect(taskStore.moveTask).not.toHaveBeenCalled();
|
||||||
|
expect(taskStore.updateTask).not.toHaveBeenCalled();
|
||||||
|
expect(missionStore.updateFeatureStatus).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clears task error when retrying", async () => {
|
||||||
|
wireMissionTask();
|
||||||
|
autopilot.watchMission("M-TEST1");
|
||||||
|
|
||||||
|
await autopilot.handleTaskFailure("FN-001");
|
||||||
|
|
||||||
|
expect(taskStore.updateTask).toHaveBeenCalledWith(
|
||||||
|
"FN-001",
|
||||||
|
expect.objectContaining({ error: null, status: null }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// ── Advance to Next Slice ────────────────────────────────────────
|
// ── Advance to Next Slice ────────────────────────────────────────
|
||||||
|
|
||||||
describe("advanceToNextSlice", () => {
|
describe("advanceToNextSlice", () => {
|
||||||
@@ -500,33 +609,364 @@ describe("MissionAutopilot", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("health check", () => {
|
||||||
|
it("fixes feature status when linked task is done", async () => {
|
||||||
|
autopilot.start();
|
||||||
|
autopilot.watchMission("M-TEST1");
|
||||||
|
missionStore.getMissionWithHierarchy.mockReturnValue({
|
||||||
|
...createMockMission(),
|
||||||
|
milestones: [{
|
||||||
|
...createMockMilestone(),
|
||||||
|
slices: [{
|
||||||
|
...createMockSlice({ status: "active" }),
|
||||||
|
features: [createMockFeature({ id: "F-001", status: "triaged", taskId: "FN-001" })],
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
taskStore.getTask.mockResolvedValue({ id: "FN-001", column: "done" });
|
||||||
|
|
||||||
|
await (autopilot as any).runHealthCheck();
|
||||||
|
|
||||||
|
expect(missionStore.updateFeatureStatus).toHaveBeenCalledWith("F-001", "done");
|
||||||
|
autopilot.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fixes feature status when task is in-progress but feature is triaged", async () => {
|
||||||
|
autopilot.start();
|
||||||
|
autopilot.watchMission("M-TEST1");
|
||||||
|
missionStore.getMissionWithHierarchy.mockReturnValue({
|
||||||
|
...createMockMission(),
|
||||||
|
milestones: [{
|
||||||
|
...createMockMilestone(),
|
||||||
|
slices: [{
|
||||||
|
...createMockSlice({ status: "active" }),
|
||||||
|
features: [createMockFeature({ id: "F-001", status: "triaged", taskId: "FN-001" })],
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
taskStore.getTask.mockResolvedValue({ id: "FN-001", column: "in-progress" });
|
||||||
|
|
||||||
|
await (autopilot as any).runHealthCheck();
|
||||||
|
|
||||||
|
expect(missionStore.updateFeatureStatus).toHaveBeenCalledWith("F-001", "in-progress");
|
||||||
|
autopilot.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fixes feature status when task regresses to todo/triage", async () => {
|
||||||
|
autopilot.start();
|
||||||
|
autopilot.watchMission("M-TEST1");
|
||||||
|
missionStore.getMissionWithHierarchy.mockReturnValue({
|
||||||
|
...createMockMission(),
|
||||||
|
milestones: [{
|
||||||
|
...createMockMilestone(),
|
||||||
|
slices: [{
|
||||||
|
...createMockSlice({ status: "active" }),
|
||||||
|
features: [createMockFeature({ id: "F-001", status: "in-progress", taskId: "FN-001" })],
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
taskStore.getTask.mockResolvedValue({ id: "FN-001", column: "todo" });
|
||||||
|
|
||||||
|
await (autopilot as any).runHealthCheck();
|
||||||
|
|
||||||
|
expect(missionStore.updateFeatureStatus).toHaveBeenCalledWith("F-001", "triaged");
|
||||||
|
autopilot.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("triggers failure recovery for failed in-progress tasks", async () => {
|
||||||
|
autopilot.start();
|
||||||
|
autopilot.watchMission("M-TEST1");
|
||||||
|
missionStore.getMissionWithHierarchy.mockReturnValue({
|
||||||
|
...createMockMission(),
|
||||||
|
milestones: [{
|
||||||
|
...createMockMilestone(),
|
||||||
|
slices: [{
|
||||||
|
...createMockSlice({ status: "active" }),
|
||||||
|
features: [createMockFeature({ id: "F-001", status: "in-progress", taskId: "FN-001" })],
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
taskStore.getTask.mockResolvedValue({ id: "FN-001", column: "in-progress", status: "failed" });
|
||||||
|
const failureSpy = vi.spyOn(autopilot, "handleTaskFailure").mockResolvedValue();
|
||||||
|
|
||||||
|
await (autopilot as any).runHealthCheck();
|
||||||
|
|
||||||
|
expect(failureSpy).toHaveBeenCalledWith("FN-001");
|
||||||
|
autopilot.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves consistent feature/task states unchanged", async () => {
|
||||||
|
autopilot.start();
|
||||||
|
autopilot.watchMission("M-TEST1");
|
||||||
|
missionStore.getMissionWithHierarchy.mockReturnValue({
|
||||||
|
...createMockMission(),
|
||||||
|
milestones: [{
|
||||||
|
...createMockMilestone(),
|
||||||
|
slices: [{
|
||||||
|
...createMockSlice({ status: "active" }),
|
||||||
|
features: [createMockFeature({ id: "F-001", status: "in-progress", taskId: "FN-001" })],
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
taskStore.getTask.mockResolvedValue({ id: "FN-001", column: "in-progress" });
|
||||||
|
|
||||||
|
await (autopilot as any).runHealthCheck();
|
||||||
|
|
||||||
|
expect(missionStore.updateFeatureStatus).not.toHaveBeenCalled();
|
||||||
|
autopilot.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips features that do not have linked tasks", async () => {
|
||||||
|
autopilot.start();
|
||||||
|
autopilot.watchMission("M-TEST1");
|
||||||
|
missionStore.getMissionWithHierarchy.mockReturnValue({
|
||||||
|
...createMockMission(),
|
||||||
|
milestones: [{
|
||||||
|
...createMockMilestone(),
|
||||||
|
slices: [{
|
||||||
|
...createMockSlice({ status: "active" }),
|
||||||
|
features: [createMockFeature({ id: "F-001", status: "in-progress", taskId: undefined })],
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
|
||||||
|
await (autopilot as any).runHealthCheck();
|
||||||
|
|
||||||
|
expect(taskStore.getTask).not.toHaveBeenCalled();
|
||||||
|
expect(missionStore.updateFeatureStatus).not.toHaveBeenCalled();
|
||||||
|
autopilot.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not create a health-check timer when disabled", async () => {
|
||||||
|
taskStore.getSettings.mockResolvedValue({
|
||||||
|
missionStaleThresholdMs: 600_000,
|
||||||
|
missionMaxTaskRetries: 3,
|
||||||
|
missionHealthCheckIntervalMs: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
autopilot.start();
|
||||||
|
await vi.runOnlyPendingTimersAsync();
|
||||||
|
|
||||||
|
expect((autopilot as any).healthCheckTimer).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses default health-check interval when setting is undefined", async () => {
|
||||||
|
taskStore.getSettings.mockResolvedValue({
|
||||||
|
missionStaleThresholdMs: 600_000,
|
||||||
|
missionMaxTaskRetries: 3,
|
||||||
|
missionHealthCheckIntervalMs: undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
autopilot.start();
|
||||||
|
await vi.runOnlyPendingTimersAsync();
|
||||||
|
|
||||||
|
expect((autopilot as any).healthCheckTimer).not.toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// ── Poll / stale detection ──────────────────────────────────────
|
// ── Poll / stale detection ──────────────────────────────────────
|
||||||
|
|
||||||
describe("poll stale detection", () => {
|
describe("poll stale detection", () => {
|
||||||
it("logs warning events for stale watched missions", () => {
|
it("recovers stale activating missions back to watching and advances slices", async () => {
|
||||||
const staleMission = createMockMission({
|
const staleMission = createMockMission({
|
||||||
lastAutopilotActivityAt: new Date(Date.now() - 10 * 60 * 1000).toISOString(),
|
autopilotState: "activating",
|
||||||
|
lastAutopilotActivityAt: new Date(Date.now() - 11 * 60 * 1000).toISOString(),
|
||||||
});
|
});
|
||||||
const store = createMockMissionStore([staleMission]);
|
const store = createMockMissionStore([staleMission]);
|
||||||
const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler });
|
const localScheduler = createMockScheduler();
|
||||||
|
localScheduler.activateNextPendingSlice.mockResolvedValue(
|
||||||
|
createMockSlice({ id: "SL-002", status: "active" }),
|
||||||
|
);
|
||||||
|
|
||||||
|
store.getMissionWithHierarchy.mockReturnValue({
|
||||||
|
...staleMission,
|
||||||
|
milestones: [{
|
||||||
|
...createMockMilestone({ missionId: staleMission.id }),
|
||||||
|
slices: [{
|
||||||
|
...createMockSlice({ id: "SL-001", status: "active" }),
|
||||||
|
features: [createMockFeature({ status: "done" })],
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
|
||||||
|
const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler: localScheduler });
|
||||||
|
ap.start();
|
||||||
|
ap.watchMission("M-TEST1");
|
||||||
|
store.updateMission("M-TEST1", {
|
||||||
|
autopilotState: "activating",
|
||||||
|
lastAutopilotActivityAt: staleMission.lastAutopilotActivityAt,
|
||||||
|
});
|
||||||
|
store.updateMission.mockClear();
|
||||||
|
store.logMissionEvent.mockClear();
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(60_000);
|
||||||
|
|
||||||
|
expect(store.updateMission).toHaveBeenCalledWith(
|
||||||
|
"M-TEST1",
|
||||||
|
expect.objectContaining({ autopilotState: "watching" }),
|
||||||
|
);
|
||||||
|
expect(localScheduler.activateNextPendingSlice).toHaveBeenCalledWith("M-TEST1");
|
||||||
|
expect(store.logMissionEvent).toHaveBeenCalledWith(
|
||||||
|
"M-TEST1",
|
||||||
|
"autopilot_stale",
|
||||||
|
expect.stringContaining("stale"),
|
||||||
|
expect.objectContaining({ staleThresholdMs: 600_000 }),
|
||||||
|
);
|
||||||
|
|
||||||
|
ap.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not recover missions that are not in activating state", async () => {
|
||||||
|
const staleWatchingMission = createMockMission({
|
||||||
|
autopilotState: "watching",
|
||||||
|
lastAutopilotActivityAt: new Date(Date.now() - 11 * 60 * 1000).toISOString(),
|
||||||
|
});
|
||||||
|
const store = createMockMissionStore([staleWatchingMission]);
|
||||||
|
const localScheduler = createMockScheduler();
|
||||||
|
const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler: localScheduler });
|
||||||
|
|
||||||
ap.start();
|
ap.start();
|
||||||
ap.watchMission("M-TEST1");
|
ap.watchMission("M-TEST1");
|
||||||
store.logMissionEvent.mockClear();
|
store.logMissionEvent.mockClear();
|
||||||
|
|
||||||
vi.advanceTimersByTime(60_000);
|
await vi.advanceTimersByTimeAsync(60_000);
|
||||||
|
|
||||||
expect(store.logMissionEvent).toHaveBeenCalledWith(
|
expect(localScheduler.activateNextPendingSlice).not.toHaveBeenCalled();
|
||||||
|
expect(store.logMissionEvent).not.toHaveBeenCalledWith(
|
||||||
"M-TEST1",
|
"M-TEST1",
|
||||||
"warning",
|
"autopilot_stale",
|
||||||
expect.stringContaining("stale"),
|
expect.any(String),
|
||||||
expect.objectContaining({ category: "autopilot_stale" }),
|
expect.anything(),
|
||||||
);
|
);
|
||||||
|
|
||||||
ap.stop();
|
ap.stop();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("recoverStaleMission", () => {
|
||||||
|
it("activates pending work when active slice is complete", async () => {
|
||||||
|
const mission = createMockMission();
|
||||||
|
const store = createMockMissionStore([mission]);
|
||||||
|
const localScheduler = createMockScheduler();
|
||||||
|
localScheduler.activateNextPendingSlice.mockResolvedValue(
|
||||||
|
createMockSlice({ id: "SL-002", status: "active" }),
|
||||||
|
);
|
||||||
|
const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler: localScheduler });
|
||||||
|
|
||||||
|
store.getMissionWithHierarchy.mockReturnValue({
|
||||||
|
...mission,
|
||||||
|
milestones: [{
|
||||||
|
...createMockMilestone({ missionId: mission.id }),
|
||||||
|
slices: [{
|
||||||
|
...createMockSlice({ id: "SL-001", status: "active" }),
|
||||||
|
features: [createMockFeature({ status: "done" })],
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
|
||||||
|
ap.watchMission("M-TEST1");
|
||||||
|
await ap.recoverStaleMission("M-TEST1");
|
||||||
|
|
||||||
|
expect(localScheduler.activateNextPendingSlice).toHaveBeenCalledWith("M-TEST1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("handles mission not found gracefully", async () => {
|
||||||
|
missionStore.getMissionWithHierarchy.mockReturnValue(undefined);
|
||||||
|
|
||||||
|
await expect(autopilot.recoverStaleMission("M-TEST1")).resolves.toBeUndefined();
|
||||||
|
expect(scheduler.activateNextPendingSlice).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("recoverMissions", () => {
|
||||||
|
it("watches eligible missions and skips complete/archived missions", async () => {
|
||||||
|
const missions = [
|
||||||
|
createMockMission({ id: "M-ONE", status: "active", autopilotEnabled: true }),
|
||||||
|
createMockMission({ id: "M-TWO", status: "complete", autopilotEnabled: true }),
|
||||||
|
createMockMission({ id: "M-THREE", status: "archived", autopilotEnabled: true }),
|
||||||
|
createMockMission({ id: "M-FOUR", status: "active", autopilotEnabled: false }),
|
||||||
|
];
|
||||||
|
const store = createMockMissionStore(missions);
|
||||||
|
const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler });
|
||||||
|
|
||||||
|
store.getMissionWithHierarchy.mockReturnValue(undefined);
|
||||||
|
await ap.recoverMissions(store as any);
|
||||||
|
|
||||||
|
expect(ap.isWatching("M-ONE")).toBe(true);
|
||||||
|
expect(ap.isWatching("M-TWO")).toBe(false);
|
||||||
|
expect(ap.isWatching("M-THREE")).toBe(false);
|
||||||
|
expect(ap.isWatching("M-FOUR")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("recovers missions stuck in activating state", async () => {
|
||||||
|
const mission = createMockMission({ autopilotState: "activating" });
|
||||||
|
const store = createMockMissionStore([mission]);
|
||||||
|
const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler });
|
||||||
|
const recoverStaleSpy = vi.spyOn(ap, "recoverStaleMission").mockResolvedValue();
|
||||||
|
|
||||||
|
store.getMissionWithHierarchy.mockReturnValue(undefined);
|
||||||
|
await ap.recoverMissions(store as any);
|
||||||
|
|
||||||
|
expect(recoverStaleSpy).toHaveBeenCalledWith(mission.id);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fixes feature/task inconsistencies during recovery", async () => {
|
||||||
|
const mission = createMockMission();
|
||||||
|
const store = createMockMissionStore([mission]);
|
||||||
|
const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler });
|
||||||
|
|
||||||
|
store.getMissionWithHierarchy.mockReturnValue({
|
||||||
|
...mission,
|
||||||
|
milestones: [{
|
||||||
|
...createMockMilestone({ missionId: mission.id }),
|
||||||
|
slices: [{
|
||||||
|
...createMockSlice({ status: "active" }),
|
||||||
|
features: [createMockFeature({ id: "F-001", status: "triaged", taskId: "FN-001" })],
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
taskStore.getTask.mockResolvedValue({ id: "FN-001", column: "done" });
|
||||||
|
|
||||||
|
await ap.recoverMissions(store as any);
|
||||||
|
|
||||||
|
expect(store.updateFeatureStatus).toHaveBeenCalledWith("F-001", "done");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("advances slices when active slice features are already done", async () => {
|
||||||
|
const mission = createMockMission();
|
||||||
|
const store = createMockMissionStore([mission]);
|
||||||
|
const localScheduler = createMockScheduler();
|
||||||
|
localScheduler.activateNextPendingSlice.mockResolvedValue(
|
||||||
|
createMockSlice({ id: "SL-002", status: "active" }),
|
||||||
|
);
|
||||||
|
const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler: localScheduler });
|
||||||
|
|
||||||
|
const hierarchy = {
|
||||||
|
...mission,
|
||||||
|
milestones: [{
|
||||||
|
...createMockMilestone({ missionId: mission.id }),
|
||||||
|
slices: [{
|
||||||
|
...createMockSlice({ id: "SL-001", status: "active" }),
|
||||||
|
features: [createMockFeature({ id: "F-001", status: "done", taskId: "FN-001" })],
|
||||||
|
}],
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
store.getMissionWithHierarchy.mockReturnValue(hierarchy);
|
||||||
|
taskStore.getTask.mockResolvedValue({ id: "FN-001", column: "done" });
|
||||||
|
|
||||||
|
await ap.recoverMissions(store as any);
|
||||||
|
|
||||||
|
expect(localScheduler.activateNextPendingSlice).toHaveBeenCalledWith(mission.id);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("handles empty mission lists", async () => {
|
||||||
|
const store = createMockMissionStore([]);
|
||||||
|
const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler });
|
||||||
|
|
||||||
|
await expect(ap.recoverMissions(store as any)).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// ── Stop cleanup ─────────────────────────────────────────────────
|
// ── Stop cleanup ─────────────────────────────────────────────────
|
||||||
|
|
||||||
describe("stop cleanup", () => {
|
describe("stop cleanup", () => {
|
||||||
|
|||||||
@@ -39,8 +39,14 @@ const RETRY_BASE_DELAY_MS = 1000;
|
|||||||
/** Background poll interval for checking mission health (ms). */
|
/** Background poll interval for checking mission health (ms). */
|
||||||
const POLL_INTERVAL_MS = 60_000;
|
const POLL_INTERVAL_MS = 60_000;
|
||||||
|
|
||||||
/** Time after which a mission is considered stale (5 minutes). */
|
/** Default time after which a mission activation is considered stale (10 minutes). */
|
||||||
const STALE_THRESHOLD_MS = 5 * 60 * 1000;
|
const DEFAULT_STALE_THRESHOLD_MS = 10 * 60 * 1000;
|
||||||
|
|
||||||
|
/** Default per-task retry budget before a feature is marked blocked. */
|
||||||
|
const DEFAULT_MAX_TASK_RETRIES = 3;
|
||||||
|
|
||||||
|
/** Default cadence for mission consistency sweeps (5 minutes). */
|
||||||
|
const DEFAULT_HEALTH_CHECK_INTERVAL_MS = 5 * 60 * 1000;
|
||||||
|
|
||||||
/** Per-mission tracking state. */
|
/** Per-mission tracking state. */
|
||||||
interface WatchedMissionState {
|
interface WatchedMissionState {
|
||||||
@@ -65,8 +71,10 @@ export interface MissionAutopilotOptions {
|
|||||||
*/
|
*/
|
||||||
export class MissionAutopilot {
|
export class MissionAutopilot {
|
||||||
private watchedMissions = new Map<string, WatchedMissionState>();
|
private watchedMissions = new Map<string, WatchedMissionState>();
|
||||||
|
private perMissionTaskRetries = new Map<string, Map<string, number>>();
|
||||||
private running = false;
|
private running = false;
|
||||||
private pollTimer: ReturnType<typeof setInterval> | null = null;
|
private pollTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
|
private healthCheckTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
private scheduler: MissionAutopilotOptions["scheduler"];
|
private scheduler: MissionAutopilotOptions["scheduler"];
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@@ -95,7 +103,12 @@ export class MissionAutopilot {
|
|||||||
start(): void {
|
start(): void {
|
||||||
if (this.running) return;
|
if (this.running) return;
|
||||||
this.running = true;
|
this.running = true;
|
||||||
this.pollTimer = setInterval(() => this.poll(), POLL_INTERVAL_MS);
|
this.pollTimer = setInterval(() => {
|
||||||
|
void this.poll().catch((err) => {
|
||||||
|
autopilotLog.error("Error during autopilot poll:", err);
|
||||||
|
});
|
||||||
|
}, POLL_INTERVAL_MS);
|
||||||
|
void this.startHealthCheck();
|
||||||
autopilotLog.log("Started");
|
autopilotLog.log("Started");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -111,6 +124,7 @@ export class MissionAutopilot {
|
|||||||
clearInterval(this.pollTimer);
|
clearInterval(this.pollTimer);
|
||||||
this.pollTimer = null;
|
this.pollTimer = null;
|
||||||
}
|
}
|
||||||
|
this.stopHealthCheck();
|
||||||
|
|
||||||
// Unwatch all missions
|
// Unwatch all missions
|
||||||
for (const [missionId] of this.watchedMissions) {
|
for (const [missionId] of this.watchedMissions) {
|
||||||
@@ -121,6 +135,7 @@ export class MissionAutopilot {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.watchedMissions.clear();
|
this.watchedMissions.clear();
|
||||||
|
this.perMissionTaskRetries.clear();
|
||||||
autopilotLog.log("Stopped");
|
autopilotLog.log("Stopped");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -176,6 +191,7 @@ export class MissionAutopilot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
this.watchedMissions.delete(missionId);
|
this.watchedMissions.delete(missionId);
|
||||||
|
this.perMissionTaskRetries.delete(missionId);
|
||||||
try {
|
try {
|
||||||
this.setAutopilotState(missionId, "inactive");
|
this.setAutopilotState(missionId, "inactive");
|
||||||
} catch {
|
} catch {
|
||||||
@@ -252,6 +268,9 @@ export class MissionAutopilot {
|
|||||||
// Only proceed if we're watching this mission
|
// Only proceed if we're watching this mission
|
||||||
if (!this.isWatching(missionId)) return;
|
if (!this.isWatching(missionId)) return;
|
||||||
|
|
||||||
|
// Successful completion resets retry budget for this specific task.
|
||||||
|
this.perMissionTaskRetries.get(missionId)?.delete(taskId);
|
||||||
|
|
||||||
// Check if all features in the slice are done
|
// Check if all features in the slice are done
|
||||||
const features = this.missionStore.listFeatures(slice.id);
|
const features = this.missionStore.listFeatures(slice.id);
|
||||||
const allDone = features.length > 0 && features.every((f) => f.status === "done");
|
const allDone = features.length > 0 && features.every((f) => f.status === "done");
|
||||||
@@ -265,6 +284,72 @@ export class MissionAutopilot {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called when a mission-linked task fails execution.
|
||||||
|
* Applies retry budgets per mission/task and blocks features that exceed the budget.
|
||||||
|
*/
|
||||||
|
async handleTaskFailure(taskId: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
const feature = this.missionStore.getFeatureByTaskId(taskId);
|
||||||
|
if (!feature) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const slice = this.missionStore.getSlice(feature.sliceId);
|
||||||
|
if (!slice) {
|
||||||
|
autopilotLog.warn(`Task failure ${taskId}: slice ${feature.sliceId} not found`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const milestone = this.missionStore.getMilestone(slice.milestoneId);
|
||||||
|
if (!milestone) {
|
||||||
|
autopilotLog.warn(`Task failure ${taskId}: milestone ${slice.milestoneId} not found`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const missionId = milestone.missionId;
|
||||||
|
if (!this.isWatching(missionId)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const settings = await this.taskStore.getSettings();
|
||||||
|
const maxRetries = settings.missionMaxTaskRetries ?? DEFAULT_MAX_TASK_RETRIES;
|
||||||
|
const missionRetries = this.perMissionTaskRetries.get(missionId) ?? new Map<string, number>();
|
||||||
|
this.perMissionTaskRetries.set(missionId, missionRetries);
|
||||||
|
|
||||||
|
const retryCount = (missionRetries.get(taskId) ?? 0) + 1;
|
||||||
|
missionRetries.set(taskId, retryCount);
|
||||||
|
|
||||||
|
if (retryCount > maxRetries) {
|
||||||
|
this.missionStore.updateFeatureStatus(feature.id, "blocked");
|
||||||
|
await this.taskStore.updateTask(taskId, { status: "failed", paused: true });
|
||||||
|
this.logMissionEventSafe(
|
||||||
|
missionId,
|
||||||
|
"error",
|
||||||
|
`Feature ${feature.id} blocked after max retries (${retryCount}/${maxRetries})`,
|
||||||
|
{ taskId, featureId: feature.id, retryCount, maxRetries },
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logMissionEventSafe(
|
||||||
|
missionId,
|
||||||
|
"autopilot_retry",
|
||||||
|
`Retrying failed mission task ${taskId} (${retryCount}/${maxRetries})`,
|
||||||
|
{ taskId, featureId: feature.id, retryCount, maxRetries },
|
||||||
|
);
|
||||||
|
|
||||||
|
const task = await this.taskStore.getTask(taskId);
|
||||||
|
if (task?.column !== "todo") {
|
||||||
|
await this.taskStore.moveTask(taskId, "todo");
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.taskStore.updateTask(taskId, { error: null, status: null, paused: false });
|
||||||
|
} catch (err) {
|
||||||
|
autopilotLog.error(`Error handling task failure for ${taskId}:`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Activate the next pending slice in a mission.
|
* Activate the next pending slice in a mission.
|
||||||
* Uses the scheduler's `activateNextPendingSlice()` method.
|
* Uses the scheduler's `activateNextPendingSlice()` method.
|
||||||
@@ -398,6 +483,7 @@ export class MissionAutopilot {
|
|||||||
this.updateActivity(missionId);
|
this.updateActivity(missionId);
|
||||||
this.setAutopilotState(missionId, "inactive");
|
this.setAutopilotState(missionId, "inactive");
|
||||||
this.watchedMissions.delete(missionId);
|
this.watchedMissions.delete(missionId);
|
||||||
|
this.perMissionTaskRetries.delete(missionId);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -410,9 +496,9 @@ export class MissionAutopilot {
|
|||||||
* Periodic health check for watched missions.
|
* Periodic health check for watched missions.
|
||||||
* - Re-watches missions with `autopilotEnabled: true` that aren't being tracked
|
* - Re-watches missions with `autopilotEnabled: true` that aren't being tracked
|
||||||
* - Starts missions in `planning` with autopilot enabled
|
* - Starts missions in `planning` with autopilot enabled
|
||||||
* - Flags stale missions
|
* - Recovers stale missions stuck in `activating`
|
||||||
*/
|
*/
|
||||||
private poll(): void {
|
private async poll(): Promise<void> {
|
||||||
if (!this.running) return;
|
if (!this.running) return;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -427,10 +513,13 @@ export class MissionAutopilot {
|
|||||||
|
|
||||||
// Start planning missions with autopilot
|
// Start planning missions with autopilot
|
||||||
if (mission.autopilotEnabled && mission.status === "planning" && this.isWatching(mission.id)) {
|
if (mission.autopilotEnabled && mission.status === "planning" && this.isWatching(mission.id)) {
|
||||||
void this.checkAndStartMission(mission.id);
|
await this.checkAndStartMission(mission.id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const settings = await this.taskStore.getSettings();
|
||||||
|
const staleThresholdMs = settings.missionStaleThresholdMs ?? DEFAULT_STALE_THRESHOLD_MS;
|
||||||
|
|
||||||
// Check for stale missions
|
// Check for stale missions
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
for (const [missionId, state] of this.watchedMissions) {
|
for (const [missionId, state] of this.watchedMissions) {
|
||||||
@@ -438,34 +527,273 @@ export class MissionAutopilot {
|
|||||||
if (!mission) {
|
if (!mission) {
|
||||||
// Mission deleted — unwatch
|
// Mission deleted — unwatch
|
||||||
this.watchedMissions.delete(missionId);
|
this.watchedMissions.delete(missionId);
|
||||||
|
this.perMissionTaskRetries.delete(missionId);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (mission.lastAutopilotActivityAt) {
|
if (!mission.lastAutopilotActivityAt || mission.autopilotState !== "activating") {
|
||||||
const lastActivity = new Date(mission.lastAutopilotActivityAt).getTime();
|
continue;
|
||||||
if (now - lastActivity > STALE_THRESHOLD_MS) {
|
|
||||||
const staleMinutes = Math.round((now - lastActivity) / 60_000);
|
|
||||||
this.logMissionEventSafe(
|
|
||||||
missionId,
|
|
||||||
"warning",
|
|
||||||
`Mission autopilot appears stale (no activity for ${staleMinutes} minutes)`,
|
|
||||||
{
|
|
||||||
staleMinutes,
|
|
||||||
staleThresholdMs: STALE_THRESHOLD_MS,
|
|
||||||
lastActivityAt: mission.lastAutopilotActivityAt,
|
|
||||||
retryCount: state.retryCount,
|
|
||||||
category: "autopilot_stale",
|
|
||||||
},
|
|
||||||
);
|
|
||||||
autopilotLog.warn(`Mission ${missionId} is stale (no activity for ${staleMinutes} minutes)`);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const lastActivity = new Date(mission.lastAutopilotActivityAt).getTime();
|
||||||
|
if (now - lastActivity <= staleThresholdMs) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const staleMinutes = Math.round((now - lastActivity) / 60_000);
|
||||||
|
this.logMissionEventSafe(
|
||||||
|
missionId,
|
||||||
|
"autopilot_stale",
|
||||||
|
`Mission autopilot is stale and will be recovered (${staleMinutes} minutes inactive)` ,
|
||||||
|
{
|
||||||
|
staleMinutes,
|
||||||
|
staleThresholdMs,
|
||||||
|
lastActivityAt: mission.lastAutopilotActivityAt,
|
||||||
|
retryCount: state.retryCount,
|
||||||
|
previousState: mission.autopilotState,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
autopilotLog.warn(`Mission ${missionId} stale while activating (inactive ${staleMinutes}m) — recovering`);
|
||||||
|
|
||||||
|
this.setAutopilotState(missionId, "watching");
|
||||||
|
state.retryCount = 0;
|
||||||
|
await this.recoverStaleMission(missionId);
|
||||||
|
this.updateActivity(missionId);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
autopilotLog.error("Error during autopilot poll:", err);
|
autopilotLog.error("Error during autopilot poll:", err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attempt to recover a mission that appears stalled in the activating state.
|
||||||
|
* Re-evaluates active/pending slices and advances when progression is possible.
|
||||||
|
*/
|
||||||
|
async recoverStaleMission(missionId: string): Promise<void> {
|
||||||
|
try {
|
||||||
|
const mission = this.missionStore.getMissionWithHierarchy(missionId);
|
||||||
|
if (!mission) {
|
||||||
|
autopilotLog.warn(`recoverStaleMission: mission ${missionId} not found`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const activeSlices = mission.milestones.flatMap((milestone) => milestone.slices)
|
||||||
|
.filter((slice) => slice.status === "active");
|
||||||
|
|
||||||
|
let advanced = false;
|
||||||
|
|
||||||
|
if (activeSlices.length > 0) {
|
||||||
|
const hasCompletedActiveSlice = activeSlices.some((slice) =>
|
||||||
|
slice.features.length > 0 && slice.features.every((feature) => feature.status === "done"),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (hasCompletedActiveSlice) {
|
||||||
|
await this.advanceToNextSlice(missionId);
|
||||||
|
advanced = true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const hasPendingSlice = mission.milestones.some((milestone) =>
|
||||||
|
milestone.slices.some((slice) => slice.status === "pending"),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (hasPendingSlice) {
|
||||||
|
await this.advanceToNextSlice(missionId);
|
||||||
|
advanced = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logMissionEventSafe(
|
||||||
|
missionId,
|
||||||
|
"autopilot_stale",
|
||||||
|
advanced
|
||||||
|
? `Recovered stale mission ${missionId} and resumed slice progression`
|
||||||
|
: `Recovered stale mission ${missionId}; no immediate slice progression needed`,
|
||||||
|
{
|
||||||
|
source: "recoverStaleMission",
|
||||||
|
activeSliceCount: activeSlices.length,
|
||||||
|
advanced,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
autopilotLog.error(`recoverStaleMission failed for ${missionId}:`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async startHealthCheck(): Promise<void> {
|
||||||
|
this.stopHealthCheck();
|
||||||
|
|
||||||
|
let intervalMs = DEFAULT_HEALTH_CHECK_INTERVAL_MS;
|
||||||
|
try {
|
||||||
|
const settings = await this.taskStore.getSettings();
|
||||||
|
intervalMs = settings.missionHealthCheckIntervalMs ?? DEFAULT_HEALTH_CHECK_INTERVAL_MS;
|
||||||
|
} catch (err) {
|
||||||
|
autopilotLog.warn("Failed to read mission health check settings; using defaults", err);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this.running) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (intervalMs <= 0) {
|
||||||
|
autopilotLog.log("Mission health checks disabled (missionHealthCheckIntervalMs=0)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.healthCheckTimer = setInterval(() => {
|
||||||
|
void this.runHealthCheck();
|
||||||
|
}, intervalMs);
|
||||||
|
autopilotLog.log(`Mission health checks started (every ${intervalMs}ms)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private stopHealthCheck(): void {
|
||||||
|
if (!this.healthCheckTimer) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
clearInterval(this.healthCheckTimer);
|
||||||
|
this.healthCheckTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runHealthCheck(): Promise<void> {
|
||||||
|
if (!this.running || this.watchedMissions.size === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
let fixedCount = 0;
|
||||||
|
|
||||||
|
for (const missionId of this.watchedMissions.keys()) {
|
||||||
|
const mission = this.missionStore.getMissionWithHierarchy(missionId);
|
||||||
|
if (!mission) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
fixedCount += await this.reconcileMissionConsistency(mission);
|
||||||
|
}
|
||||||
|
|
||||||
|
autopilotLog.log(`Mission health check complete: fixed ${fixedCount} inconsistenc${fixedCount === 1 ? "y" : "ies"}`);
|
||||||
|
} catch (err) {
|
||||||
|
autopilotLog.error("Mission health check failed:", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recover autopilot state after process restart.
|
||||||
|
* Watches active missions and performs a one-time consistency sweep.
|
||||||
|
*/
|
||||||
|
async recoverMissions(missionStore: MissionStore): Promise<void> {
|
||||||
|
try {
|
||||||
|
const missions = missionStore.listMissions();
|
||||||
|
let watchedCount = 0;
|
||||||
|
let recoveredActivatingCount = 0;
|
||||||
|
let inconsistencyFixes = 0;
|
||||||
|
|
||||||
|
for (const mission of missions) {
|
||||||
|
if (!mission.autopilotEnabled || mission.status === "complete" || mission.status === "archived") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this.isWatching(mission.id)) {
|
||||||
|
this.watchMission(mission.id);
|
||||||
|
watchedCount++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (mission.autopilotState === "activating") {
|
||||||
|
await this.recoverStaleMission(mission.id);
|
||||||
|
recoveredActivatingCount++;
|
||||||
|
}
|
||||||
|
|
||||||
|
const hierarchy = missionStore.getMissionWithHierarchy(mission.id);
|
||||||
|
if (!hierarchy) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
inconsistencyFixes += await this.reconcileMissionConsistency(hierarchy);
|
||||||
|
|
||||||
|
const refreshedHierarchy = missionStore.getMissionWithHierarchy(mission.id);
|
||||||
|
if (!refreshedHierarchy) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const hasCompletedActiveSlice = refreshedHierarchy.milestones
|
||||||
|
.flatMap((milestone) => milestone.slices)
|
||||||
|
.filter((slice) => slice.status === "active")
|
||||||
|
.some((slice) => slice.features.length > 0 && slice.features.every((feature) => feature.status === "done"));
|
||||||
|
|
||||||
|
if (hasCompletedActiveSlice) {
|
||||||
|
await this.advanceToNextSlice(mission.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
autopilotLog.log(
|
||||||
|
`Mission recovery complete: watched ${watchedCount}, recovered ${recoveredActivatingCount} activating missions, fixed ${inconsistencyFixes} inconsistenc${inconsistencyFixes === 1 ? "y" : "ies"}`,
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
autopilotLog.error("Mission recovery failed:", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async reconcileMissionConsistency(
|
||||||
|
mission: ReturnType<MissionStore["getMissionWithHierarchy"]>,
|
||||||
|
): Promise<number> {
|
||||||
|
if (!mission) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const activeSlices = mission.milestones
|
||||||
|
.flatMap((milestone) => milestone.slices)
|
||||||
|
.filter((slice) => slice.status === "active");
|
||||||
|
if (activeSlices.length === 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let fixedCount = 0;
|
||||||
|
|
||||||
|
for (const slice of activeSlices) {
|
||||||
|
for (const feature of slice.features) {
|
||||||
|
if (!feature.taskId) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const task = await this.taskStore.getTask(feature.taskId);
|
||||||
|
if (!task) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (task.status === "failed" && feature.status === "in-progress") {
|
||||||
|
await this.handleTaskFailure(feature.taskId);
|
||||||
|
fixedCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (task.column === "done" && feature.status !== "done") {
|
||||||
|
this.missionStore.updateFeatureStatus(feature.id, "done");
|
||||||
|
fixedCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
task.column === "in-progress"
|
||||||
|
&& (feature.status === "triaged" || feature.status === "defined")
|
||||||
|
) {
|
||||||
|
this.missionStore.updateFeatureStatus(feature.id, "in-progress");
|
||||||
|
fixedCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
(task.column === "triage" || task.column === "todo")
|
||||||
|
&& feature.status === "in-progress"
|
||||||
|
) {
|
||||||
|
this.missionStore.updateFeatureStatus(feature.id, "triaged");
|
||||||
|
fixedCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fixedCount;
|
||||||
|
}
|
||||||
|
|
||||||
// ── Helpers ────────────────────────────────────────────────────────
|
// ── Helpers ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -477,8 +805,25 @@ export class MissionAutopilot {
|
|||||||
description: string,
|
description: string,
|
||||||
metadata?: Record<string, unknown>,
|
metadata?: Record<string, unknown>,
|
||||||
): void {
|
): void {
|
||||||
|
const missionStoreWithEvents = this.missionStore as MissionStore & {
|
||||||
|
logMissionEvent?: (
|
||||||
|
missionId: string,
|
||||||
|
eventType: MissionEventType,
|
||||||
|
description: string,
|
||||||
|
metadata?: Record<string, unknown>,
|
||||||
|
) => unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (typeof missionStoreWithEvents.logMissionEvent !== "function") {
|
||||||
|
autopilotLog.warn(
|
||||||
|
`[${eventType}] ${missionId}: ${description}`,
|
||||||
|
metadata ?? {},
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
this.missionStore.logMissionEvent(missionId, eventType, description, metadata);
|
missionStoreWithEvents.logMissionEvent(missionId, eventType, description, metadata);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
autopilotLog.error(
|
autopilotLog.error(
|
||||||
`Failed to persist mission event (${eventType}) for ${missionId}:`,
|
`Failed to persist mission event (${eventType}) for ${missionId}:`,
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import { runtimeLog } from "../logger.js";
|
|||||||
import type { StuckTaskDetector } from "../stuck-task-detector.js";
|
import type { StuckTaskDetector } from "../stuck-task-detector.js";
|
||||||
import type { UsageLimitPauser } from "../usage-limit-detector.js";
|
import type { UsageLimitPauser } from "../usage-limit-detector.js";
|
||||||
import { SelfHealingManager } from "../self-healing.js";
|
import { SelfHealingManager } from "../self-healing.js";
|
||||||
|
import { MissionAutopilot } from "../mission-autopilot.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* InProcessRuntime runs a project within the main process.
|
* InProcessRuntime runs a project within the main process.
|
||||||
@@ -137,11 +138,21 @@ export class InProcessRuntime
|
|||||||
|
|
||||||
// 4. Initialize Scheduler
|
// 4. Initialize Scheduler
|
||||||
const missionStore = this.taskStore.getMissionStore();
|
const missionStore = this.taskStore.getMissionStore();
|
||||||
|
const missionAutopilot = missionStore
|
||||||
|
? new MissionAutopilot(this.taskStore, missionStore)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
this.scheduler = new Scheduler(this.taskStore, {
|
this.scheduler = new Scheduler(this.taskStore, {
|
||||||
maxConcurrent: this.config.maxConcurrent,
|
maxConcurrent: this.config.maxConcurrent,
|
||||||
maxWorktrees: this.config.maxWorktrees,
|
maxWorktrees: this.config.maxWorktrees,
|
||||||
semaphore: this.globalSemaphore,
|
semaphore: this.globalSemaphore,
|
||||||
missionStore,
|
missionStore,
|
||||||
|
missionAutopilot,
|
||||||
|
onTaskFailed: (taskId) => {
|
||||||
|
if (missionAutopilot) {
|
||||||
|
void missionAutopilot.handleTaskFailure(taskId);
|
||||||
|
}
|
||||||
|
},
|
||||||
onSchedule: (task) => {
|
onSchedule: (task) => {
|
||||||
this.recordActivity();
|
this.recordActivity();
|
||||||
runtimeLog.log(`Scheduled task ${task.id}`);
|
runtimeLog.log(`Scheduled task ${task.id}`);
|
||||||
@@ -193,6 +204,22 @@ export class InProcessRuntime
|
|||||||
this.recordActivity();
|
this.recordActivity();
|
||||||
runtimeLog.error(`Task ${task.id} failed:`, error.message);
|
runtimeLog.error(`Task ${task.id} failed:`, error.message);
|
||||||
this.recordTaskCompletion(task.id, false);
|
this.recordTaskCompletion(task.id, false);
|
||||||
|
|
||||||
|
// Mission-linked failures should be re-queued to todo so autopilot retry
|
||||||
|
// policies can decide whether to retry or block the feature.
|
||||||
|
if (task.sliceId) {
|
||||||
|
void (async () => {
|
||||||
|
try {
|
||||||
|
const latest = await this.taskStore.getTask(task.id);
|
||||||
|
if (latest?.column === "in-progress") {
|
||||||
|
await this.taskStore.moveTask(task.id, "todo");
|
||||||
|
}
|
||||||
|
} catch (moveErr) {
|
||||||
|
runtimeLog.warn(`Failed to requeue mission task ${task.id} after error:`, moveErr);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
|
||||||
// Update agent state to terminated (failed)
|
// Update agent state to terminated (failed)
|
||||||
const agentId = this.taskAgentMap.get(task.id);
|
const agentId = this.taskAgentMap.get(task.id);
|
||||||
if (agentId && this.agentStore) {
|
if (agentId && this.agentStore) {
|
||||||
@@ -288,6 +315,13 @@ export class InProcessRuntime
|
|||||||
// 10. Start scheduler
|
// 10. Start scheduler
|
||||||
this.scheduler.start();
|
this.scheduler.start();
|
||||||
|
|
||||||
|
// Mission crash recovery: restore autopilot state for missions that were active before crash
|
||||||
|
const activeMissionStore = this.taskStore.getMissionStore();
|
||||||
|
const activeMissionAutopilot = this.scheduler.getMissionAutopilot?.();
|
||||||
|
if (activeMissionStore && activeMissionAutopilot) {
|
||||||
|
void activeMissionAutopilot.recoverMissions(activeMissionStore);
|
||||||
|
}
|
||||||
|
|
||||||
this.setStatus("active");
|
this.setStatus("active");
|
||||||
runtimeLog.log(`InProcessRuntime started for project ${this.config.projectId}`);
|
runtimeLog.log(`InProcessRuntime started for project ${this.config.projectId}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -59,6 +59,8 @@ export interface SchedulerOptions {
|
|||||||
onSchedule?: (task: Task) => void;
|
onSchedule?: (task: Task) => void;
|
||||||
/** Called when a task is blocked by deps */
|
/** Called when a task is blocked by deps */
|
||||||
onBlocked?: (task: Task, blockedBy: string[]) => void;
|
onBlocked?: (task: Task, blockedBy: string[]) => void;
|
||||||
|
/** Called when a mission-linked task fails and is queued for retry handling. */
|
||||||
|
onTaskFailed?: (taskId: string) => void | Promise<void>;
|
||||||
/** Optional PR monitor for tracking in-review PRs */
|
/** Optional PR monitor for tracking in-review PRs */
|
||||||
prMonitor?: PrMonitor;
|
prMonitor?: PrMonitor;
|
||||||
/** Optional MissionStore for slice activation and auto-advance */
|
/** Optional MissionStore for slice activation and auto-advance */
|
||||||
@@ -105,6 +107,8 @@ export class Scheduler {
|
|||||||
private activePollMs: number | null = null;
|
private activePollMs: number | null = null;
|
||||||
/** Tracks which task IDs are currently paused, to detect unpause transitions. */
|
/** Tracks which task IDs are currently paused, to detect unpause transitions. */
|
||||||
private pausedTaskIds = new Set<string>();
|
private pausedTaskIds = new Set<string>();
|
||||||
|
/** Tracks mission-linked tasks observed with status=failed before moveTask clears status/error. */
|
||||||
|
private failedTaskIds = new Set<string>();
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private store: TaskStore,
|
private store: TaskStore,
|
||||||
@@ -194,6 +198,17 @@ export class Scheduler {
|
|||||||
void this.handleMissionTaskCompletion(task.id, task.sliceId);
|
void this.handleMissionTaskCompletion(task.id, task.sliceId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Mission failure tracking: status/error are cleared during moveTask(in-progress → todo),
|
||||||
|
// so we pair this with failedTaskIds captured from task:updated events.
|
||||||
|
if (task.sliceId && to === "todo" && this.options.onTaskFailed) {
|
||||||
|
if (task.status === "failed" || this.failedTaskIds.has(task.id)) {
|
||||||
|
this.failedTaskIds.delete(task.id);
|
||||||
|
void Promise.resolve(this.options.onTaskFailed(task.id)).catch((err) => {
|
||||||
|
schedulerLog.error(`Error in onTaskFailed for ${task.id}:`, err);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Event-driven scheduling: when a task moves to "done" (completion) or "todo" (retry/manual move),
|
// Event-driven scheduling: when a task moves to "done" (completion) or "todo" (retry/manual move),
|
||||||
// trigger scheduling immediately so waiting tasks can start without waiting
|
// trigger scheduling immediately so waiting tasks can start without waiting
|
||||||
// for the next poll interval (up to 15 seconds).
|
// for the next poll interval (up to 15 seconds).
|
||||||
@@ -208,6 +223,13 @@ export class Scheduler {
|
|||||||
* Also detects task-level unpause transitions and triggers immediate scheduling.
|
* Also detects task-level unpause transitions and triggers immediate scheduling.
|
||||||
*/
|
*/
|
||||||
this.store.on("task:updated", (task) => {
|
this.store.on("task:updated", (task) => {
|
||||||
|
// Track mission failure signals before moveTask clears failure metadata.
|
||||||
|
if (task.sliceId && task.column === "in-progress" && task.status === "failed") {
|
||||||
|
this.failedTaskIds.add(task.id);
|
||||||
|
} else if (task.status !== "failed") {
|
||||||
|
this.failedTaskIds.delete(task.id);
|
||||||
|
}
|
||||||
|
|
||||||
// Track pause state transitions for event-driven scheduling on unpause.
|
// Track pause state transitions for event-driven scheduling on unpause.
|
||||||
// When a previously-paused task is unpaused in a schedulable column,
|
// When a previously-paused task is unpaused in a schedulable column,
|
||||||
// trigger a scheduling pass immediately instead of waiting for the next
|
// trigger a scheduling pass immediately instead of waiting for the next
|
||||||
@@ -313,6 +335,7 @@ export class Scheduler {
|
|||||||
if (this.options.missionAutopilot) {
|
if (this.options.missionAutopilot) {
|
||||||
this.options.missionAutopilot.stop();
|
this.options.missionAutopilot.stop();
|
||||||
}
|
}
|
||||||
|
this.failedTaskIds.clear();
|
||||||
schedulerLog.log("Stopped");
|
schedulerLog.log("Stopped");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,6 +355,10 @@ export class Scheduler {
|
|||||||
schedulerLog.log(`Poll interval updated to ${newIntervalMs}ms`);
|
schedulerLog.log(`Poll interval updated to ${newIntervalMs}ms`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getMissionAutopilot(): import("./mission-autopilot.js").MissionAutopilot | undefined {
|
||||||
|
return this.options.missionAutopilot;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolve the base branch for a task being started.
|
* Resolve the base branch for a task being started.
|
||||||
*
|
*
|
||||||
|
|||||||
Reference in New Issue
Block a user