FN-5901: reap stale mission validator runs
Add self-healing recovery for stale mission validator runs left behind after execution ownership disappears. - add mission-store support to find and reap stale running validator runs, preserving terminal error status and resetting eligible features to needs_fix - teach the mission execution loop and self-healing maintenance sweep to skip live validations, reap abandoned runs, record audit events, and avoid double-completing runs - extend regression coverage, mission docs, architecture notes, and add a published-package changeset for the new recovery behavior Files changed: .changeset/fn-5901-validator-run-reaper.md | 7 + AGENTS.md | 1 + docs/architecture.md | 2 + docs/missions.md | 26 ++- packages/core/src/__tests__/mission-store.test.ts | 99 +++++++++ packages/core/src/mission-store.ts | 91 ++++++++ packages/engine/src/__tests__/mission-execution-loop.test.ts | 232 +++++++++++++++++++++ packages/engine/src/__tests__/reliability-interactions/mission-validator-run-reaper.test.ts | 181 ++++++++++++++++ packages/engine/src/mission-execution-loop.ts | 102 +++++++-- packages/engine/src/runtimes/in-process-runtime.ts | 8 +- packages/engine/src/self-healing.ts | 22 ++ 11 files changed, 746 insertions(+), 25 deletions(-) Fusion-Task-Id: FN-5901 Fusion-Task-Lineage: 87eb2f3f-fc31-4e0a-b0fc-b771f6dc48a3
This commit is contained in:
gsxdsm
@@ -220,9 +220,43 @@ function createMockMissionStore() {
|
||||
validatorRuns.set(run.id, run);
|
||||
return run;
|
||||
}),
|
||||
listStaleRunningValidatorRuns: vi.fn((_maxAgeMs: number) => [...validatorRuns.values()].filter((run) => run.status === "running")),
|
||||
reapValidatorRun: vi.fn((id: string, reason: string) => {
|
||||
const run = validatorRuns.get(id);
|
||||
if (!run) {
|
||||
throw new Error(`Validator run ${id} not found`);
|
||||
}
|
||||
if (run.status !== "running") {
|
||||
return run;
|
||||
}
|
||||
const updated = {
|
||||
...run,
|
||||
status: "error" as const,
|
||||
summary: reason,
|
||||
completedAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
validatorRuns.set(id, updated);
|
||||
|
||||
const feature = features.get(run.featureId);
|
||||
if (feature) {
|
||||
features.set(run.featureId, {
|
||||
...feature,
|
||||
loopState: "needs_fix",
|
||||
lastValidatorStatus: "error",
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
return updated;
|
||||
}),
|
||||
getValidatorRun: vi.fn((id: string) => validatorRuns.get(id)),
|
||||
completeValidatorRun: vi.fn((id: string, status: MissionValidatorRun["status"], summary?: string) => {
|
||||
const run = validatorRuns.get(id);
|
||||
if (!run) throw new Error(`Validator run ${id} not found`);
|
||||
if (run.status !== "running") {
|
||||
throw new Error(`Validator run ${id} is not in 'running' status`);
|
||||
}
|
||||
const updated = {
|
||||
...run,
|
||||
status,
|
||||
@@ -370,6 +404,7 @@ function createMockTaskStore() {
|
||||
missionStaleThresholdMs: 600_000,
|
||||
missionMaxTaskRetries: 3,
|
||||
}),
|
||||
recordRunAuditEvent: vi.fn(),
|
||||
on: vi.fn(),
|
||||
off: vi.fn(),
|
||||
|
||||
@@ -511,6 +546,127 @@ describe("MissionExecutionLoop", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("reapStaleValidatorRuns", () => {
|
||||
it("reaps stale runs across trigger types and records audit metadata", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-06-01T12:00:00.000Z"));
|
||||
|
||||
const mission = createMockMission({ id: "M-001" });
|
||||
missionStore._setMission(mission);
|
||||
const featureManual = createMockFeature({ id: "F-manual", taskId: "FN-manual", loopState: "validating" });
|
||||
const featureAuto = createMockFeature({ id: "F-auto", taskId: "FN-auto", loopState: "validating" });
|
||||
missionStore._setFeature(featureManual);
|
||||
missionStore._setFeature(featureAuto);
|
||||
missionStore.getMilestone = vi.fn(() => createMockMilestone({ id: "MS-001", missionId: mission.id }));
|
||||
missionStore.listStaleRunningValidatorRuns = vi.fn(() => [
|
||||
createMockValidatorRun({ id: "VR-manual", featureId: featureManual.id, triggerType: "manual", startedAt: "2026-06-01T11:40:00.000Z" }),
|
||||
createMockValidatorRun({ id: "VR-auto", featureId: featureAuto.id, triggerType: "auto", startedAt: "2026-06-01T11:50:00.000Z" }),
|
||||
]);
|
||||
missionStore.reapValidatorRun = vi.fn((id: string, reason: string) => ({
|
||||
...createMockValidatorRun({
|
||||
id,
|
||||
featureId: id === "VR-manual" ? featureManual.id : featureAuto.id,
|
||||
triggerType: id === "VR-manual" ? "manual" : "auto",
|
||||
startedAt: id === "VR-manual" ? "2026-06-01T11:40:00.000Z" : "2026-06-01T11:50:00.000Z",
|
||||
}),
|
||||
status: "error",
|
||||
summary: reason,
|
||||
completedAt: "2026-06-01T12:00:00.000Z",
|
||||
updatedAt: "2026-06-01T12:00:00.000Z",
|
||||
}));
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
|
||||
const result = await loop.reapStaleValidatorRuns(15 * 60 * 1000);
|
||||
|
||||
expect(result).toEqual({ reapedCount: 2 });
|
||||
expect(missionStore.reapValidatorRun).toHaveBeenCalledTimes(2);
|
||||
expect(taskStore.recordRunAuditEvent).toHaveBeenNthCalledWith(1, expect.objectContaining({
|
||||
agentId: "store",
|
||||
runId: "validator-run-reaper",
|
||||
domain: "database",
|
||||
mutationType: "mission:validator-run-reaped",
|
||||
target: "VR-manual",
|
||||
metadata: expect.objectContaining({
|
||||
runId: "VR-manual",
|
||||
featureId: featureManual.id,
|
||||
missionId: mission.id,
|
||||
triggerType: "manual",
|
||||
elapsedMs: 20 * 60 * 1000,
|
||||
}),
|
||||
}));
|
||||
expect(taskStore.recordRunAuditEvent).toHaveBeenNthCalledWith(2, expect.objectContaining({
|
||||
target: "VR-auto",
|
||||
metadata: expect.objectContaining({
|
||||
runId: "VR-auto",
|
||||
featureId: featureAuto.id,
|
||||
missionId: mission.id,
|
||||
triggerType: "auto",
|
||||
elapsedMs: 10 * 60 * 1000,
|
||||
}),
|
||||
}));
|
||||
});
|
||||
|
||||
it("skips stale runs still actively owned in-process", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-06-01T12:00:00.000Z"));
|
||||
|
||||
const feature = createMockFeature({ id: "F-live", taskId: "FN-live", loopState: "implementing" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.listStaleRunningValidatorRuns = vi.fn(() => [
|
||||
createMockValidatorRun({ id: "VR-live", featureId: feature.id, startedAt: "2026-06-01T11:30:00.000Z" }),
|
||||
]);
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
(loop as any).activeValidations.add(feature.id);
|
||||
|
||||
const reaped = await loop.reapStaleValidatorRuns(15 * 60 * 1000);
|
||||
|
||||
expect(reaped).toEqual({ reapedCount: 0 });
|
||||
expect(missionStore.reapValidatorRun).not.toHaveBeenCalled();
|
||||
expect(taskStore.recordRunAuditEvent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("isolates per-run reap failures", async () => {
|
||||
missionStore.listStaleRunningValidatorRuns = vi.fn(() => [
|
||||
createMockValidatorRun({ id: "VR-bad", featureId: "F-bad" }),
|
||||
createMockValidatorRun({ id: "VR-good", featureId: "F-good" }),
|
||||
]);
|
||||
missionStore.reapValidatorRun = vi.fn((id: string) => {
|
||||
if (id === "VR-bad") {
|
||||
throw new Error("boom");
|
||||
}
|
||||
return {
|
||||
...createMockValidatorRun({ id, featureId: "F-good" }),
|
||||
status: "error",
|
||||
summary: "reaped",
|
||||
completedAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
|
||||
const result = await loop.reapStaleValidatorRuns(15 * 60 * 1000);
|
||||
|
||||
expect(result).toEqual({ reapedCount: 1 });
|
||||
expect(missionStore.reapValidatorRun).toHaveBeenCalledTimes(2);
|
||||
expect(taskStore.recordRunAuditEvent).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── processTaskOutcome ───────────────────────────────────────────────────
|
||||
|
||||
describe("processTaskOutcome", () => {
|
||||
@@ -575,6 +731,39 @@ describe("MissionExecutionLoop", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("requeues needs_fix features back through validation", async () => {
|
||||
const assertions = makeAssertions(1);
|
||||
const response = JSON.stringify({
|
||||
status: "pass",
|
||||
assertions: [{ assertionId: "CA-1", passed: true, message: "OK" }],
|
||||
summary: "Recovered validation passed",
|
||||
});
|
||||
|
||||
mockSessionHolder.session.state.messages = [
|
||||
{ role: "user", content: "Validate this" },
|
||||
{ role: "assistant", content: response },
|
||||
];
|
||||
|
||||
const feature = createMockFeature({ loopState: "needs_fix", taskId: "FN-NEEDS-FIX" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue(assertions);
|
||||
taskStore._setTask({ id: "FN-NEEDS-FIX", title: "Test", description: "Implementation", log: [], column: "done" });
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-NEEDS-FIX");
|
||||
|
||||
expect(missionStore.transitionLoopState).toHaveBeenCalledWith("F-001", "implementing");
|
||||
expect(missionStore.startValidatorRun).toHaveBeenCalled();
|
||||
expect(missionStore.completeValidatorRun).toHaveBeenCalledWith(expect.any(String), "passed", "Recovered validation passed");
|
||||
});
|
||||
|
||||
it("should auto-pass if feature has no linked assertions", async () => {
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-001" });
|
||||
missionStore._setFeature(feature);
|
||||
@@ -1304,6 +1493,49 @@ describe("MissionExecutionLoop", () => {
|
||||
// Autopilot notified
|
||||
expect(notifySpy).toHaveBeenCalledWith("F-001", "passed");
|
||||
});
|
||||
|
||||
it("skips completion when the validator run was reaped mid-flight", async () => {
|
||||
const assertions = makeAssertions(1);
|
||||
const response = JSON.stringify({
|
||||
status: "pass",
|
||||
assertions: [{ assertionId: "CA-1", passed: true, message: "OK" }],
|
||||
summary: "All assertions passed",
|
||||
});
|
||||
|
||||
mockSessionHolder.session.state.messages = [
|
||||
{ role: "user", content: "Validate this" },
|
||||
{ role: "assistant", content: response },
|
||||
];
|
||||
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-REAPED", id: "F-REAPED" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue(assertions);
|
||||
taskStore._setTask({ id: "FN-REAPED", title: "Test", description: "Implementation", log: [] });
|
||||
|
||||
const originalStartValidatorRun = missionStore.startValidatorRun;
|
||||
missionStore.startValidatorRun = vi.fn((featureId: string, triggerType?: string, taskId?: string) => {
|
||||
const run = originalStartValidatorRun(featureId, triggerType, taskId);
|
||||
missionStore.reapValidatorRun(run.id, "stale");
|
||||
return run;
|
||||
});
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
const emitSpy = vi.spyOn(loop, "emit");
|
||||
loop.start();
|
||||
|
||||
await expect(loop.processTaskOutcome("FN-REAPED")).resolves.not.toThrow();
|
||||
|
||||
expect(missionStore.completeValidatorRun).not.toHaveBeenCalledWith(expect.any(String), "passed", expect.any(String));
|
||||
expect(emitSpy).toHaveBeenCalledWith(
|
||||
"validation:passed",
|
||||
expect.objectContaining({ featureId: "F-REAPED" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── handleValidationFail ──────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { TaskStore, type MissionFeature } from "@fusion/core";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { MissionExecutionLoop } from "../../mission-execution-loop.js";
|
||||
|
||||
async function createHarness() {
|
||||
const rootDir = await mkdtemp(join(tmpdir(), "fusion-mission-validator-reaper-"));
|
||||
const taskStore = new TaskStore(rootDir, undefined, { inMemoryDb: true });
|
||||
await taskStore.init();
|
||||
const missionStore = taskStore.getMissionStore();
|
||||
|
||||
const loop = new MissionExecutionLoop({
|
||||
taskStore,
|
||||
missionStore,
|
||||
rootDir,
|
||||
});
|
||||
|
||||
vi.spyOn(loop as any, "runValidation").mockResolvedValue({
|
||||
status: "pass",
|
||||
assertions: [],
|
||||
summary: "validator passed",
|
||||
});
|
||||
|
||||
const createLinkedFeature = async (input: {
|
||||
missionTitle: string;
|
||||
missionStatus?: "active" | "complete" | "archived";
|
||||
autopilotEnabled?: boolean;
|
||||
featureTitle: string;
|
||||
taskId: string;
|
||||
taskColumn?: "done" | "archived";
|
||||
}) => {
|
||||
const mission = missionStore.createMission({
|
||||
title: input.missionTitle,
|
||||
autopilotEnabled: input.autopilotEnabled ?? true,
|
||||
});
|
||||
if (input.missionStatus && input.missionStatus !== "active") {
|
||||
missionStore.updateMission(mission.id, { status: input.missionStatus });
|
||||
}
|
||||
const milestone = missionStore.addMilestone(mission.id, { title: `${input.missionTitle} milestone` });
|
||||
const slice = missionStore.addSlice(milestone.id, { title: `${input.missionTitle} slice` });
|
||||
const feature = missionStore.addFeature(slice.id, { title: input.featureTitle });
|
||||
const task = await taskStore.createTask({
|
||||
id: input.taskId,
|
||||
title: input.featureTitle,
|
||||
description: `${input.featureTitle} task`,
|
||||
column: input.taskColumn ?? "done",
|
||||
status: input.taskColumn === "archived" ? "done" : "done",
|
||||
steps: [],
|
||||
prompt: "## File Scope\n- packages/engine/src/**\n",
|
||||
} as any);
|
||||
missionStore.linkFeatureToTask(feature.id, task.id);
|
||||
const assertion = missionStore.addContractAssertion(milestone.id, {
|
||||
title: `${input.featureTitle} assertion`,
|
||||
assertion: `Verify ${input.featureTitle}`,
|
||||
sourceFeatureId: feature.id,
|
||||
});
|
||||
missionStore.linkFeatureToAssertion(feature.id, assertion.id);
|
||||
return { mission, milestone, slice, feature: missionStore.getFeature(feature.id)!, task };
|
||||
};
|
||||
|
||||
const ageRun = (runId: string, startedAt: string) => {
|
||||
(missionStore as any).db.prepare("UPDATE mission_validator_runs SET startedAt = ?, updatedAt = ? WHERE id = ?").run(startedAt, startedAt, runId);
|
||||
};
|
||||
|
||||
return {
|
||||
rootDir,
|
||||
taskStore,
|
||||
missionStore,
|
||||
loop,
|
||||
createLinkedFeature,
|
||||
ageRun,
|
||||
cleanup: async () => {
|
||||
loop.stop();
|
||||
taskStore.close();
|
||||
await rm(rootDir, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("FN-5901 reliability: mission validator run reaper", () => {
|
||||
it("reaps stale manual + automatic validator runs, unwedges the feature, and emits audit events", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-06-01T12:00:00.000Z"));
|
||||
|
||||
const h = await createHarness();
|
||||
try {
|
||||
const wedged = await h.createLinkedFeature({
|
||||
missionTitle: "Wedged mission",
|
||||
featureTitle: "Wedged feature",
|
||||
taskId: "FN-WEDGED",
|
||||
});
|
||||
const independent = await h.createLinkedFeature({
|
||||
missionTitle: "Independent mission",
|
||||
featureTitle: "Independent feature",
|
||||
taskId: "FN-INDEPENDENT",
|
||||
});
|
||||
const archivedParent = await h.createLinkedFeature({
|
||||
missionTitle: "Archived mission",
|
||||
missionStatus: "archived",
|
||||
featureTitle: "Archived feature",
|
||||
taskId: "FN-ARCHIVED",
|
||||
});
|
||||
|
||||
const manualRun = h.missionStore.startValidatorRun(wedged.feature.id, "manual");
|
||||
const archivedAutoRun = h.missionStore.startValidatorRun(archivedParent.feature.id, "auto");
|
||||
h.ageRun(manualRun.id, "2026-05-01T12:00:00.000Z");
|
||||
h.ageRun(archivedAutoRun.id, "2026-05-01T13:00:00.000Z");
|
||||
|
||||
h.missionStore.updateFeature(archivedParent.feature.id, {
|
||||
status: "done",
|
||||
loopState: "passed",
|
||||
lastValidatorStatus: "passed",
|
||||
});
|
||||
|
||||
h.loop.start();
|
||||
|
||||
await h.loop.processTaskOutcome(wedged.task.id);
|
||||
expect(h.missionStore.getFeature(wedged.feature.id)?.status).toBe("triaged");
|
||||
expect(h.missionStore.getValidatorRun(manualRun.id)?.status).toBe("running");
|
||||
|
||||
await h.loop.processTaskOutcome(independent.task.id);
|
||||
expect(h.missionStore.getFeature(independent.feature.id)?.status).toBe("done");
|
||||
|
||||
const reapResult = await h.loop.reapStaleValidatorRuns(6 * 60 * 60 * 1000);
|
||||
expect(reapResult).toEqual({ reapedCount: 2 });
|
||||
|
||||
const reapedManualRun = h.missionStore.getValidatorRun(manualRun.id);
|
||||
expect(reapedManualRun?.status).toBe("error");
|
||||
expect(reapedManualRun?.summary).toContain("stale threshold");
|
||||
expect(h.missionStore.getFeature(wedged.feature.id)).toMatchObject({
|
||||
loopState: "needs_fix",
|
||||
lastValidatorStatus: "error",
|
||||
lastValidatorRunId: manualRun.id,
|
||||
});
|
||||
|
||||
const archivedFeatureAfterReap = h.missionStore.getFeature(archivedParent.feature.id) as MissionFeature;
|
||||
expect(h.missionStore.getValidatorRun(archivedAutoRun.id)?.status).toBe("error");
|
||||
expect(archivedFeatureAfterReap).toMatchObject({
|
||||
status: "done",
|
||||
loopState: "passed",
|
||||
lastValidatorStatus: "passed",
|
||||
lastValidatorRunId: archivedAutoRun.id,
|
||||
});
|
||||
|
||||
const auditEvents = h.taskStore.getRunAuditEvents({ mutationType: "mission:validator-run-reaped" });
|
||||
expect(auditEvents).toHaveLength(2);
|
||||
expect(auditEvents.map((event) => event.metadata?.runId)).toEqual(expect.arrayContaining([manualRun.id, archivedAutoRun.id]));
|
||||
expect(auditEvents).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
target: manualRun.id,
|
||||
metadata: expect.objectContaining({
|
||||
runId: manualRun.id,
|
||||
featureId: wedged.feature.id,
|
||||
missionId: wedged.mission.id,
|
||||
triggerType: "manual",
|
||||
elapsedMs: 31 * 24 * 60 * 60 * 1000,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
target: archivedAutoRun.id,
|
||||
metadata: expect.objectContaining({
|
||||
runId: archivedAutoRun.id,
|
||||
featureId: archivedParent.feature.id,
|
||||
missionId: archivedParent.mission.id,
|
||||
triggerType: "auto",
|
||||
elapsedMs: 30 * 24 * 60 * 60 * 1000 + 23 * 60 * 60 * 1000,
|
||||
}),
|
||||
}),
|
||||
]));
|
||||
|
||||
await h.loop.processTaskOutcome(wedged.task.id);
|
||||
expect(h.missionStore.getFeature(wedged.feature.id)?.status).toBe("done");
|
||||
expect(h.missionStore.getFeature(wedged.feature.id)?.lastValidatorStatus).toBe("passed");
|
||||
} finally {
|
||||
await h.cleanup();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -139,6 +139,57 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
return this.running;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reap validator runs that have been left in status='running' beyond the stale window.
|
||||
*
|
||||
* Runs still actively owned by this process are skipped so live validations are never
|
||||
* terminated by maintenance while their session is still in-flight.
|
||||
*/
|
||||
async reapStaleValidatorRuns(maxAgeMs: number): Promise<{ reapedCount: number }> {
|
||||
const staleRuns = this.missionStore.listStaleRunningValidatorRuns(maxAgeMs);
|
||||
let reapedCount = 0;
|
||||
|
||||
for (const run of staleRuns) {
|
||||
if (this.activeValidations.has(run.featureId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const reapedRun = this.missionStore.reapValidatorRun(
|
||||
run.id,
|
||||
`Validator run reaped after exceeding stale threshold (${maxAgeMs}ms) without a live owner.`,
|
||||
);
|
||||
reapedCount += 1;
|
||||
|
||||
try {
|
||||
const milestone = this.missionStore.getMilestone(reapedRun.milestoneId);
|
||||
const missionId = milestone ? this.missionStore.getMission(milestone.missionId)?.id : undefined;
|
||||
const elapsedMs = Math.max(0, Date.now() - new Date(run.startedAt).getTime());
|
||||
this.taskStore.recordRunAuditEvent({
|
||||
agentId: "store",
|
||||
runId: "validator-run-reaper",
|
||||
domain: "database",
|
||||
mutationType: "mission:validator-run-reaped",
|
||||
target: reapedRun.id,
|
||||
metadata: {
|
||||
runId: reapedRun.id,
|
||||
featureId: reapedRun.featureId,
|
||||
missionId,
|
||||
triggerType: reapedRun.triggerType,
|
||||
elapsedMs,
|
||||
},
|
||||
});
|
||||
} catch (auditErr) {
|
||||
loopLog.warn(`Failed to record validator-run reaper audit for ${run.id}:`, auditErr);
|
||||
}
|
||||
} catch (err) {
|
||||
loopLog.warn(`Failed to reap stale validator run ${run.id}:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
return { reapedCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover active missions on startup.
|
||||
*
|
||||
@@ -279,6 +330,11 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
return;
|
||||
}
|
||||
|
||||
if (feature.loopState === "needs_fix") {
|
||||
this.missionStore.transitionLoopState(feature.id, "implementing");
|
||||
feature.loopState = "implementing";
|
||||
}
|
||||
|
||||
// Only validate features in "implementing" state
|
||||
if (feature.loopState !== "implementing") {
|
||||
loopLog.log(`Feature ${feature.id} loopState is "${feature.loopState}"; skipping validation`);
|
||||
@@ -842,6 +898,30 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
private completeValidatorRunIfStillRunning(
|
||||
runId: string | undefined,
|
||||
status: "passed" | "failed" | "blocked" | "error",
|
||||
summaryOrReason?: string,
|
||||
): boolean {
|
||||
if (!runId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (typeof this.missionStore.getValidatorRun !== "function") {
|
||||
this.missionStore.completeValidatorRun(runId, status, summaryOrReason);
|
||||
return true;
|
||||
}
|
||||
|
||||
const run = this.missionStore.getValidatorRun(runId);
|
||||
if (!run || run.status !== "running") {
|
||||
loopLog.warn(`Validator run ${runId} is no longer running; skipping ${status} completion.`);
|
||||
return false;
|
||||
}
|
||||
|
||||
this.missionStore.completeValidatorRun(runId, status, summaryOrReason);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a successful validation (pass).
|
||||
*/
|
||||
@@ -851,9 +931,7 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
|
||||
summary: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
if (runId) {
|
||||
this.missionStore.completeValidatorRun(runId, "passed", summary);
|
||||
}
|
||||
this.completeValidatorRunIfStillRunning(runId, "passed", summary);
|
||||
|
||||
const feature = this.missionStore.getFeature(featureId);
|
||||
if (feature && feature.status !== "done") {
|
||||
@@ -920,13 +998,15 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
|
||||
actual: a.actual,
|
||||
}));
|
||||
|
||||
if (runId && failures.length > 0) {
|
||||
const canCompleteRun = runId
|
||||
? typeof this.missionStore.getValidatorRun !== "function" || this.missionStore.getValidatorRun(runId)?.status === "running"
|
||||
: false;
|
||||
|
||||
if (runId && failures.length > 0 && canCompleteRun) {
|
||||
this.missionStore.recordValidatorFailures(runId, failures);
|
||||
}
|
||||
|
||||
if (runId) {
|
||||
this.missionStore.completeValidatorRun(runId, "failed", result.summary);
|
||||
}
|
||||
this.completeValidatorRunIfStillRunning(runId, "failed", result.summary);
|
||||
|
||||
loopLog.log(`Feature ${featureId} failed validation with ${failures.length} failures`);
|
||||
|
||||
@@ -984,9 +1064,7 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
|
||||
blockedReason: string | undefined,
|
||||
): Promise<void> {
|
||||
try {
|
||||
if (runId) {
|
||||
this.missionStore.completeValidatorRun(runId, "blocked", blockedReason);
|
||||
}
|
||||
this.completeValidatorRunIfStillRunning(runId, "blocked", blockedReason);
|
||||
loopLog.log(`Feature ${featureId} blocked: ${blockedReason}`);
|
||||
this.logFeatureErrorEvent(featureId, "validation_blocked", `Validation blocked for feature ${featureId}: ${blockedReason ?? "no reason provided"}`, {
|
||||
runId,
|
||||
@@ -1013,9 +1091,7 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
|
||||
error: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
if (runId) {
|
||||
this.missionStore.completeValidatorRun(runId, "error", error);
|
||||
}
|
||||
this.completeValidatorRunIfStillRunning(runId, "error", error);
|
||||
loopLog.error(`Feature ${featureId} validation error: ${error}`);
|
||||
this.logFeatureErrorEvent(featureId, "validation_error", `Validation error for feature ${featureId}: ${error}`, {
|
||||
runId,
|
||||
|
||||
@@ -34,7 +34,7 @@ import type {
|
||||
import { runtimeLog } from "../logger.js";
|
||||
import { StuckTaskDetector } from "../stuck-task-detector.js";
|
||||
import type { UsageLimitPauser } from "../usage-limit-detector.js";
|
||||
import { SelfHealingManager } from "../self-healing.js";
|
||||
import { SelfHealingManager, VALIDATOR_RUN_STALE_MAX_AGE_MS } from "../self-healing.js";
|
||||
import { RestartRecoveryCoordinator } from "../restart-recovery-coordinator.js";
|
||||
import { MeshLeaseManager } from "../mesh-lease-manager.js";
|
||||
import { PluginRunner } from "../plugin-runner.js";
|
||||
@@ -738,6 +738,12 @@ export class InProcessRuntime
|
||||
}
|
||||
return this.missionExecutionLoop.recoverActiveMissions();
|
||||
},
|
||||
reapStaleMissionValidatorRuns: async () => {
|
||||
if (!this.missionExecutionLoop) {
|
||||
return { reapedCount: 0 };
|
||||
}
|
||||
return this.missionExecutionLoop.reapStaleValidatorRuns(VALIDATOR_RUN_STALE_MAX_AGE_MS);
|
||||
},
|
||||
reconcileAllMissionFeatures: async () => this.scheduler.reconcileAllMissionFeatures(),
|
||||
chatStore: this.chatStore,
|
||||
messageStore: this.messageStore,
|
||||
|
||||
@@ -289,6 +289,8 @@ export interface SelfHealingOptions {
|
||||
reconcileAllMissionFeatures?: () => Promise<number>;
|
||||
/** Optional callback to re-run mission validation recovery during maintenance. */
|
||||
recoverActiveMissionValidations?: () => Promise<{ recoveredCount: number }>;
|
||||
/** Optional callback to reap stale mission validator runs during startup and maintenance. */
|
||||
reapStaleMissionValidatorRuns?: () => Promise<{ reapedCount: number }>;
|
||||
}
|
||||
|
||||
const APPROVED_TRIAGE_RECOVERY_GRACE_MS = 60_000;
|
||||
@@ -296,6 +298,7 @@ const STARVED_REFINEMENT_RECOVERY_GRACE_MS = 10 * 60_000;
|
||||
const STARVED_PEER_PROGRESS_THRESHOLD = 3;
|
||||
const STARVED_REFINEMENT_ESCALATION_COOLDOWN_MS = STARVED_REFINEMENT_RECOVERY_GRACE_MS * 4;
|
||||
const ORPHANED_EXECUTION_RECOVERY_GRACE_MS = 60_000;
|
||||
export const VALIDATOR_RUN_STALE_MAX_AGE_MS = 6 * 60 * 60 * 1000;
|
||||
const ACTIVE_MERGE_STATUSES = new Set(["merging", "merging-pr", "merging-fix"]);
|
||||
const NON_TERMINAL_STEP_STATUSES = new Set(["pending", "in-progress"]);
|
||||
const STRANDED_COMPLETED_TODO_ACTIVE_STATUSES = new Set([
|
||||
@@ -854,6 +857,16 @@ export class SelfHealingManager {
|
||||
{ name: "orphaned-planning", fn: () => this.recoverOrphanedPlanningTasks().then(() => undefined) },
|
||||
{ name: "recover-orphaned-agents", fn: () => this.recoverOrphanedAgents().then(() => undefined) },
|
||||
{ name: "recover-stale-heartbeat-runs", fn: () => this.recoverStaleHeartbeatRuns().then(() => undefined) },
|
||||
{
|
||||
name: "reap-stale-mission-validator-runs",
|
||||
fn: async () => {
|
||||
if (!this.options.reapStaleMissionValidatorRuns) {
|
||||
return undefined;
|
||||
}
|
||||
await this.options.reapStaleMissionValidatorRuns();
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
{ name: "recover-running-on-inactive-tasks", fn: () => this.recoverAgentsRunningOnInactiveTasks().then(() => undefined) },
|
||||
{ name: "recover-drifted-agent-task-links", fn: () => this.recoverDriftedAgentTaskLinks().then(() => undefined) },
|
||||
{ name: "reconcile-soft-delete-column-drift", fn: () => this.reconcileSoftDeletedColumnDrift().then(() => undefined) },
|
||||
@@ -1676,6 +1689,15 @@ export class SelfHealingManager {
|
||||
await this.options.recoverActiveMissionValidations();
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "reap-stale-mission-validator-runs",
|
||||
fn: async () => {
|
||||
if (!this.options.reapStaleMissionValidatorRuns) {
|
||||
return;
|
||||
}
|
||||
await this.options.reapStaleMissionValidatorRuns();
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "reconcile-mission-features",
|
||||
fn: async () => {
|
||||
|
||||
Reference in New Issue
Block a user