FN-5901: reap stale mission validator runs
Add self-healing recovery for stale mission validator runs left behind after execution ownership disappears. - add mission-store support to find and reap stale running validator runs, preserving terminal error status and resetting eligible features to needs_fix - teach the mission execution loop and self-healing maintenance sweep to skip live validations, reap abandoned runs, record audit events, and avoid double-completing runs - extend regression coverage, mission docs, architecture notes, and add a published-package changeset for the new recovery behavior Files changed: .changeset/fn-5901-validator-run-reaper.md | 7 + AGENTS.md | 1 + docs/architecture.md | 2 + docs/missions.md | 26 ++- packages/core/src/__tests__/mission-store.test.ts | 99 +++++++++ packages/core/src/mission-store.ts | 91 ++++++++ packages/engine/src/__tests__/mission-execution-loop.test.ts | 232 +++++++++++++++++++++ packages/engine/src/__tests__/reliability-interactions/mission-validator-run-reaper.test.ts | 181 ++++++++++++++++ packages/engine/src/mission-execution-loop.ts | 102 +++++++-- packages/engine/src/runtimes/in-process-runtime.ts | 8 +- packages/engine/src/self-healing.ts | 22 ++ 11 files changed, 746 insertions(+), 25 deletions(-) Fusion-Task-Id: FN-5901 Fusion-Task-Lineage: 87eb2f3f-fc31-4e0a-b0fc-b771f6dc48a3
This commit is contained in:
gsxdsm
@@ -220,9 +220,43 @@ function createMockMissionStore() {
|
||||
validatorRuns.set(run.id, run);
|
||||
return run;
|
||||
}),
|
||||
listStaleRunningValidatorRuns: vi.fn((_maxAgeMs: number) => [...validatorRuns.values()].filter((run) => run.status === "running")),
|
||||
reapValidatorRun: vi.fn((id: string, reason: string) => {
|
||||
const run = validatorRuns.get(id);
|
||||
if (!run) {
|
||||
throw new Error(`Validator run ${id} not found`);
|
||||
}
|
||||
if (run.status !== "running") {
|
||||
return run;
|
||||
}
|
||||
const updated = {
|
||||
...run,
|
||||
status: "error" as const,
|
||||
summary: reason,
|
||||
completedAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
validatorRuns.set(id, updated);
|
||||
|
||||
const feature = features.get(run.featureId);
|
||||
if (feature) {
|
||||
features.set(run.featureId, {
|
||||
...feature,
|
||||
loopState: "needs_fix",
|
||||
lastValidatorStatus: "error",
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
return updated;
|
||||
}),
|
||||
getValidatorRun: vi.fn((id: string) => validatorRuns.get(id)),
|
||||
completeValidatorRun: vi.fn((id: string, status: MissionValidatorRun["status"], summary?: string) => {
|
||||
const run = validatorRuns.get(id);
|
||||
if (!run) throw new Error(`Validator run ${id} not found`);
|
||||
if (run.status !== "running") {
|
||||
throw new Error(`Validator run ${id} is not in 'running' status`);
|
||||
}
|
||||
const updated = {
|
||||
...run,
|
||||
status,
|
||||
@@ -370,6 +404,7 @@ function createMockTaskStore() {
|
||||
missionStaleThresholdMs: 600_000,
|
||||
missionMaxTaskRetries: 3,
|
||||
}),
|
||||
recordRunAuditEvent: vi.fn(),
|
||||
on: vi.fn(),
|
||||
off: vi.fn(),
|
||||
|
||||
@@ -511,6 +546,127 @@ describe("MissionExecutionLoop", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("reapStaleValidatorRuns", () => {
|
||||
it("reaps stale runs across trigger types and records audit metadata", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-06-01T12:00:00.000Z"));
|
||||
|
||||
const mission = createMockMission({ id: "M-001" });
|
||||
missionStore._setMission(mission);
|
||||
const featureManual = createMockFeature({ id: "F-manual", taskId: "FN-manual", loopState: "validating" });
|
||||
const featureAuto = createMockFeature({ id: "F-auto", taskId: "FN-auto", loopState: "validating" });
|
||||
missionStore._setFeature(featureManual);
|
||||
missionStore._setFeature(featureAuto);
|
||||
missionStore.getMilestone = vi.fn(() => createMockMilestone({ id: "MS-001", missionId: mission.id }));
|
||||
missionStore.listStaleRunningValidatorRuns = vi.fn(() => [
|
||||
createMockValidatorRun({ id: "VR-manual", featureId: featureManual.id, triggerType: "manual", startedAt: "2026-06-01T11:40:00.000Z" }),
|
||||
createMockValidatorRun({ id: "VR-auto", featureId: featureAuto.id, triggerType: "auto", startedAt: "2026-06-01T11:50:00.000Z" }),
|
||||
]);
|
||||
missionStore.reapValidatorRun = vi.fn((id: string, reason: string) => ({
|
||||
...createMockValidatorRun({
|
||||
id,
|
||||
featureId: id === "VR-manual" ? featureManual.id : featureAuto.id,
|
||||
triggerType: id === "VR-manual" ? "manual" : "auto",
|
||||
startedAt: id === "VR-manual" ? "2026-06-01T11:40:00.000Z" : "2026-06-01T11:50:00.000Z",
|
||||
}),
|
||||
status: "error",
|
||||
summary: reason,
|
||||
completedAt: "2026-06-01T12:00:00.000Z",
|
||||
updatedAt: "2026-06-01T12:00:00.000Z",
|
||||
}));
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
|
||||
const result = await loop.reapStaleValidatorRuns(15 * 60 * 1000);
|
||||
|
||||
expect(result).toEqual({ reapedCount: 2 });
|
||||
expect(missionStore.reapValidatorRun).toHaveBeenCalledTimes(2);
|
||||
expect(taskStore.recordRunAuditEvent).toHaveBeenNthCalledWith(1, expect.objectContaining({
|
||||
agentId: "store",
|
||||
runId: "validator-run-reaper",
|
||||
domain: "database",
|
||||
mutationType: "mission:validator-run-reaped",
|
||||
target: "VR-manual",
|
||||
metadata: expect.objectContaining({
|
||||
runId: "VR-manual",
|
||||
featureId: featureManual.id,
|
||||
missionId: mission.id,
|
||||
triggerType: "manual",
|
||||
elapsedMs: 20 * 60 * 1000,
|
||||
}),
|
||||
}));
|
||||
expect(taskStore.recordRunAuditEvent).toHaveBeenNthCalledWith(2, expect.objectContaining({
|
||||
target: "VR-auto",
|
||||
metadata: expect.objectContaining({
|
||||
runId: "VR-auto",
|
||||
featureId: featureAuto.id,
|
||||
missionId: mission.id,
|
||||
triggerType: "auto",
|
||||
elapsedMs: 10 * 60 * 1000,
|
||||
}),
|
||||
}));
|
||||
});
|
||||
|
||||
it("skips stale runs still actively owned in-process", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-06-01T12:00:00.000Z"));
|
||||
|
||||
const feature = createMockFeature({ id: "F-live", taskId: "FN-live", loopState: "implementing" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.listStaleRunningValidatorRuns = vi.fn(() => [
|
||||
createMockValidatorRun({ id: "VR-live", featureId: feature.id, startedAt: "2026-06-01T11:30:00.000Z" }),
|
||||
]);
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
(loop as any).activeValidations.add(feature.id);
|
||||
|
||||
const reaped = await loop.reapStaleValidatorRuns(15 * 60 * 1000);
|
||||
|
||||
expect(reaped).toEqual({ reapedCount: 0 });
|
||||
expect(missionStore.reapValidatorRun).not.toHaveBeenCalled();
|
||||
expect(taskStore.recordRunAuditEvent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("isolates per-run reap failures", async () => {
|
||||
missionStore.listStaleRunningValidatorRuns = vi.fn(() => [
|
||||
createMockValidatorRun({ id: "VR-bad", featureId: "F-bad" }),
|
||||
createMockValidatorRun({ id: "VR-good", featureId: "F-good" }),
|
||||
]);
|
||||
missionStore.reapValidatorRun = vi.fn((id: string) => {
|
||||
if (id === "VR-bad") {
|
||||
throw new Error("boom");
|
||||
}
|
||||
return {
|
||||
...createMockValidatorRun({ id, featureId: "F-good" }),
|
||||
status: "error",
|
||||
summary: "reaped",
|
||||
completedAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
|
||||
const result = await loop.reapStaleValidatorRuns(15 * 60 * 1000);
|
||||
|
||||
expect(result).toEqual({ reapedCount: 1 });
|
||||
expect(missionStore.reapValidatorRun).toHaveBeenCalledTimes(2);
|
||||
expect(taskStore.recordRunAuditEvent).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── processTaskOutcome ───────────────────────────────────────────────────
|
||||
|
||||
describe("processTaskOutcome", () => {
|
||||
@@ -575,6 +731,39 @@ describe("MissionExecutionLoop", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("requeues needs_fix features back through validation", async () => {
|
||||
const assertions = makeAssertions(1);
|
||||
const response = JSON.stringify({
|
||||
status: "pass",
|
||||
assertions: [{ assertionId: "CA-1", passed: true, message: "OK" }],
|
||||
summary: "Recovered validation passed",
|
||||
});
|
||||
|
||||
mockSessionHolder.session.state.messages = [
|
||||
{ role: "user", content: "Validate this" },
|
||||
{ role: "assistant", content: response },
|
||||
];
|
||||
|
||||
const feature = createMockFeature({ loopState: "needs_fix", taskId: "FN-NEEDS-FIX" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue(assertions);
|
||||
taskStore._setTask({ id: "FN-NEEDS-FIX", title: "Test", description: "Implementation", log: [], column: "done" });
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-NEEDS-FIX");
|
||||
|
||||
expect(missionStore.transitionLoopState).toHaveBeenCalledWith("F-001", "implementing");
|
||||
expect(missionStore.startValidatorRun).toHaveBeenCalled();
|
||||
expect(missionStore.completeValidatorRun).toHaveBeenCalledWith(expect.any(String), "passed", "Recovered validation passed");
|
||||
});
|
||||
|
||||
it("should auto-pass if feature has no linked assertions", async () => {
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-001" });
|
||||
missionStore._setFeature(feature);
|
||||
@@ -1304,6 +1493,49 @@ describe("MissionExecutionLoop", () => {
|
||||
// Autopilot notified
|
||||
expect(notifySpy).toHaveBeenCalledWith("F-001", "passed");
|
||||
});
|
||||
|
||||
it("skips completion when the validator run was reaped mid-flight", async () => {
|
||||
const assertions = makeAssertions(1);
|
||||
const response = JSON.stringify({
|
||||
status: "pass",
|
||||
assertions: [{ assertionId: "CA-1", passed: true, message: "OK" }],
|
||||
summary: "All assertions passed",
|
||||
});
|
||||
|
||||
mockSessionHolder.session.state.messages = [
|
||||
{ role: "user", content: "Validate this" },
|
||||
{ role: "assistant", content: response },
|
||||
];
|
||||
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-REAPED", id: "F-REAPED" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue(assertions);
|
||||
taskStore._setTask({ id: "FN-REAPED", title: "Test", description: "Implementation", log: [] });
|
||||
|
||||
const originalStartValidatorRun = missionStore.startValidatorRun;
|
||||
missionStore.startValidatorRun = vi.fn((featureId: string, triggerType?: string, taskId?: string) => {
|
||||
const run = originalStartValidatorRun(featureId, triggerType, taskId);
|
||||
missionStore.reapValidatorRun(run.id, "stale");
|
||||
return run;
|
||||
});
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
const emitSpy = vi.spyOn(loop, "emit");
|
||||
loop.start();
|
||||
|
||||
await expect(loop.processTaskOutcome("FN-REAPED")).resolves.not.toThrow();
|
||||
|
||||
expect(missionStore.completeValidatorRun).not.toHaveBeenCalledWith(expect.any(String), "passed", expect.any(String));
|
||||
expect(emitSpy).toHaveBeenCalledWith(
|
||||
"validation:passed",
|
||||
expect.objectContaining({ featureId: "F-REAPED" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ── handleValidationFail ──────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { TaskStore, type MissionFeature } from "@fusion/core";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { MissionExecutionLoop } from "../../mission-execution-loop.js";
|
||||
|
||||
async function createHarness() {
|
||||
const rootDir = await mkdtemp(join(tmpdir(), "fusion-mission-validator-reaper-"));
|
||||
const taskStore = new TaskStore(rootDir, undefined, { inMemoryDb: true });
|
||||
await taskStore.init();
|
||||
const missionStore = taskStore.getMissionStore();
|
||||
|
||||
const loop = new MissionExecutionLoop({
|
||||
taskStore,
|
||||
missionStore,
|
||||
rootDir,
|
||||
});
|
||||
|
||||
vi.spyOn(loop as any, "runValidation").mockResolvedValue({
|
||||
status: "pass",
|
||||
assertions: [],
|
||||
summary: "validator passed",
|
||||
});
|
||||
|
||||
const createLinkedFeature = async (input: {
|
||||
missionTitle: string;
|
||||
missionStatus?: "active" | "complete" | "archived";
|
||||
autopilotEnabled?: boolean;
|
||||
featureTitle: string;
|
||||
taskId: string;
|
||||
taskColumn?: "done" | "archived";
|
||||
}) => {
|
||||
const mission = missionStore.createMission({
|
||||
title: input.missionTitle,
|
||||
autopilotEnabled: input.autopilotEnabled ?? true,
|
||||
});
|
||||
if (input.missionStatus && input.missionStatus !== "active") {
|
||||
missionStore.updateMission(mission.id, { status: input.missionStatus });
|
||||
}
|
||||
const milestone = missionStore.addMilestone(mission.id, { title: `${input.missionTitle} milestone` });
|
||||
const slice = missionStore.addSlice(milestone.id, { title: `${input.missionTitle} slice` });
|
||||
const feature = missionStore.addFeature(slice.id, { title: input.featureTitle });
|
||||
const task = await taskStore.createTask({
|
||||
id: input.taskId,
|
||||
title: input.featureTitle,
|
||||
description: `${input.featureTitle} task`,
|
||||
column: input.taskColumn ?? "done",
|
||||
status: input.taskColumn === "archived" ? "done" : "done",
|
||||
steps: [],
|
||||
prompt: "## File Scope\n- packages/engine/src/**\n",
|
||||
} as any);
|
||||
missionStore.linkFeatureToTask(feature.id, task.id);
|
||||
const assertion = missionStore.addContractAssertion(milestone.id, {
|
||||
title: `${input.featureTitle} assertion`,
|
||||
assertion: `Verify ${input.featureTitle}`,
|
||||
sourceFeatureId: feature.id,
|
||||
});
|
||||
missionStore.linkFeatureToAssertion(feature.id, assertion.id);
|
||||
return { mission, milestone, slice, feature: missionStore.getFeature(feature.id)!, task };
|
||||
};
|
||||
|
||||
const ageRun = (runId: string, startedAt: string) => {
|
||||
(missionStore as any).db.prepare("UPDATE mission_validator_runs SET startedAt = ?, updatedAt = ? WHERE id = ?").run(startedAt, startedAt, runId);
|
||||
};
|
||||
|
||||
return {
|
||||
rootDir,
|
||||
taskStore,
|
||||
missionStore,
|
||||
loop,
|
||||
createLinkedFeature,
|
||||
ageRun,
|
||||
cleanup: async () => {
|
||||
loop.stop();
|
||||
taskStore.close();
|
||||
await rm(rootDir, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("FN-5901 reliability: mission validator run reaper", () => {
|
||||
it("reaps stale manual + automatic validator runs, unwedges the feature, and emits audit events", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-06-01T12:00:00.000Z"));
|
||||
|
||||
const h = await createHarness();
|
||||
try {
|
||||
const wedged = await h.createLinkedFeature({
|
||||
missionTitle: "Wedged mission",
|
||||
featureTitle: "Wedged feature",
|
||||
taskId: "FN-WEDGED",
|
||||
});
|
||||
const independent = await h.createLinkedFeature({
|
||||
missionTitle: "Independent mission",
|
||||
featureTitle: "Independent feature",
|
||||
taskId: "FN-INDEPENDENT",
|
||||
});
|
||||
const archivedParent = await h.createLinkedFeature({
|
||||
missionTitle: "Archived mission",
|
||||
missionStatus: "archived",
|
||||
featureTitle: "Archived feature",
|
||||
taskId: "FN-ARCHIVED",
|
||||
});
|
||||
|
||||
const manualRun = h.missionStore.startValidatorRun(wedged.feature.id, "manual");
|
||||
const archivedAutoRun = h.missionStore.startValidatorRun(archivedParent.feature.id, "auto");
|
||||
h.ageRun(manualRun.id, "2026-05-01T12:00:00.000Z");
|
||||
h.ageRun(archivedAutoRun.id, "2026-05-01T13:00:00.000Z");
|
||||
|
||||
h.missionStore.updateFeature(archivedParent.feature.id, {
|
||||
status: "done",
|
||||
loopState: "passed",
|
||||
lastValidatorStatus: "passed",
|
||||
});
|
||||
|
||||
h.loop.start();
|
||||
|
||||
await h.loop.processTaskOutcome(wedged.task.id);
|
||||
expect(h.missionStore.getFeature(wedged.feature.id)?.status).toBe("triaged");
|
||||
expect(h.missionStore.getValidatorRun(manualRun.id)?.status).toBe("running");
|
||||
|
||||
await h.loop.processTaskOutcome(independent.task.id);
|
||||
expect(h.missionStore.getFeature(independent.feature.id)?.status).toBe("done");
|
||||
|
||||
const reapResult = await h.loop.reapStaleValidatorRuns(6 * 60 * 60 * 1000);
|
||||
expect(reapResult).toEqual({ reapedCount: 2 });
|
||||
|
||||
const reapedManualRun = h.missionStore.getValidatorRun(manualRun.id);
|
||||
expect(reapedManualRun?.status).toBe("error");
|
||||
expect(reapedManualRun?.summary).toContain("stale threshold");
|
||||
expect(h.missionStore.getFeature(wedged.feature.id)).toMatchObject({
|
||||
loopState: "needs_fix",
|
||||
lastValidatorStatus: "error",
|
||||
lastValidatorRunId: manualRun.id,
|
||||
});
|
||||
|
||||
const archivedFeatureAfterReap = h.missionStore.getFeature(archivedParent.feature.id) as MissionFeature;
|
||||
expect(h.missionStore.getValidatorRun(archivedAutoRun.id)?.status).toBe("error");
|
||||
expect(archivedFeatureAfterReap).toMatchObject({
|
||||
status: "done",
|
||||
loopState: "passed",
|
||||
lastValidatorStatus: "passed",
|
||||
lastValidatorRunId: archivedAutoRun.id,
|
||||
});
|
||||
|
||||
const auditEvents = h.taskStore.getRunAuditEvents({ mutationType: "mission:validator-run-reaped" });
|
||||
expect(auditEvents).toHaveLength(2);
|
||||
expect(auditEvents.map((event) => event.metadata?.runId)).toEqual(expect.arrayContaining([manualRun.id, archivedAutoRun.id]));
|
||||
expect(auditEvents).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
target: manualRun.id,
|
||||
metadata: expect.objectContaining({
|
||||
runId: manualRun.id,
|
||||
featureId: wedged.feature.id,
|
||||
missionId: wedged.mission.id,
|
||||
triggerType: "manual",
|
||||
elapsedMs: 31 * 24 * 60 * 60 * 1000,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
target: archivedAutoRun.id,
|
||||
metadata: expect.objectContaining({
|
||||
runId: archivedAutoRun.id,
|
||||
featureId: archivedParent.feature.id,
|
||||
missionId: archivedParent.mission.id,
|
||||
triggerType: "auto",
|
||||
elapsedMs: 30 * 24 * 60 * 60 * 1000 + 23 * 60 * 60 * 1000,
|
||||
}),
|
||||
}),
|
||||
]));
|
||||
|
||||
await h.loop.processTaskOutcome(wedged.task.id);
|
||||
expect(h.missionStore.getFeature(wedged.feature.id)?.status).toBe("done");
|
||||
expect(h.missionStore.getFeature(wedged.feature.id)?.lastValidatorStatus).toBe("passed");
|
||||
} finally {
|
||||
await h.cleanup();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user