root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FN-5901: reap stale mission validator runs

Browse Source 
Add self-healing recovery for stale mission validator runs left behind after execution ownership disappears.

- add mission-store support to find and reap stale running validator runs, preserving terminal error status and resetting eligible features to needs_fix
- teach the mission execution loop and self-healing maintenance sweep to skip live validations, reap abandoned runs, record audit events, and avoid double-completing runs
- extend regression coverage, mission docs, architecture notes, and add a published-package changeset for the new recovery behavior

Files changed:
 .changeset/fn-5901-validator-run-reaper.md         |   7 +
 AGENTS.md                                          |   1 +
 docs/architecture.md                               |   2 +
 docs/missions.md                                   |  26 ++-
 packages/core/src/__tests__/mission-store.test.ts  |  99 +++++++++
 packages/core/src/mission-store.ts                 |  91 ++++++++
 packages/engine/src/__tests__/mission-execution-loop.test.ts   | 232 +++++++++++++++++++++
 packages/engine/src/__tests__/reliability-interactions/mission-validator-run-reaper.test.ts           | 181 ++++++++++++++++
 packages/engine/src/mission-execution-loop.ts      | 102 +++++++--
 packages/engine/src/runtimes/in-process-runtime.ts |   8 +-
 packages/engine/src/self-healing.ts                |  22 ++
 11 files changed, 746 insertions(+), 25 deletions(-)

Fusion-Task-Id: FN-5901

Fusion-Task-Lineage: 87eb2f3f-fc31-4e0a-b0fc-b771f6dc48a3
This commit is contained in:
gsxdsm
2026-06-02 15:33:31 -07:00
parent 93e8bd9940
commit 3b9ff42073
11 changed files with 746 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
packages/core/src/__tests__/mission-store.test.ts
View File

@@ -3868,6 +3868,105 @@ describe("MissionStore", () => {
expect(retrieved!.status).toBe("running");
});
it("listStaleRunningValidatorRuns filters by age", () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-01-15T12:00:00.000Z"));
const mission = store.createMission({ title: "Stale Run Test" });
const milestone = store.addMilestone(mission.id, { title: "MS" });
const slice = store.addSlice(milestone.id, { title: "SL" });
const staleFeature = store.addFeature(slice.id, { title: "Stale Feature" });
const freshFeature = store.addFeature(slice.id, { title: "Fresh Feature" });
const staleRun = store.startValidatorRun(staleFeature.id, "manual");
vi.setSystemTime(new Date("2026-01-15T12:09:00.000Z"));
const freshRun = store.startValidatorRun(freshFeature.id, "auto");
const staleRuns = store.listStaleRunningValidatorRuns(5 * 60 * 1000, new Date("2026-01-15T12:10:00.000Z").getTime());
expect(staleRuns.map((run) => run.id)).toEqual([staleRun.id]);
expect(staleRuns.some((run) => run.id === freshRun.id)).toBe(false);
vi.useRealTimers();
});
it("reapValidatorRun transitions running run to error and unwedges live feature", () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-01-15T12:00:00.000Z"));
const mission = store.createMission({ title: "Reap Test" });
const milestone = store.addMilestone(mission.id, { title: "MS" });
const slice = store.addSlice(milestone.id, { title: "SL" });
const feature = store.addFeature(slice.id, { title: "Test Feature" });
const run = store.startValidatorRun(feature.id, "manual");
vi.setSystemTime(new Date("2026-01-15T12:06:00.000Z"));
const completedListener = vi.fn();
store.on("validator-run:completed", completedListener);
const reapedRun = store.reapValidatorRun(run.id, "stale owner");
expect(reapedRun.status).toBe("error");
expect(reapedRun.summary).toBe("stale owner");
expect(reapedRun.completedAt).toBe("2026-01-15T12:06:00.000Z");
expect(store.getFeature(feature.id)).toMatchObject({
loopState: "needs_fix",
lastValidatorStatus: "error",
lastValidatorRunId: run.id,
});
expect(completedListener).toHaveBeenCalledWith(reapedRun, "error", 360000);
store.off("validator-run:completed", completedListener);
vi.useRealTimers();
});
it("reapValidatorRun leaves completed or archived parent state untouched", () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-01-15T12:00:00.000Z"));
const completeMission = store.createMission({ title: "Complete Parent" });
const completeMilestone = store.addMilestone(completeMission.id, { title: "MS" });
const completeSlice = store.addSlice(completeMilestone.id, { title: "SL" });
const completeFeature = store.addFeature(completeSlice.id, { title: "Feature" });
const completeRun = store.startValidatorRun(completeFeature.id, "manual");
store.updateFeature(completeFeature.id, { loopState: "passed", lastValidatorStatus: "passed", status: "done" });
store.updateMission(completeMission.id, { status: "complete" });
const archivedMission = store.createMission({ title: "Archived Parent" });
const archivedMilestone = store.addMilestone(archivedMission.id, { title: "MS" });
const archivedSlice = store.addSlice(archivedMilestone.id, { title: "SL" });
const archivedFeature = store.addFeature(archivedSlice.id, { title: "Feature" });
const archivedRun = store.startValidatorRun(archivedFeature.id, "auto");
store.updateFeature(archivedFeature.id, { loopState: "blocked", lastValidatorStatus: "blocked" });
store.updateMission(archivedMission.id, { status: "archived" });
vi.setSystemTime(new Date("2026-01-15T12:08:00.000Z"));
expect(store.reapValidatorRun(completeRun.id, "complete mission stale").status).toBe("error");
expect(store.reapValidatorRun(archivedRun.id, "archived mission stale").status).toBe("error");
expect(store.getFeature(completeFeature.id)).toMatchObject({ loopState: "passed", lastValidatorStatus: "passed", lastValidatorRunId: completeRun.id });
expect(store.getFeature(archivedFeature.id)).toMatchObject({ loopState: "blocked", lastValidatorStatus: "blocked", lastValidatorRunId: archivedRun.id });
vi.useRealTimers();
});
it("reapValidatorRun is idempotent for terminal runs", () => {
const mission = store.createMission({ title: "Idempotent Reap Test" });
const milestone = store.addMilestone(mission.id, { title: "MS" });
const slice = store.addSlice(milestone.id, { title: "SL" });
const feature = store.addFeature(slice.id, { title: "Test Feature" });
const run = store.startValidatorRun(feature.id, "manual");
const reaped = store.reapValidatorRun(run.id, "first reap");
const featureAfterFirstReap = store.getFeature(feature.id);
const second = store.reapValidatorRun(run.id, "second reap");
const featureAfterSecondReap = store.getFeature(feature.id);
expect(second).toEqual(reaped);
expect(featureAfterSecondReap).toEqual(featureAfterFirstReap);
});
it("startValidatorRun emits validator-run:started event", () => {
const mission = store.createMission({ title: "Event Test" });
const milestone = store.addMilestone(mission.id, { title: "MS" });

91
packages/core/src/mission-store.ts
View File

@@ -2732,6 +2732,97 @@ export class MissionStore extends EventEmitter<MissionStoreEvents> {
return (rows as unknown as ValidatorRunRow[]).map((row) => this.rowToValidatorRun(row));
}
/**
* List validator runs that are still marked running even though their startedAt is older
* than the supplied age threshold.
*/
listStaleRunningValidatorRuns(maxAgeMs: number, now = Date.now()): MissionValidatorRun[] {
const cutoff = new Date(now - maxAgeMs).toISOString();
const rows = this.db.prepare(
"SELECT * FROM mission_validator_runs WHERE status = 'running' AND startedAt < ? ORDER BY startedAt ASC"
).all(cutoff);
return (rows as unknown as ValidatorRunRow[]).map((row) => this.rowToValidatorRun(row));
}
/**
* Reap a stale validator run whose owning execution no longer exists.
*
* Intentionally does not delegate to completeValidatorRun(): the generic error path keeps
* the feature in loopState='validating', but a stale-owner recovery must move live features
* back to loopState='needs_fix' so the mission loop can retry validation later.
*
* The feature's lastValidatorRunId is intentionally left pointing at this now-terminal run so
* readers can resolve it and observe the authoritative terminal status instead of a dangling gap.
*/
reapValidatorRun(runId: string, reason: string): MissionValidatorRun {
const run = this.getValidatorRun(runId);
if (!run) {
throw new Error(`Validator run ${runId} not found`);
}
if (run.status !== "running") {
return run;
}
const feature = this.getFeature(run.featureId);
if (!feature) {
throw new Error(`Feature ${run.featureId} not found`);
}
const slice = this.getSlice(feature.sliceId);
if (!slice) {
throw new Error(`Slice ${feature.sliceId} not found`);
}
const milestone = this.getMilestone(slice.milestoneId);
if (!milestone) {
throw new Error(`Milestone ${slice.milestoneId} not found`);
}
const mission = this.getMission(milestone.missionId);
if (!mission) {
throw new Error(`Mission ${milestone.missionId} not found`);
}
const now = new Date().toISOString();
const completedAt = now;
const startedAtMs = new Date(run.startedAt).getTime();
const completedAtMs = new Date(completedAt).getTime();
const durationMs = Math.max(0, completedAtMs - startedAtMs);
const shouldUpdateFeature = mission.status !== "archived" && mission.status !== "complete" && feature.status !== "done";
this.db.transaction(() => {
this.db.prepare(`
UPDATE mission_validator_runs SET
status = ?,
summary = ?,
completedAt = ?,
updatedAt = ?
WHERE id = ?
`).run(
"error",
reason,
completedAt,
now,
runId,
);
if (shouldUpdateFeature) {
this.updateFeature(run.featureId, {
loopState: "needs_fix",
lastValidatorStatus: "error",
});
}
});
this.db.bumpLastModified();
const updatedRun = this.getValidatorRun(runId)!;
this.emit("validator-run:completed", updatedRun, "error", durationMs);
return updatedRun;
}
/**
* Create a generated fix feature for a failed validation.
*