fix(core): restore validating mission recovery (#2372)
Re-lands #2336 directly on current main after its temporary base branch was merged and deleted.\n\n- allows validating → implementing during startup recovery\n- shares the transition table across sync and PostgreSQL mission stores\n- preserves retry-budget enforcement\n- includes sync and PostgreSQL regressions plus a release changeset\n\nValidation on current main: focused sync regression 1/1 and @fusion/core typecheck passed. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Bug Fixes** * Improved mission recovery after an engine restart when a feature was interrupted during validation. * Features can resume implementation correctly after recovering from loop-state transitions. * Synchronous and PostgreSQL mission storage now apply the same validation-to-implementation recovery behavior. * **Tests** * Added integration and unit test coverage for startup recovery and rejection of unknown persisted loop states. * **Release** * Included in a patch update for `@runfusion/fusion`. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: v <v@v.speedport.ip>
This commit is contained in:
7
.changeset/restore-mission-validation-recovery.md
Normal file
7
.changeset/restore-mission-validation-recovery.md
Normal file
@@ -0,0 +1,7 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
summary: Resume mission features that were interrupted during validation after an engine restart.
|
||||
category: fix
|
||||
dev: Shares one feature-loop transition contract across synchronous and PostgreSQL mission stores.
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
FNXC:MissionRecovery 2026-07-19-15:30:
|
||||
The synchronous MissionStore remains a supported transition surface even though
|
||||
PostgreSQL owns persistence. Exercise its real transition method so the shared
|
||||
feature-loop table cannot drift from AsyncMissionStore recovery behavior.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { Database } from "../db.js";
|
||||
import { MissionStore } from "../mission-store.js";
|
||||
import type { MissionFeature } from "../mission-types.js";
|
||||
|
||||
describe("MissionStore synchronous loop transitions", () => {
|
||||
it("allows startup recovery to move an interrupted validation back to implementing", () => {
|
||||
const db = {
|
||||
prepare: vi.fn().mockReturnValue({ get: vi.fn().mockReturnValue(undefined) }),
|
||||
bumpLastModified: vi.fn(),
|
||||
} as unknown as Database;
|
||||
const store = new MissionStore("/tmp/fusion-mission-store-test", db);
|
||||
const feature: MissionFeature = {
|
||||
id: "F-RECOVERY",
|
||||
sliceId: "SL-RECOVERY",
|
||||
title: "Interrupted validation",
|
||||
status: "in-progress",
|
||||
loopState: "validating",
|
||||
implementationAttemptCount: 1,
|
||||
createdAt: "2026-07-19T00:00:00.000Z",
|
||||
updatedAt: "2026-07-19T00:00:00.000Z",
|
||||
};
|
||||
|
||||
vi.spyOn(store, "getFeature").mockReturnValue(feature);
|
||||
const updateFeature = vi.spyOn(store, "updateFeature").mockImplementation((_id, updates) => ({
|
||||
...feature,
|
||||
...updates,
|
||||
}));
|
||||
|
||||
expect(store.transitionLoopState(feature.id, "implementing")).toMatchObject({
|
||||
id: feature.id,
|
||||
loopState: "implementing",
|
||||
});
|
||||
expect(updateFeature).toHaveBeenCalledWith(feature.id, { loopState: "implementing" });
|
||||
});
|
||||
});
|
||||
@@ -317,6 +317,45 @@ pgTest("MissionStore (PostgreSQL backend mode)", () => {
|
||||
expect((await m.getFeature(fix.id))?.loopState).toBe("needs_fix");
|
||||
});
|
||||
|
||||
it("allows startup recovery to move an interrupted validation back to implementing", async () => {
|
||||
const m = missions();
|
||||
const mission = await m.createMission({ title: "Interrupted validation" });
|
||||
const milestone = await m.addMilestone(mission.id, { title: "MS" });
|
||||
const slice = await m.addSlice(milestone.id, { title: "SL" });
|
||||
const feature = await m.addFeature(slice.id, { title: "Feature" });
|
||||
|
||||
await m.transitionLoopState(feature.id, "implementing");
|
||||
const interruptedRun = await m.startValidatorRun(feature.id, "scheduled");
|
||||
expect((await m.getFeature(feature.id))?.loopState).toBe("validating");
|
||||
|
||||
await expect(m.transitionLoopState(feature.id, "implementing")).resolves.toMatchObject({
|
||||
id: feature.id,
|
||||
loopState: "implementing",
|
||||
lastValidatorStatus: "error",
|
||||
});
|
||||
await expect(m.getValidatorRun(interruptedRun.id)).resolves.toMatchObject({
|
||||
status: "error",
|
||||
summary: "Interrupted validation was superseded by loop-state recovery",
|
||||
});
|
||||
expect((await m.listStaleRunningValidatorRuns(-1)).map((run) => run.id)).not.toContain(interruptedRun.id);
|
||||
});
|
||||
|
||||
it("rejects an unknown persisted loop state with the normal transition error", async () => {
|
||||
const m = missions();
|
||||
const mission = await m.createMission({ title: "Legacy state" });
|
||||
const milestone = await m.addMilestone(mission.id, { title: "MS" });
|
||||
const slice = await m.addSlice(milestone.id, { title: "SL" });
|
||||
const feature = await m.addFeature(slice.id, { title: "Feature" });
|
||||
await h.layer().db
|
||||
.update(schema.project.missionFeatures)
|
||||
.set({ loopState: "legacy_state" as never })
|
||||
.where(sql`${schema.project.missionFeatures.id} = ${feature.id}`);
|
||||
|
||||
await expect(m.transitionLoopState(feature.id, "implementing")).rejects.toThrow(
|
||||
"Invalid loop state transition from 'legacy_state' to 'implementing'. Allowed transitions from 'legacy_state': none",
|
||||
);
|
||||
});
|
||||
|
||||
it("allows exactly one terminal validator transition when completion races the stale reaper", async () => {
|
||||
const primary = missions();
|
||||
const competing = new AsyncMissionStore(h.layer(), h.store());
|
||||
|
||||
@@ -9,7 +9,7 @@ import { EventEmitter } from "node:events";
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import * as schema from "./postgres/schema/index.js";
|
||||
import type { AsyncDataLayer } from "./postgres/data-layer.js";
|
||||
import { normalizeMissionAssertionType } from "./mission-types.js";
|
||||
import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType } from "./mission-types.js";
|
||||
import type {
|
||||
Mission,
|
||||
Milestone,
|
||||
@@ -1230,12 +1230,54 @@ export class AsyncMissionStore extends EventEmitter<MissionStoreEvents> {
|
||||
const feature = await getFeature(this.db, featureId);
|
||||
if (!feature) throw new Error(`Feature ${featureId} not found`);
|
||||
const current = feature.loopState ?? "idle";
|
||||
const valid: Record<FeatureLoopState, FeatureLoopState[]> = { idle: ["implementing"], implementing: ["validating"], validating: ["needs_fix", "passed", "blocked"], needs_fix: ["implementing"], passed: [], blocked: [] };
|
||||
if (!valid[current].includes(newState)) throw new Error(`Invalid loop state transition from '${current}' to '${newState}'. Allowed transitions from '${current}': ${valid[current].join(", ") || "none"}`);
|
||||
const allowedNextStates = FEATURE_LOOP_TRANSITIONS[current] || [];
|
||||
if (!allowedNextStates.includes(newState)) throw new Error(`Invalid loop state transition from '${current}' to '${newState}'. Allowed transitions from '${current}': ${allowedNextStates.join(", ") || "none"}`);
|
||||
if (newState === "implementing" && (feature.implementationAttemptCount ?? 0) >= DEFAULT_IMPLEMENTATION_RETRY_BUDGET) {
|
||||
await this.updateFeature(featureId, { loopState: "blocked" });
|
||||
throw new Error(`Feature ${featureId} has exhausted its retry budget (${DEFAULT_IMPLEMENTATION_RETRY_BUDGET} attempts). Transitioning to 'blocked' state.`);
|
||||
}
|
||||
|
||||
/*
|
||||
FNXC:MissionRecovery 2026-07-21-21:30:
|
||||
Recovering validating to implementing must terminalize the interrupted validator run in the same transaction as the feature transition. A stale reaper or delayed validator completion must not overwrite the resumed feature with an outcome from the abandoned validation cycle.
|
||||
*/
|
||||
if (current === "validating" && newState === "implementing" && feature.lastValidatorRunId) {
|
||||
const run = await getValidatorRun(this.db, feature.lastValidatorRunId);
|
||||
if (run?.status === "running") {
|
||||
const now = new Date().toISOString();
|
||||
const interruptedRun: MissionValidatorRun = {
|
||||
...run,
|
||||
status: "error",
|
||||
summary: "Interrupted validation was superseded by loop-state recovery",
|
||||
completedAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
const won = await this.layer.transactionImmediate(async (tx) => {
|
||||
const terminalRun = await transitionRunningValidatorRun(tx, interruptedRun);
|
||||
if (!terminalRun) return false;
|
||||
await updateFeature(tx, {
|
||||
...feature,
|
||||
loopState: "implementing",
|
||||
lastValidatorStatus: "error",
|
||||
updatedAt: now,
|
||||
});
|
||||
return true;
|
||||
});
|
||||
if (won) {
|
||||
const updated = await getFeature(this.db, featureId);
|
||||
if (!updated) throw new Error(`Feature ${featureId} not found after recovery`);
|
||||
this.emit("feature:updated", updated);
|
||||
this.emit("validator-run:completed", interruptedRun, "error", Math.max(0, Date.parse(now) - Date.parse(run.startedAt)));
|
||||
return updated;
|
||||
}
|
||||
|
||||
const freshFeature = await getFeature(this.db, featureId);
|
||||
const freshCurrent = freshFeature?.loopState ?? "idle";
|
||||
if (freshCurrent !== "validating") {
|
||||
throw new Error(`Invalid loop state transition from '${freshCurrent}' to '${newState}'. Allowed transitions from '${freshCurrent}': ${(FEATURE_LOOP_TRANSITIONS[freshCurrent] || []).join(", ") || "none"}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return this.updateFeature(featureId, { loopState: newState });
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import type { Database } from "./db.js";
|
||||
import { fromJson, toJson, toJsonNullable } from "./db.js";
|
||||
import { normalizeMissionAssertionType } from "./mission-types.js";
|
||||
import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType } from "./mission-types.js";
|
||||
import type { Goal, GoalStatus } from "./goal-types.js";
|
||||
import type {
|
||||
Mission,
|
||||
@@ -3303,6 +3303,7 @@ export class MissionStore extends EventEmitter<MissionStoreEvents> {
|
||||
* Valid transitions:
|
||||
* - idle → implementing
|
||||
* - implementing → validating
|
||||
* - validating → implementing (startup recovery)
|
||||
* - validating → needs_fix
|
||||
* - validating → passed
|
||||
* - validating → blocked
|
||||
@@ -3325,16 +3326,7 @@ export class MissionStore extends EventEmitter<MissionStoreEvents> {
|
||||
const currentState = feature.loopState ?? "idle";
|
||||
|
||||
// Validate the transition
|
||||
const validTransitions: Record<FeatureLoopState, FeatureLoopState[]> = {
|
||||
idle: ["implementing"],
|
||||
implementing: ["validating"],
|
||||
validating: ["needs_fix", "passed", "blocked"],
|
||||
needs_fix: ["implementing"],
|
||||
passed: [],
|
||||
blocked: [],
|
||||
};
|
||||
|
||||
const allowedNextStates = validTransitions[currentState] || [];
|
||||
const allowedNextStates = FEATURE_LOOP_TRANSITIONS[currentState] || [];
|
||||
if (!allowedNextStates.includes(newState)) {
|
||||
throw new Error(
|
||||
`Invalid loop state transition from '${currentState}' to '${newState}'. ` +
|
||||
|
||||
@@ -37,6 +37,19 @@ export type FeatureStatus = (typeof FEATURE_STATUSES)[number];
|
||||
export const FEATURE_LOOP_STATES = ["idle", "implementing", "validating", "needs_fix", "passed", "blocked"] as const;
|
||||
export type FeatureLoopState = (typeof FEATURE_LOOP_STATES)[number];
|
||||
|
||||
/**
|
||||
* FNXC:MissionRecovery 2026-07-19-14:30:
|
||||
* Startup recovery re-drives work interrupted during validation by moving the feature back to implementing. Both mission-store backends must share this transition table so recovery cannot be accepted by the engine but rejected by persistence.
|
||||
*/
|
||||
export const FEATURE_LOOP_TRANSITIONS: Readonly<Record<FeatureLoopState, readonly FeatureLoopState[]>> = {
|
||||
idle: ["implementing"],
|
||||
implementing: ["validating"],
|
||||
validating: ["implementing", "needs_fix", "passed", "blocked"],
|
||||
needs_fix: ["implementing"],
|
||||
passed: [],
|
||||
blocked: [],
|
||||
};
|
||||
|
||||
/** Status values for a validator run */
|
||||
export const VALIDATOR_RUN_STATUSES = ["running", "passed", "failed", "blocked", "error"] as const;
|
||||
export type ValidatorRunStatus = (typeof VALIDATOR_RUN_STATUSES)[number];
|
||||
|
||||
Reference in New Issue
Block a user