fix(core): restore validating mission recovery (#2372)

Re-lands #2336 directly on current main after its temporary base branch
was merged and deleted.\n\n- allows validating → implementing during
startup recovery\n- shares the transition table across sync and
PostgreSQL mission stores\n- preserves retry-budget enforcement\n-
includes sync and PostgreSQL regressions plus a release
changeset\n\nValidation on current main: focused sync regression 1/1 and
@fusion/core typecheck passed.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Bug Fixes**
* Improved mission recovery after an engine restart when a feature was
interrupted during validation.
* Features can resume implementation correctly after recovering from
loop-state transitions.
* Synchronous and PostgreSQL mission storage now apply the same
validation-to-implementation recovery behavior.
* **Tests**
* Added integration and unit test coverage for startup recovery and
rejection of unknown persisted loop states.
* **Release**
  * Included in a patch update for `@runfusion/fusion`.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: v <v@v.speedport.ip>
This commit is contained in:
flexi767
2026-07-22 02:10:25 +02:00
committed by GitHub
parent c71a9545b0
commit 0e29d9d43c
6 changed files with 150 additions and 14 deletions

View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Resume mission features that were interrupted during validation after an engine restart.
category: fix
dev: Shares one feature-loop transition contract across synchronous and PostgreSQL mission stores.

View File

@@ -0,0 +1,43 @@
/*
FNXC:MissionRecovery 2026-07-19-15:30:
The synchronous MissionStore remains a supported transition surface even though
PostgreSQL owns persistence. Exercise its real transition method so the shared
feature-loop table cannot drift from AsyncMissionStore recovery behavior.
*/
import { describe, expect, it, vi } from "vitest";
import type { Database } from "../db.js";
import { MissionStore } from "../mission-store.js";
import type { MissionFeature } from "../mission-types.js";
describe("MissionStore synchronous loop transitions", () => {
it("allows startup recovery to move an interrupted validation back to implementing", () => {
const db = {
prepare: vi.fn().mockReturnValue({ get: vi.fn().mockReturnValue(undefined) }),
bumpLastModified: vi.fn(),
} as unknown as Database;
const store = new MissionStore("/tmp/fusion-mission-store-test", db);
const feature: MissionFeature = {
id: "F-RECOVERY",
sliceId: "SL-RECOVERY",
title: "Interrupted validation",
status: "in-progress",
loopState: "validating",
implementationAttemptCount: 1,
createdAt: "2026-07-19T00:00:00.000Z",
updatedAt: "2026-07-19T00:00:00.000Z",
};
vi.spyOn(store, "getFeature").mockReturnValue(feature);
const updateFeature = vi.spyOn(store, "updateFeature").mockImplementation((_id, updates) => ({
...feature,
...updates,
}));
expect(store.transitionLoopState(feature.id, "implementing")).toMatchObject({
id: feature.id,
loopState: "implementing",
});
expect(updateFeature).toHaveBeenCalledWith(feature.id, { loopState: "implementing" });
});
});

View File

@@ -317,6 +317,45 @@ pgTest("MissionStore (PostgreSQL backend mode)", () => {
expect((await m.getFeature(fix.id))?.loopState).toBe("needs_fix"); expect((await m.getFeature(fix.id))?.loopState).toBe("needs_fix");
}); });
it("allows startup recovery to move an interrupted validation back to implementing", async () => {
const m = missions();
const mission = await m.createMission({ title: "Interrupted validation" });
const milestone = await m.addMilestone(mission.id, { title: "MS" });
const slice = await m.addSlice(milestone.id, { title: "SL" });
const feature = await m.addFeature(slice.id, { title: "Feature" });
await m.transitionLoopState(feature.id, "implementing");
const interruptedRun = await m.startValidatorRun(feature.id, "scheduled");
expect((await m.getFeature(feature.id))?.loopState).toBe("validating");
await expect(m.transitionLoopState(feature.id, "implementing")).resolves.toMatchObject({
id: feature.id,
loopState: "implementing",
lastValidatorStatus: "error",
});
await expect(m.getValidatorRun(interruptedRun.id)).resolves.toMatchObject({
status: "error",
summary: "Interrupted validation was superseded by loop-state recovery",
});
expect((await m.listStaleRunningValidatorRuns(-1)).map((run) => run.id)).not.toContain(interruptedRun.id);
});
it("rejects an unknown persisted loop state with the normal transition error", async () => {
const m = missions();
const mission = await m.createMission({ title: "Legacy state" });
const milestone = await m.addMilestone(mission.id, { title: "MS" });
const slice = await m.addSlice(milestone.id, { title: "SL" });
const feature = await m.addFeature(slice.id, { title: "Feature" });
await h.layer().db
.update(schema.project.missionFeatures)
.set({ loopState: "legacy_state" as never })
.where(sql`${schema.project.missionFeatures.id} = ${feature.id}`);
await expect(m.transitionLoopState(feature.id, "implementing")).rejects.toThrow(
"Invalid loop state transition from 'legacy_state' to 'implementing'. Allowed transitions from 'legacy_state': none",
);
});
it("allows exactly one terminal validator transition when completion races the stale reaper", async () => { it("allows exactly one terminal validator transition when completion races the stale reaper", async () => {
const primary = missions(); const primary = missions();
const competing = new AsyncMissionStore(h.layer(), h.store()); const competing = new AsyncMissionStore(h.layer(), h.store());

View File

@@ -9,7 +9,7 @@ import { EventEmitter } from "node:events";
import { and, eq, inArray, sql } from "drizzle-orm"; import { and, eq, inArray, sql } from "drizzle-orm";
import * as schema from "./postgres/schema/index.js"; import * as schema from "./postgres/schema/index.js";
import type { AsyncDataLayer } from "./postgres/data-layer.js"; import type { AsyncDataLayer } from "./postgres/data-layer.js";
import { normalizeMissionAssertionType } from "./mission-types.js"; import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType } from "./mission-types.js";
import type { import type {
Mission, Mission,
Milestone, Milestone,
@@ -1230,12 +1230,54 @@ export class AsyncMissionStore extends EventEmitter<MissionStoreEvents> {
const feature = await getFeature(this.db, featureId); const feature = await getFeature(this.db, featureId);
if (!feature) throw new Error(`Feature ${featureId} not found`); if (!feature) throw new Error(`Feature ${featureId} not found`);
const current = feature.loopState ?? "idle"; const current = feature.loopState ?? "idle";
const valid: Record<FeatureLoopState, FeatureLoopState[]> = { idle: ["implementing"], implementing: ["validating"], validating: ["needs_fix", "passed", "blocked"], needs_fix: ["implementing"], passed: [], blocked: [] }; const allowedNextStates = FEATURE_LOOP_TRANSITIONS[current] || [];
if (!valid[current].includes(newState)) throw new Error(`Invalid loop state transition from '${current}' to '${newState}'. Allowed transitions from '${current}': ${valid[current].join(", ") || "none"}`); if (!allowedNextStates.includes(newState)) throw new Error(`Invalid loop state transition from '${current}' to '${newState}'. Allowed transitions from '${current}': ${allowedNextStates.join(", ") || "none"}`);
if (newState === "implementing" && (feature.implementationAttemptCount ?? 0) >= DEFAULT_IMPLEMENTATION_RETRY_BUDGET) { if (newState === "implementing" && (feature.implementationAttemptCount ?? 0) >= DEFAULT_IMPLEMENTATION_RETRY_BUDGET) {
await this.updateFeature(featureId, { loopState: "blocked" }); await this.updateFeature(featureId, { loopState: "blocked" });
throw new Error(`Feature ${featureId} has exhausted its retry budget (${DEFAULT_IMPLEMENTATION_RETRY_BUDGET} attempts). Transitioning to 'blocked' state.`); throw new Error(`Feature ${featureId} has exhausted its retry budget (${DEFAULT_IMPLEMENTATION_RETRY_BUDGET} attempts). Transitioning to 'blocked' state.`);
} }
/*
FNXC:MissionRecovery 2026-07-21-21:30:
Recovering validating to implementing must terminalize the interrupted validator run in the same transaction as the feature transition. A stale reaper or delayed validator completion must not overwrite the resumed feature with an outcome from the abandoned validation cycle.
*/
if (current === "validating" && newState === "implementing" && feature.lastValidatorRunId) {
const run = await getValidatorRun(this.db, feature.lastValidatorRunId);
if (run?.status === "running") {
const now = new Date().toISOString();
const interruptedRun: MissionValidatorRun = {
...run,
status: "error",
summary: "Interrupted validation was superseded by loop-state recovery",
completedAt: now,
updatedAt: now,
};
const won = await this.layer.transactionImmediate(async (tx) => {
const terminalRun = await transitionRunningValidatorRun(tx, interruptedRun);
if (!terminalRun) return false;
await updateFeature(tx, {
...feature,
loopState: "implementing",
lastValidatorStatus: "error",
updatedAt: now,
});
return true;
});
if (won) {
const updated = await getFeature(this.db, featureId);
if (!updated) throw new Error(`Feature ${featureId} not found after recovery`);
this.emit("feature:updated", updated);
this.emit("validator-run:completed", interruptedRun, "error", Math.max(0, Date.parse(now) - Date.parse(run.startedAt)));
return updated;
}
const freshFeature = await getFeature(this.db, featureId);
const freshCurrent = freshFeature?.loopState ?? "idle";
if (freshCurrent !== "validating") {
throw new Error(`Invalid loop state transition from '${freshCurrent}' to '${newState}'. Allowed transitions from '${freshCurrent}': ${(FEATURE_LOOP_TRANSITIONS[freshCurrent] || []).join(", ") || "none"}`);
}
}
}
return this.updateFeature(featureId, { loopState: newState }); return this.updateFeature(featureId, { loopState: newState });
} }

View File

@@ -14,7 +14,7 @@
import { EventEmitter } from "node:events"; import { EventEmitter } from "node:events";
import type { Database } from "./db.js"; import type { Database } from "./db.js";
import { fromJson, toJson, toJsonNullable } from "./db.js"; import { fromJson, toJson, toJsonNullable } from "./db.js";
import { normalizeMissionAssertionType } from "./mission-types.js"; import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType } from "./mission-types.js";
import type { Goal, GoalStatus } from "./goal-types.js"; import type { Goal, GoalStatus } from "./goal-types.js";
import type { import type {
Mission, Mission,
@@ -3303,6 +3303,7 @@ export class MissionStore extends EventEmitter<MissionStoreEvents> {
* Valid transitions: * Valid transitions:
* - idle → implementing * - idle → implementing
* - implementing → validating * - implementing → validating
* - validating → implementing (startup recovery)
* - validating → needs_fix * - validating → needs_fix
* - validating → passed * - validating → passed
* - validating → blocked * - validating → blocked
@@ -3325,16 +3326,7 @@ export class MissionStore extends EventEmitter<MissionStoreEvents> {
const currentState = feature.loopState ?? "idle"; const currentState = feature.loopState ?? "idle";
// Validate the transition // Validate the transition
const validTransitions: Record<FeatureLoopState, FeatureLoopState[]> = { const allowedNextStates = FEATURE_LOOP_TRANSITIONS[currentState] || [];
idle: ["implementing"],
implementing: ["validating"],
validating: ["needs_fix", "passed", "blocked"],
needs_fix: ["implementing"],
passed: [],
blocked: [],
};
const allowedNextStates = validTransitions[currentState] || [];
if (!allowedNextStates.includes(newState)) { if (!allowedNextStates.includes(newState)) {
throw new Error( throw new Error(
`Invalid loop state transition from '${currentState}' to '${newState}'. ` + `Invalid loop state transition from '${currentState}' to '${newState}'. ` +

View File

@@ -37,6 +37,19 @@ export type FeatureStatus = (typeof FEATURE_STATUSES)[number];
export const FEATURE_LOOP_STATES = ["idle", "implementing", "validating", "needs_fix", "passed", "blocked"] as const; export const FEATURE_LOOP_STATES = ["idle", "implementing", "validating", "needs_fix", "passed", "blocked"] as const;
export type FeatureLoopState = (typeof FEATURE_LOOP_STATES)[number]; export type FeatureLoopState = (typeof FEATURE_LOOP_STATES)[number];
/**
* FNXC:MissionRecovery 2026-07-19-14:30:
* Startup recovery re-drives work interrupted during validation by moving the feature back to implementing. Both mission-store backends must share this transition table so recovery cannot be accepted by the engine but rejected by persistence.
*/
export const FEATURE_LOOP_TRANSITIONS: Readonly<Record<FeatureLoopState, readonly FeatureLoopState[]>> = {
idle: ["implementing"],
implementing: ["validating"],
validating: ["implementing", "needs_fix", "passed", "blocked"],
needs_fix: ["implementing"],
passed: [],
blocked: [],
};
/** Status values for a validator run */ /** Status values for a validator run */
export const VALIDATOR_RUN_STATUSES = ["running", "passed", "failed", "blocked", "error"] as const; export const VALIDATOR_RUN_STATUSES = ["running", "passed", "failed", "blocked", "error"] as const;
export type ValidatorRunStatus = (typeof VALIDATOR_RUN_STATUSES)[number]; export type ValidatorRunStatus = (typeof VALIDATOR_RUN_STATUSES)[number];