FN-5755: harden mission validation recovery flow
Ensure mission feature assertions recover and validate consistently across startup and periodic maintenance. - add FN-5755 changeset and AGENTS reliability backstop note - document canonical zero-assertion auto-pass and assertion validation lifecycle updates in missions docs - expand mission execution loop and reliability interaction tests for startup recovery, periodic replay, and idempotency - wire self-healing maintenance to replay active mission validation recovery via runtime callback Files changed: .../fn-5755-mission-validation-end-to-end.md | 5 ++ AGENTS.md | 1 + docs/missions-completion-contract.md | 15 ++-- docs/missions.md | 4 +- .../src/__tests__/mission-execution-loop.test.ts | 88 +++++++++++++++++++++- .../mission-validation-trigger-gap.test.ts | 42 ++++++++++- packages/engine/src/runtimes/in-process-runtime.ts | 6 ++ packages/engine/src/self-healing.ts | 11 +++ 8 files changed, 159 insertions(+), 13 deletions(-) Fusion-Task-Id: FN-5755 Fusion-Task-Lineage: e54ac4dc-7b8c-4fc9-8cd6-18702708546f
This commit is contained in:
@@ -216,6 +216,40 @@ function createMockMissionStore() {
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
validatorRuns.set(id, updated);
|
||||
|
||||
const feature = features.get(run.featureId);
|
||||
if (feature) {
|
||||
if (status === "passed") {
|
||||
features.set(run.featureId, {
|
||||
...feature,
|
||||
loopState: "passed",
|
||||
lastValidatorStatus: "passed",
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
} else if (status === "failed") {
|
||||
features.set(run.featureId, {
|
||||
...feature,
|
||||
loopState: "needs_fix",
|
||||
lastValidatorStatus: "failed",
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
} else if (status === "blocked") {
|
||||
features.set(run.featureId, {
|
||||
...feature,
|
||||
loopState: "blocked",
|
||||
lastValidatorStatus: "blocked",
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
} else if (status === "error") {
|
||||
features.set(run.featureId, {
|
||||
...feature,
|
||||
loopState: "validating",
|
||||
lastValidatorStatus: "error",
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return updated;
|
||||
}),
|
||||
recordValidatorFailures: vi.fn(() => []),
|
||||
@@ -238,7 +272,7 @@ function createMockMissionStore() {
|
||||
const updatedSource = {
|
||||
...sourceFeature,
|
||||
implementationAttemptCount: (sourceFeature.implementationAttemptCount ?? 0) + 1,
|
||||
loopState: "needs_fix" as const,
|
||||
loopState: "implementing" as const,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
features.set(sourceFeatureId, updatedSource);
|
||||
@@ -680,6 +714,58 @@ describe("MissionExecutionLoop", () => {
|
||||
"task_completion",
|
||||
);
|
||||
});
|
||||
|
||||
it("runs linked assertions and marks completion only when validation passes", async () => {
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-ASSERT-PASS", status: "in-progress" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue(makeAssertions(2));
|
||||
taskStore._setTask({ id: "FN-ASSERT-PASS", title: "Assertion pass", description: "Implementation", log: [] });
|
||||
mockSessionHolder.session.state.messages = [
|
||||
{ role: "assistant", content: JSON.stringify({ status: "pass", assertions: [{ assertionId: "CA-1", passed: true }, { assertionId: "CA-2", passed: true }], summary: "all good" }) },
|
||||
];
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-ASSERT-PASS");
|
||||
|
||||
expect(missionStore.startValidatorRun).toHaveBeenCalledWith("F-001", "task_completion");
|
||||
expect(missionStore.completeValidatorRun).toHaveBeenCalledWith(expect.any(String), "passed", expect.any(String));
|
||||
expect(missionStore.getFeature("F-001")?.loopState).toBe("passed");
|
||||
expect(missionStore.getFeature("F-001")?.lastValidatorStatus).toBe("passed");
|
||||
expect(missionStore.updateFeatureStatus).toHaveBeenCalledWith("F-001", "done");
|
||||
});
|
||||
|
||||
it("routes failed assertion validation to fix flow and does not pass feature", async () => {
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-ASSERT-FAIL", status: "in-progress" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue(makeAssertions(2));
|
||||
taskStore._setTask({ id: "FN-ASSERT-FAIL", title: "Assertion fail", description: "Implementation", log: [] });
|
||||
mockSessionHolder.session.state.messages = [
|
||||
{ role: "assistant", content: JSON.stringify({ status: "fail", assertions: [{ assertionId: "CA-1", passed: false, message: "miss" }], summary: "failed" }) },
|
||||
];
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-ASSERT-FAIL");
|
||||
|
||||
expect(missionStore.completeValidatorRun).toHaveBeenCalledWith(expect.any(String), "failed", expect.any(String));
|
||||
expect(missionStore.createGeneratedFixFeature).toHaveBeenCalled();
|
||||
expect(missionStore.getFeature("F-001")?.lastValidatorStatus).toBe("failed");
|
||||
expect(missionStore.getFeature("F-001")?.loopState).toBe("implementing");
|
||||
expect(missionStore.getFeature("F-001")?.status).not.toBe("done");
|
||||
});
|
||||
});
|
||||
|
||||
// ── recoverActiveMissions ────────────────────────────────────────────────
|
||||
|
||||
@@ -120,6 +120,41 @@ describe("FN-5715 reliability: mission validation trigger gap", () => {
|
||||
loop.stop();
|
||||
});
|
||||
|
||||
it("recovery trigger for done implementing feature is idempotent across subsequent passes", async () => {
|
||||
const feature = makeFeature({ status: "done", lastValidatorStatus: undefined, loopState: "implementing" });
|
||||
const missionStore = {
|
||||
listMissions: vi.fn(() => [{ id: "M-001", status: "active" }]),
|
||||
getMissionWithHierarchy: vi.fn(() => ({
|
||||
id: "M-001",
|
||||
status: "active",
|
||||
milestones: [{ status: "active", slices: [{ status: "active", features: [feature] }] }],
|
||||
})),
|
||||
listAssertionsForFeature: vi.fn(() => [{ id: "CA-1" }]),
|
||||
getFeature: vi.fn(() => feature),
|
||||
transitionLoopState: vi.fn(),
|
||||
};
|
||||
const taskStore = {
|
||||
getTask: vi.fn(async () => ({ id: "FN-001", column: "done" })),
|
||||
};
|
||||
|
||||
const loop = new MissionExecutionLoop({
|
||||
missionStore: missionStore as any,
|
||||
taskStore: taskStore as any,
|
||||
rootDir: process.cwd(),
|
||||
});
|
||||
const processSpy = vi.spyOn(loop, "processTaskOutcome").mockImplementation(async () => {
|
||||
feature.lastValidatorStatus = "passed";
|
||||
feature.loopState = "passed";
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.recoverActiveMissions();
|
||||
await loop.recoverActiveMissions();
|
||||
|
||||
expect(processSpy).toHaveBeenCalledTimes(1);
|
||||
loop.stop();
|
||||
});
|
||||
|
||||
it("is idempotent for already-passed implementing features", async () => {
|
||||
const feature = makeFeature({ status: "done", lastValidatorStatus: "passed" });
|
||||
const missionStore = {
|
||||
@@ -151,7 +186,7 @@ describe("FN-5715 reliability: mission validation trigger gap", () => {
|
||||
loop.stop();
|
||||
});
|
||||
|
||||
it("recovery replays implementing done tasks with zero assertions and advances loop state", async () => {
|
||||
it("periodic recovery pass replays implementing done tasks with zero assertions and advances loop state", async () => {
|
||||
const feature = makeFeature({ status: "done", lastValidatorStatus: undefined, loopState: "implementing" });
|
||||
const currentFeature = { ...feature };
|
||||
const missionStore = {
|
||||
@@ -187,8 +222,9 @@ describe("FN-5715 reliability: mission validation trigger gap", () => {
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.recoverActiveMissions();
|
||||
await loop.recoverActiveMissions();
|
||||
const periodicMaintenancePass = async () => loop.recoverActiveMissions();
|
||||
await periodicMaintenancePass();
|
||||
await periodicMaintenancePass();
|
||||
|
||||
expect(missionStore.updateFeature).toHaveBeenCalledTimes(1);
|
||||
expect(missionStore.updateFeature).toHaveBeenCalledWith(
|
||||
|
||||
@@ -732,6 +732,12 @@ export class InProcessRuntime
|
||||
getActiveMergeTaskId: () => this.activeMergeTaskIdProvider?.() ?? null,
|
||||
leaseManager: this.leaseManager,
|
||||
hasActiveAgentExecution: (agentId: string) => this.heartbeatMonitor?.getTrackedAgents().includes(agentId) ?? false,
|
||||
recoverActiveMissionValidations: async () => {
|
||||
if (!this.missionExecutionLoop) {
|
||||
return { recoveredCount: 0 };
|
||||
}
|
||||
return this.missionExecutionLoop.recoverActiveMissions();
|
||||
},
|
||||
reconcileAllMissionFeatures: async () => this.scheduler.reconcileAllMissionFeatures(),
|
||||
chatStore: this.chatStore,
|
||||
messageStore: this.messageStore,
|
||||
|
||||
@@ -276,6 +276,8 @@ export interface SelfHealingOptions {
|
||||
getProjectId?: () => string;
|
||||
/** Optional callback to reconcile active mission features during maintenance. */
|
||||
reconcileAllMissionFeatures?: () => Promise<number>;
|
||||
/** Optional callback to re-run mission validation recovery during maintenance. */
|
||||
recoverActiveMissionValidations?: () => Promise<{ recoveredCount: number }>;
|
||||
}
|
||||
|
||||
const APPROVED_TRIAGE_RECOVERY_GRACE_MS = 60_000;
|
||||
@@ -1470,6 +1472,15 @@ export class SelfHealingManager {
|
||||
} else {
|
||||
// Batch 2 — Task recovery (operations are independent of each other)
|
||||
const batch2Fns: Array<{ name: string; fn: () => Promise<unknown> }> = [
|
||||
{
|
||||
name: "recover-active-mission-validations",
|
||||
fn: async () => {
|
||||
if (!this.options.recoverActiveMissionValidations) {
|
||||
return;
|
||||
}
|
||||
await this.options.recoverActiveMissionValidations();
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "reconcile-mission-features",
|
||||
fn: async () => {
|
||||
|
||||
Reference in New Issue
Block a user