From c2604d5e52797f3d521bdba19ca7e3c3a9821d7e Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 3 Jun 2026 07:06:30 -0700 Subject: [PATCH 1/2] fix(engine): recover missions wedged by stranded done features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mission feature could be left status="done" while its loopState never advanced past "implementing" and it had no linked board task, so it was never validated. The slice-completion gate (computeSliceStatus) correctly refuses to count an assertion-linked done feature until its validator passes, but nothing re-drove a task-less feature — so the slice, milestone, and whole mission could never auto-progress. Active-mission recovery now detects these stranded done features and re-runs assertion validation directly (read-only judge, no board task): on pass the feature becomes legitimately complete, on fail the normal fix-feature flow takes over. Extracted the feature-validation path into a shared runFeatureValidation helper used by both task-completion and recovery. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../fix-stranded-done-feature-recovery.md | 9 ++ .../__tests__/mission-execution-loop.test.ts | 81 ++++++++++++ packages/engine/src/mission-execution-loop.ts | 119 ++++++++++++------ 3 files changed, 174 insertions(+), 35 deletions(-) create mode 100644 .changeset/fix-stranded-done-feature-recovery.md diff --git a/.changeset/fix-stranded-done-feature-recovery.md b/.changeset/fix-stranded-done-feature-recovery.md new file mode 100644 index 0000000000..222c1d437d --- /dev/null +++ b/.changeset/fix-stranded-done-feature-recovery.md @@ -0,0 +1,9 @@ +--- +"@runfusion/fusion": patch +--- + +Fix missions stalling when a feature is marked `done` but stranded mid-loop. + +A mission feature could be left `status: "done"` while its `loopState` never advanced past `"implementing"` and it had no linked board task (so it was never validated). The slice-completion gate (`MissionStore.computeSliceStatus`) correctly refuses to count an assertion-linked `done` feature until its validator passes, but nothing re-drove a task-less feature, so the slice — and the whole mission — could never auto-progress. + +Active-mission recovery now detects these stranded `done` features and re-runs assertion validation directly (no board task), so the gate can resolve: on pass the feature becomes legitimately complete, on fail the normal fix-feature flow takes over. The feature-validation path was extracted into a shared `runFeatureValidation` helper used by both task-completion and recovery. diff --git a/packages/engine/src/__tests__/mission-execution-loop.test.ts b/packages/engine/src/__tests__/mission-execution-loop.test.ts index 41b9182d12..e25a50ada8 100644 --- a/packages/engine/src/__tests__/mission-execution-loop.test.ts +++ b/packages/engine/src/__tests__/mission-execution-loop.test.ts @@ -546,6 +546,87 @@ describe("MissionExecutionLoop", () => { }); }); + describe("recoverActiveMissions stranded done features", () => { + function wireHierarchy(slice: Slice, features: MissionFeature[]) { + missionStore.getMissionWithHierarchy = vi.fn((id: string) => { + const mission = missionStore.getMission(id); + if (!mission) return undefined; + return { + ...mission, + milestones: [ + { + ...createMockMilestone({ missionId: id }), + slices: [{ ...slice, features }], + }, + ], + }; + }) as any; + } + + it("re-validates a done feature stranded in 'implementing' with no linked task", async () => { + // Regression: a feature marked "done" whose loopState never left + // "implementing" (and which was never validated and has no board task) + // can never validate on its own — the prior recovery loop only re-drove + // implementing features that still had a taskId. The slice-completion + // gate then refuses to count it, wedging the whole mission. Recovery + // must re-drive validation so the slice can eventually complete. + const mission = createMockMission({ id: "M-STRAND", status: "active" }); + missionStore._setMission(mission); + + const slice = createMockSlice({ id: "SL-STRAND", milestoneId: "MS-001", status: "active" }); + const orphan = createMockFeature({ + id: "F-STRAND", + sliceId: "SL-STRAND", + status: "done", + loopState: "implementing", + lastValidatorStatus: undefined, + taskId: undefined, + }); + (missionStore as any)._addFeatureWithManagedAssertion(orphan); + wireHierarchy(slice, [missionStore.getFeature("F-STRAND") as MissionFeature]); + + loop = new MissionExecutionLoop({ + taskStore: taskStore as any, + missionStore: missionStore as any, + rootDir: "/tmp", + }); + loop.start(); + + const result = await loop.recoverActiveMissions(); + + expect(missionStore.startValidatorRun).toHaveBeenCalledWith("F-STRAND", "task_completion"); + expect(result.recoveredCount).toBeGreaterThanOrEqual(1); + }); + + it("leaves an already-validated done feature untouched", async () => { + const mission = createMockMission({ id: "M-OK", status: "active" }); + missionStore._setMission(mission); + + const slice = createMockSlice({ id: "SL-OK", milestoneId: "MS-001", status: "active" }); + const validated = createMockFeature({ + id: "F-OK", + sliceId: "SL-OK", + status: "done", + loopState: "passed", + lastValidatorStatus: "passed", + taskId: undefined, + }); + (missionStore as any)._addFeatureWithManagedAssertion(validated); + wireHierarchy(slice, [missionStore.getFeature("F-OK") as MissionFeature]); + + loop = new MissionExecutionLoop({ + taskStore: taskStore as any, + missionStore: missionStore as any, + rootDir: "/tmp", + }); + loop.start(); + + await loop.recoverActiveMissions(); + + expect(missionStore.startValidatorRun).not.toHaveBeenCalled(); + }); + }); + describe("reapStaleValidatorRuns", () => { it("reaps stale runs across trigger types and records audit metadata", async () => { vi.useFakeTimers(); diff --git a/packages/engine/src/mission-execution-loop.ts b/packages/engine/src/mission-execution-loop.ts index 9d7b864348..6c34e4dbe1 100644 --- a/packages/engine/src/mission-execution-loop.ts +++ b/packages/engine/src/mission-execution-loop.ts @@ -292,6 +292,42 @@ export class MissionExecutionLoop extends EventEmitter { loopLog.error(`Recovery failed for implementing feature ${feature.id}:`, err); } } + + // Features marked "done" but stranded in "implementing" with no + // linked task can never validate on their own: the branches above + // only re-drive features that still carry a taskId. Meanwhile the + // slice-completion gate (MissionStore.computeSliceStatus) refuses + // to count an assertion-linked "done" feature until its validator + // passes — so the slice, milestone, and mission can never + // auto-progress. Re-drive validation directly so the gate can + // resolve. Validation is a read-only judge (no board task, no code + // changes); on pass the feature becomes legitimately complete, on + // fail the normal fix-feature flow takes over. + if ( + feature.loopState === "implementing" + && !feature.taskId + && feature.status === "done" + && feature.lastValidatorStatus !== "passed" + && !this.activeValidations.has(feature.id) + ) { + const currentFeature = this.missionStore.getFeature(feature.id) ?? feature; + if ( + currentFeature.loopState === "passed" + || currentFeature.lastValidatorStatus === "passed" + ) { + continue; + } + try { + loopLog.warn( + `Recovery: re-validating stranded "done" feature ${feature.id} ` + + `(loopState=${feature.loopState}, no linked task) so its slice can complete`, + ); + recoveredCount++; + await this.runFeatureValidation(currentFeature); + } catch (err) { + loopLog.error(`Recovery failed for stranded done feature ${feature.id}:`, err); + } + } } } } @@ -353,47 +389,60 @@ export class MissionExecutionLoop extends EventEmitter { return; } - // Get linked assertions for this feature - const assertions = this.missionStore.listAssertionsForFeature(feature.id); - if (assertions.length === 0) { - loopLog.log(`Feature ${feature.id} has no linked assertions; marking as passed`); - // No assertions = automatically pass - await this.handleValidationPass(feature.id, undefined, "No assertions linked"); - return; - } - - // Mark feature as being validated - this.activeValidations.add(feature.id); - - try { - loopLog.log(`Running internal validation for feature ${feature.id} — no board task created (policy: docs/missions.md)`); - - // Start the validator run (no board task per docs/missions.md) - const run = this.missionStore.startValidatorRun(feature.id, "task_completion"); - loopLog.log(`Started validator run ${run.id} for feature ${feature.id}`); - - // Run the validation - const result = await this.runValidation(feature, assertions, run); - - // Handle the result - if (result.status === "pass") { - await this.handleValidationPass(feature.id, run.id, result.summary); - } else if (result.status === "fail") { - await this.handleValidationFail(feature.id, run.id, result); - } else if (result.status === "blocked") { - await this.handleValidationBlocked(feature.id, run.id, result.blockedReason); - } else if (result.status === "error") { - await this.handleValidationError(feature.id, run.id, result.summary); - } - } finally { - this.activeValidations.delete(feature.id); - } + await this.runFeatureValidation(feature); } catch (err) { loopLog.error(`Error processing task outcome for ${taskId}:`, err); // Don't crash the loop - log and continue } } + /** + * Run assertion validation for a feature and apply the outcome. + * + * Shared by processTaskOutcome (task-triggered) and recoverActiveMissions + * (self-healing for features stranded mid-loop with no board task). Callers + * are responsible for confirming the feature is eligible to validate; this + * method handles the no-assertion auto-pass, validator run bookkeeping, and + * dispatch of the validation result. + */ + private async runFeatureValidation(feature: MissionFeature): Promise { + // Get linked assertions for this feature + const assertions = this.missionStore.listAssertionsForFeature(feature.id); + if (assertions.length === 0) { + loopLog.log(`Feature ${feature.id} has no linked assertions; marking as passed`); + // No assertions = automatically pass + await this.handleValidationPass(feature.id, undefined, "No assertions linked"); + return; + } + + // Mark feature as being validated + this.activeValidations.add(feature.id); + + try { + loopLog.log(`Running internal validation for feature ${feature.id} — no board task created (policy: docs/missions.md)`); + + // Start the validator run (no board task per docs/missions.md) + const run = this.missionStore.startValidatorRun(feature.id, "task_completion"); + loopLog.log(`Started validator run ${run.id} for feature ${feature.id}`); + + // Run the validation + const result = await this.runValidation(feature, assertions, run); + + // Handle the result + if (result.status === "pass") { + await this.handleValidationPass(feature.id, run.id, result.summary); + } else if (result.status === "fail") { + await this.handleValidationFail(feature.id, run.id, result); + } else if (result.status === "blocked") { + await this.handleValidationBlocked(feature.id, run.id, result.blockedReason); + } else if (result.status === "error") { + await this.handleValidationError(feature.id, run.id, result.summary); + } + } finally { + this.activeValidations.delete(feature.id); + } + } + /** * Run the validation AI session for a feature. * From c431c0e55d24ccba5e14f9fc7b516974cab9aa1a Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 3 Jun 2026 07:51:48 -0700 Subject: [PATCH 2/2] docs: capture learning for the mission auto-progress wedge Document the stranded done+implementing feature stall in docs/solutions/logic-errors/, seed CONCEPTS.md with the mission domain vocabulary, and surface both from AGENTS.md's reference-docs list. Co-Authored-By: Claude Opus 4.8 (1M context) --- AGENTS.md | 2 + CONCEPTS.md | 38 ++++++ ...opilot-stalled-by-stranded-done-feature.md | 124 ++++++++++++++++++ 3 files changed, 164 insertions(+) create mode 100644 CONCEPTS.md create mode 100644 docs/solutions/logic-errors/mission-autopilot-stalled-by-stranded-done-feature.md diff --git a/AGENTS.md b/AGENTS.md index 6a3c56f29d..673a58179a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -205,6 +205,8 @@ Scoped exception (FN-5819): shared-branch-group members (`branchContext.assignme - `./docs/soft-delete-verification-matrix.md` — mandatory soft-delete verification matrix. - `./docs/cli-reference.md` — CLI and terminal UI reference. - `./docs/contributing.md` — contributing conventions and release-adjacent context. +- `./docs/solutions/` — documented solutions to past problems (bugs, best practices, workflow patterns), organized by category with YAML frontmatter (`module`, `tags`, `problem_type`). Relevant when implementing or debugging in documented areas. +- `./CONCEPTS.md` — shared domain vocabulary (entities, named processes, status concepts) with project-specific meaning. Relevant when orienting to the codebase or discussing domain concepts. ### Lazy-Loaded Heavy Views diff --git a/CONCEPTS.md b/CONCEPTS.md new file mode 100644 index 0000000000..a57bf81484 --- /dev/null +++ b/CONCEPTS.md @@ -0,0 +1,38 @@ +# Concepts + +Shared domain vocabulary for this project — entities, named processes, and status concepts with project-specific meaning. Seeded with core domain vocabulary, then accretes as ce-compound and ce-compound-refresh process learnings; direct edits are fine. Glossary only, not a spec or catch-all. + +## Relationships + +A Mission owns an ordered list of Milestones; a Milestone owns an ordered list of Slices; a Slice owns a set of Features. Status rolls **up**, not down: a Slice's status is derived from its Features, a Milestone's from its Slices, and a Mission's from its Milestones. Autopilot acts at the Slice boundary — it advances a Mission by activating the next Slice once the current one is complete. + +## Missions + +### Mission +A unit of autonomous, multi-step work the system plans and then drives to completion on its own, decomposed into Milestones. A Mission may run under Autopilot or be advanced manually. + +### Milestone +An ordered phase of a Mission, containing Slices and optionally depending on earlier Milestones. A Milestone is complete only when all of its Slices are complete. + +### Slice +A vertically-scoped, independently-completable chunk of a Milestone, containing Features. A Slice's status is derived from its Features and reaches *complete* only when every Feature counts as done — which, for a Feature carrying Contract Assertions, requires a passing Validator Run. + +### Feature +The smallest unit of mission work: a single deliverable evaluated against its Contract Assertions. A Feature carries both a board status (its workflow column, e.g. done) and a loop state (its execution phase); the two are distinct and can legitimately disagree mid-flight, but a done Feature that never reached a terminal loop state is an invariant violation that will stall its Slice. + +### Fix Feature +A Feature auto-generated from a failed Validator Run to carry the remediation work for the assertions that failed, linked back to the Feature it descends from. + +## Mission execution + +### Autopilot +The named process that watches an active Mission and advances it — activating the next pending Slice once the current Slice completes — while tracking its own watching/activating lifecycle and handling retries. When Autopilot is not watching a Mission, slice advancement falls back to a compatibility path. + +### Contract Assertion +A checkable acceptance criterion linked to a Feature that an AI validator judges to decide whether the Feature is genuinely done. A Feature with no linked assertions auto-passes; a Feature with assertions counts toward Slice completion only after a passing Validator Run. + +### Validator Run +A single execution of the AI judge that evaluates a Feature's Contract Assertions and yields a pass, fail, blocked, or error outcome. The validator is read-only — it inspects the implementation and records a verdict, creating no board task and editing no code. A run left running after its owner disappears is reaped to a terminal error state. + +### loop state +A Feature's position in the execution loop (being implemented, awaiting or undergoing validation, awaiting a fix, passed, or blocked), distinct from its board status. Logic that gates on loop state must treat it as possibly stale and possibly contradictory with status — a Feature can be marked done while its loop state was never advanced past implementing. diff --git a/docs/solutions/logic-errors/mission-autopilot-stalled-by-stranded-done-feature.md b/docs/solutions/logic-errors/mission-autopilot-stalled-by-stranded-done-feature.md new file mode 100644 index 0000000000..5238346833 --- /dev/null +++ b/docs/solutions/logic-errors/mission-autopilot-stalled-by-stranded-done-feature.md @@ -0,0 +1,124 @@ +--- +title: "Mission autopilot stalls forever on a done+implementing feature with no task" +date: 2026-06-03 +category: docs/solutions/logic-errors +module: "engine/mission-execution-loop + core/mission-store" +problem_type: logic_error +component: background_job +symptoms: + - "A mission silently stops advancing — no error, no crash, just no progress" + - "Autopilot cycles watching to activating to watching indefinitely in mission_events, never advancing the milestone" + - "A slice stays stuck active even though all of its features report status=done" + - "Wedged feature shows the contradictory combo: status=done plus loopState=implementing plus null lastValidatorStatus plus a linked assertion plus no taskId" +root_cause: missing_workflow_step +resolution_type: code_fix +severity: high +related_components: + - "packages/engine/src/mission-execution-loop.ts (recoverActiveMissions, runFeatureValidation)" + - "packages/core/src/mission-store.ts (computeSliceStatus)" +tags: + - mission-system + - autopilot + - recovery + - slice-completion + - assertion-validation + - loop-state +--- + +# Mission autopilot stalls forever on a done+implementing feature with no task + +## Problem + +A mission feature could be left `status="done"` while its `loopState` stayed `"implementing"`, with no linked board task (`taskId`) and never validated (`lastValidatorStatus` null). The slice-completion gate correctly refuses to count an unvalidated, assertion-linked `done` feature, so the slice — and therefore the milestone and the whole mission — could never auto-progress. The mission stalled silently and indefinitely. + +## Symptoms + +- A mission stops advancing entirely — no error, no crash, just no forward motion. +- Autopilot cycles `watching → activating → watching` forever in `mission_events`, never advancing the milestone. +- A slice stays `active` even though every feature in it reports `status="done"`. +- The wedged features carry the contradictory combination: `status="done"` + `loopState="implementing"` + `lastValidatorStatus=null` + at least one linked assertion + no `taskId`. + +## What Didn't Work + +The first hypothesis came from reading code alone: an early `return` in the scheduler — the `reconciliation.kind === "blocked"` branch in `handleMissionTaskMove` — looked like it could swallow the transition before the completion handler ran. Plausible on inspection, but **not** what wedged this mission. + +The real cause only surfaced by inspecting the live per-project DB read-only (`file:.../.fusion/fusion.db?mode=ro`) and looking at the actual stored feature rows. The diagnosis was then confirmed by contrast: an already-**completed** older mission also had many `done`+`implementing` features, but with **zero** assertions — so the gate let them through. That isolated the *assertion gate* as the active ingredient, not the `done`+`implementing` pairing by itself. + +Lesson: reasoning from code alone pointed at the wrong early-return; observed data found the orphan state. + +## Solution + +Two independent, individually-correct facts interlocked into a deadlock: + +1. **The slice gate is strict (by design).** `MissionStore.computeSliceStatus` (`packages/core/src/mission-store.ts:3866-3880`, added by FN-5715) refuses to count an assertion-linked `done` feature toward slice completion unless its validator passed *or* its `loopState` is idle/undefined. +2. **The recovery sweep had a gap.** `MissionExecutionLoop.recoverActiveMissions` only re-drove `implementing` features that still carried a `taskId` (`feature.loopState === "implementing" && feature.taskId`). A task-less stranded `done` feature matched none of the recovery branches (`validating` / `needs_fix` / `implementing && taskId`), so it could never be validated. + +The fix adds a recovery branch for the orphan and extracts the validation path into a shared helper. Validation is a read-only judge (no board task created, no code edited), so it is safe to run directly from the recovery sweep. + +```ts +// packages/engine/src/mission-execution-loop.ts — recoverActiveMissions, +// after the existing implementing+taskId branch +if ( + feature.loopState === "implementing" + && !feature.taskId + && feature.status === "done" + && feature.lastValidatorStatus !== "passed" + && !this.activeValidations.has(feature.id) +) { + const currentFeature = this.missionStore.getFeature(feature.id) ?? feature; + // Live re-check: skip if it has since passed (avoids racing a concurrent pass) + if ( + currentFeature.loopState === "passed" + || currentFeature.lastValidatorStatus === "passed" + ) { + continue; + } + recoveredCount++; + await this.runFeatureValidation(currentFeature); +} +``` + +The validation execution path was lifted out of `processTaskOutcome` into a reusable private method (behavior-preserving for the existing task-completion path): + +```ts +// processTaskOutcome's inline block becomes a single call: +await this.runFeatureValidation(feature); + +// shared helper used by both task-completion and recovery: +private async runFeatureValidation(feature: MissionFeature): Promise { + const assertions = this.missionStore.listAssertionsForFeature(feature.id); + if (assertions.length === 0) { + await this.handleValidationPass(feature.id, undefined, "No assertions linked"); + return; + } + this.activeValidations.add(feature.id); + try { + const run = this.missionStore.startValidatorRun(feature.id, "task_completion"); + const result = await this.runValidation(feature, assertions, run); + // dispatch pass / fail / blocked / error as before + } finally { + this.activeValidations.delete(feature.id); + } +} +``` + +Shipped in PR #1345 (commit `c2604d5`). Tests added in `packages/engine/src/__tests__/mission-execution-loop.test.ts`; full mission-execution-loop suite plus self-healing/validator-reaper suites stayed green. + +## Why This Works + +The mission stalled because the validator never ran → `lastValidatorStatus` stayed null → `computeSliceStatus` never let the slice reach `complete` → the milestone never completed → autopilot looped forever. The gate was right to block; the bug was that nothing ever *satisfied* the gate for a task-less feature. Re-driving validation gives the orphan a terminal validator status either way: on pass it becomes legitimately complete and the slice resolves; on fail the existing fix-feature flow takes over. The live `getFeature` re-check before validating avoids racing a concurrent pass. + +## Prevention + +- **Treat `loopState` as possibly-stale and possibly-contradictory with `status`.** The `done` + non-terminal-`loopState` pairing is an invariant violation worth asserting/reconciling at write time, not just tolerating downstream. Any logic that *gates* on `loopState` inherits this fragility. +- **Recovery/self-healing sweeps keyed on `taskId` must handle the task-less orphan.** Conditions like `loopState === "implementing" && feature.taskId` silently skip any feature missing the key. Enumerate the orphan states explicitly. +- **When two individually-correct rules can interlock into a deadlock** (a strict gate + an incomplete recovery sweep), add an explicit reconciliation path rather than weakening the gate. +- **Diagnostic tip:** when a state machine stalls with no error, inspect the live DB read-only (`?mode=ro`) and read the actual stored values; contrast a wedged instance against a healthy/completed one to isolate the active ingredient. Code-reading alone misdirected this investigation. + +## Related Issues + +- `docs/missions-completion-contract.md` — the canonical FN-5715 completion-gate contract. It already covers (a) zero-assertion features going to `loopState="passed"` and (b) `taskId == null` features being re-triaged, but does **not** yet cover this specific orphan: `done` + `implementing` + no `taskId` + never validated. This learning extends that contract; the invariant belongs folded into its "Slice Status / Autopilot Advance" and "Validator/loop behavior" sections. +- `docs/missions.md:297` — documents stranded-feature (`taskId == null`) reconciliation and the `mission:stranded-feature-triaged` audit event. +- FN-5721 (#1183) — "Implement mission completion gate contract" (FN-5715 enforcement baseline); closest companion issue. +- FN-5901 — "reap stale mission validator runs": the sibling self-healing pattern for stale *validator* runs. This fix is the analogous self-heal for stranded *implementing* features. (session history) +- FN-5902 (in flight as of 2026-06-02) — "make ALL mission validation AI-run; eliminate zero-assertion auto-pass". Touches the same validation pipeline (`mission-execution-loop.ts` auto-pass branch); changing zero-assertion behavior interacts with this gate. (session history)