From 7b010ec95f4411d0535f778622059849cd22bc1d Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Fri, 7 Aug 2026 20:56:52 -0700 Subject: [PATCH] FN-8824: enforce serial mission slice advancement Keep mission autopilot progression limited to one active, milestone-ordered slice. - Add atomic store-level admission for the next serially eligible slice. - Reconcile completed active slices before advancing and route scheduler activation through the shared guard. - Cover duplicate completion signals, recovery, PostgreSQL, and API behavior; document and release the fix. Files changed: .changeset/fn-8824-mission-slice-advancement.md | 7 + docs/missions.md | 10 +- .../mission-store.sync-loop-transition.test.ts | 43 ++++++ .../postgres/mission-autopilot.pg.test.ts | 25 ++++ .../core/src/async-stores/async-mission-store.ts | 48 ++++++- packages/core/src/index.gate.ts | 1 + packages/core/src/index.ts | 1 + packages/core/src/missions/mission-store.ts | 37 ++++- packages/core/src/missions/mission-types.ts | 35 +++++ packages/dashboard/src/mission-routes.ts | 9 +- .../__tests__/mission-autopilot-end-to-end.test.ts | 95 ++++++++++-- .../engine/src/__tests__/mission-autopilot.test.ts | 22 +++ .../engine/src/__tests__/mission-scheduler.test.ts | 159 +++++++++++---------- packages/engine/src/missions/mission-autopilot.ts | 50 +++---- packages/engine/src/scheduler.ts | 42 +----- 15 files changed, 432 insertions(+), 152 deletions(-) Fusion-Task-Id: FN-8824 Fusion-Task-Lineage: 22638d56-a51b-424c-aa07-871f544fb7cc Co-authored-by: Fusion (runfusion.ai) --- .../fn-8824-mission-slice-advancement.md | 7 + docs/missions.md | 10 +- ...mission-store.sync-loop-transition.test.ts | 43 +++++ .../postgres/mission-autopilot.pg.test.ts | 25 +++ .../src/async-stores/async-mission-store.ts | 48 +++++- packages/core/src/index.gate.ts | 1 + packages/core/src/index.ts | 1 + packages/core/src/missions/mission-store.ts | 37 +++- packages/core/src/missions/mission-types.ts | 35 ++++ packages/dashboard/src/mission-routes.ts | 9 +- .../mission-autopilot-end-to-end.test.ts | 95 ++++++++++- .../src/__tests__/mission-autopilot.test.ts | 22 +++ .../src/__tests__/mission-scheduler.test.ts | 159 ++++++++++-------- .../engine/src/missions/mission-autopilot.ts | 50 +++--- packages/engine/src/scheduler.ts | 42 +---- 15 files changed, 432 insertions(+), 152 deletions(-) create mode 100644 .changeset/fn-8824-mission-slice-advancement.md diff --git a/.changeset/fn-8824-mission-slice-advancement.md b/.changeset/fn-8824-mission-slice-advancement.md new file mode 100644 index 0000000000..c2c9d28e79 --- /dev/null +++ b/.changeset/fn-8824-mission-slice-advancement.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Keep mission autopilot slice progression serial and milestone ordered. +category: fix +dev: Duplicate completion and recovery signals now stop at the shared serial admission rule. diff --git a/docs/missions.md b/docs/missions.md index 1a523d2ca2..34b12a9293 100644 --- a/docs/missions.md +++ b/docs/missions.md @@ -329,7 +329,7 @@ Milestones now carry three complementary free-text fields: Slices represent staged execution windows. - Pending slices remain inactive -- Active slices are currently allowed to progress +- Automatic progression admits at most one active slice per mission - Completion rolls up through feature → slice → milestone → mission Manual activation is available through `fn mission activate-slice `. @@ -351,7 +351,7 @@ Typical flow: 1. Mission is watched (missions updated with `autopilotEnabled: true` or explicitly started are watched) 2. Task completion updates feature status -3. If a slice is complete, autopilot activates next pending slice +3. If no slice is active, autopilot activates only the earliest pending slice after every earlier milestone and slice is complete 4. When milestones are all complete, mission transitions to complete If validation cannot run (unexpected loop state, duplicate trigger, blocked validation, or validator error), Fusion logs a mission `warning`/`error` event with structured metadata so the stuck state is visible in mission events. @@ -372,8 +372,10 @@ Mission `status` and `autopilotEnabled` transitions are atomically written with **Slice progression (on slice completion):** -- `autopilotEnabled=true` → next pending slice is automatically activated -- `autopilotEnabled=false`, `autoAdvance=true` → next pending slice is activated (legacy compat) +- `autopilotEnabled=true` → serial admission activates only the earliest eligible pending slice. Any active slice blocks admission; earlier milestones and slices must be complete before later milestones start. +- Explicit milestone dependencies are additional restrictions and never override creation order. +- Duplicate completion callbacks and stale/startup recovery calls are idempotent no-ops when a slice is already active or no eligible slice exists. +- `autopilotEnabled=false`, `autoAdvance=true` → the legacy compatibility entry uses the same serial admission rule - `autopilotEnabled=false`, `autoAdvance=false` → manual activation required **Dashboard UI:** The Mission Manager groups mission run settings together: explicit **Start mission / Stop mission / Resume mission** actions control mission run-state, while the **Autopilot** toggle controls automatic slice advancement and feature planning. The autopilot badge uses human-readable states (`Off`, `Watching`, `Activating slice`, `Completing`). When enabling autopilot on an already-active mission, the system automatically checks whether recovery is needed (no active slice or completed active slice) and progresses accordingly. diff --git a/packages/core/src/__tests__/mission-store.sync-loop-transition.test.ts b/packages/core/src/__tests__/mission-store.sync-loop-transition.test.ts index 027d0a0b7b..82b5d8d991 100644 --- a/packages/core/src/__tests__/mission-store.sync-loop-transition.test.ts +++ b/packages/core/src/__tests__/mission-store.sync-loop-transition.test.ts @@ -42,6 +42,49 @@ describe("MissionStore synchronous loop transitions", () => { }); }); +describe("MissionStore serial slice admission", () => { + it("makes a sequential duplicate callback a no-op before events or triage", async () => { + let status: "pending" | "active" = "pending"; + const db = { + transaction: (callback: () => void) => callback(), + prepare: vi.fn(() => ({ + get: vi.fn().mockReturnValue(undefined), + all: vi.fn().mockReturnValue([]), + run: vi.fn(() => { + if (status !== "pending") return { changes: 0 }; + status = "active"; + return { changes: 1 }; + }), + })), + bumpLastModified: vi.fn(), + } as unknown as Database; + const store = new MissionStore("/tmp/fusion-mission-store-test", db); + const slice = { + id: "SL-ONE", + milestoneId: "MS-ONE", + title: "Only eligible slice", + status, + orderIndex: 0, + createdAt: "2026-08-08T00:00:00.000Z", + updatedAt: "2026-08-08T00:00:00.000Z", + } as const; + vi.spyOn(store, "getMissionWithHierarchy").mockImplementation(() => ({ + id: "M-ONE", + status: "active", + milestones: [{ id: "MS-ONE", status: "planning", orderIndex: 0, dependencies: [], slices: [{ ...slice, status, features: [] }] }], + }) as never); + vi.spyOn(store, "getMilestone").mockReturnValue({ id: "MS-ONE", missionId: "M-ONE" } as never); + vi.spyOn(store, "getMission").mockReturnValue({ id: "M-ONE", autopilotEnabled: false, autoAdvance: false } as never); + vi.spyOn(store as any, "recomputeMilestoneStatus").mockImplementation(() => undefined); + const activated = vi.fn(); + store.on("slice:activated", activated); + + expect((await store.tryActivateNextPendingSlice("M-ONE"))?.id).toBe("SL-ONE"); + expect(await store.tryActivateNextPendingSlice("M-ONE")).toBeUndefined(); + expect(activated).toHaveBeenCalledTimes(1); + }); +}); + describe("MissionStore synchronous assertion schema compatibility", () => { it("adds scope and origin before querying legacy assertion rows", () => { const executed: string[] = []; diff --git a/packages/core/src/__tests__/postgres/mission-autopilot.pg.test.ts b/packages/core/src/__tests__/postgres/mission-autopilot.pg.test.ts index f554f284cb..522c6d7790 100644 --- a/packages/core/src/__tests__/postgres/mission-autopilot.pg.test.ts +++ b/packages/core/src/__tests__/postgres/mission-autopilot.pg.test.ts @@ -107,6 +107,31 @@ pgTest("Mission autopilot loop (PostgreSQL backend mode)", () => { expect(autopilot.isWatching(mission.id)).toBe(false); }); + it("serially admits one slice under simultaneous PostgreSQL progression callbacks", async () => { + const m = missions(); + const mission = await createAutopilotMission(m, "Concurrent admission"); + const firstMilestone = await m.addMilestone(mission.id, { title: "M1" }); + const completed = await m.addSlice(firstMilestone.id, { title: "Complete source" }); + await m.updateSlice(completed.id, { status: "complete" }); + const secondMilestone = await m.addMilestone(mission.id, { title: "M2" }); + const firstPending = await m.addSlice(secondMilestone.id, { title: "First pending" }); + const laterPending = await m.addSlice(secondMilestone.id, { title: "Later pending" }); + await m.updateMission(mission.id, { status: "active", autopilotEnabled: false, autoAdvance: false }); + + const activationEvents: string[] = []; + m.on("slice:activated", (slice) => activationEvents.push(slice.id)); + const admissions = await Promise.all([ + m.tryActivateNextPendingSlice(mission.id), + m.tryActivateNextPendingSlice(mission.id), + ]); + + expect(admissions.filter(Boolean)).toHaveLength(1); + expect(admissions.find(Boolean)?.id).toBe(firstPending.id); + expect((await m.getSlice(firstPending.id))?.status).toBe("active"); + expect((await m.getSlice(laterPending.id))?.status).toBe("pending"); + expect(activationEvents).toEqual([firstPending.id]); + }); + it("recoverStaleMission runs the recover path (no scheduler) and persists a recovery event", async () => { const m = missions(); const mission = await createAutopilotMission(m, "Stale mission"); diff --git a/packages/core/src/async-stores/async-mission-store.ts b/packages/core/src/async-stores/async-mission-store.ts index ab8a5ac239..c53434cf95 100644 --- a/packages/core/src/async-stores/async-mission-store.ts +++ b/packages/core/src/async-stores/async-mission-store.ts @@ -14,7 +14,7 @@ import { EventEmitter } from "node:events"; import { and, desc, eq, inArray, notInArray, sql } from "drizzle-orm"; import * as schema from "../postgres/schema/index.js"; import type { AsyncDataLayer } from "../postgres/data-layer.js"; -import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType, renderValidationCause } from "../missions/mission-types.js"; +import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType, renderValidationCause, selectNextSerialMissionSlice } from "../missions/mission-types.js"; import type { Mission, Milestone, @@ -975,6 +975,52 @@ export class AsyncMissionStore extends EventEmitter { await reorderSlices(this.layer, orderedIds); } + /** + * FNXC:MissionSliceAdmission 2026-08-08-03:07: + * Automatic slice progression obtains one project-scoped advisory lock before + * selecting and claiming work. Duplicate completion and recovery callbacks + * therefore lose without publishing an activation or minting more tasks. + */ + async tryActivateNextPendingSlice(missionId: string): Promise { + const admitted = await this.layer.transactionImmediate(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtextextended( + CONCAT('mission-slice-admission:', COALESCE(NULLIF(current_setting('fusion.project_id', true), ''), '__legacy_unscoped__'), ':', CAST(${missionId} AS text)), + 0 + ))`); + const mission = await getMission(tx, missionId); + if (!mission) return undefined; + const milestones = await listMilestones(tx, missionId); + const hierarchy: MissionWithHierarchy = { + ...mission, + milestones: await Promise.all(milestones.map(async (milestone) => ({ + ...milestone, + slices: (await listSlices(tx, milestone.id)).map((slice) => ({ ...slice, features: [] })), + }))), + }; + const candidate = selectNextSerialMissionSlice(hierarchy); + if (!candidate) return undefined; + const now = new Date().toISOString(); + const updated: Slice = { ...candidate, status: "active", activatedAt: now, updatedAt: now }; + await updateSlice(tx, updated); + return updated; + }); + if (!admitted) return undefined; + + this.emit("slice:updated", admitted); + await this.recomputeMilestoneStatus(admitted.milestoneId); + const milestone = await getMilestone(this.db, admitted.milestoneId); + const mission = milestone ? await getMission(this.db, milestone.missionId) : undefined; + if (mission?.autopilotEnabled === true || mission?.autoAdvance === true) { + try { + await this.triageSlice(admitted.id); + } catch (err) { + severityAuditLog.error(`[AsyncMissionStore] Auto-triage failed for slice ${admitted.id}:`, err); + } + } + this.emit("slice:activated", admitted); + return admitted; + } + async activateSlice(id: string): Promise { const slice = await getSlice(this.db, id); if (!slice) throw new Error(`Slice ${id} not found`); diff --git a/packages/core/src/index.gate.ts b/packages/core/src/index.gate.ts index c268ccd7e5..85d938a610 100644 --- a/packages/core/src/index.gate.ts +++ b/packages/core/src/index.gate.ts @@ -1559,6 +1559,7 @@ export { VALIDATOR_RUN_STATUSES, VALIDATION_DIAGNOSTICS_MAX_EVIDENCE_PER_ASSERTION, VALIDATION_DIAGNOSTICS_MAX_TEXT_BYTES, + selectNextSerialMissionSlice, normalizeValidationDiagnostics, renderValidationFailureDescription, renderValidationCause, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d8a43e2ef7..8bd6459671 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1722,6 +1722,7 @@ export { VALIDATOR_RUN_STATUSES, VALIDATION_DIAGNOSTICS_MAX_EVIDENCE_PER_ASSERTION, VALIDATION_DIAGNOSTICS_MAX_TEXT_BYTES, + selectNextSerialMissionSlice, normalizeValidationDiagnostics, renderValidationFailureDescription, renderValidationCause, diff --git a/packages/core/src/missions/mission-store.ts b/packages/core/src/missions/mission-store.ts index 616d1ffdcc..2fe5fa63a9 100644 --- a/packages/core/src/missions/mission-store.ts +++ b/packages/core/src/missions/mission-store.ts @@ -17,7 +17,7 @@ const severityAuditLog = createLogger("core-mission-store"); import { EventEmitter } from "node:events"; import type { Database } from "../db/db.js"; import { fromJson, toJson, toJsonNullable } from "../db/db.js"; -import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionOrigin, normalizeMissionAssertionScope, normalizeMissionAssertionType, renderValidationCause } from "./mission-types.js"; +import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionOrigin, normalizeMissionAssertionScope, normalizeMissionAssertionType, renderValidationCause, selectNextSerialMissionSlice } from "./mission-types.js"; import type { Goal, GoalStatus } from "../goals/goal-types.js"; import type { Mission, @@ -2091,6 +2091,41 @@ export class MissionStore extends EventEmitter { * @returns The activated slice * @throws Error if slice not found */ + /** + * FNXC:MissionSliceAdmission 2026-08-08-03:07: + * Compatibility storage follows the PostgreSQL admission contract: decide + * and claim serial mission work atomically, then triage only the winner. + */ + async tryActivateNextPendingSlice(missionId: string): Promise { + let admitted: Slice | undefined; + this.db.transaction(() => { + const hierarchy = this.getMissionWithHierarchy(missionId); + const candidate = hierarchy ? selectNextSerialMissionSlice(hierarchy) : undefined; + if (!candidate) return; + const now = new Date().toISOString(); + const result = this.db.prepare("UPDATE slices SET status = ?, activatedAt = ?, updatedAt = ? WHERE id = ? AND status = 'pending'") + .run("active", now, now, candidate.id); + if (result.changes !== 1) return; + admitted = { ...candidate, status: "active", activatedAt: now, updatedAt: now }; + }); + if (!admitted) return undefined; + + this.db.bumpLastModified(); + this.emit("slice:updated", admitted); + this.recomputeMilestoneStatus(admitted.milestoneId); + const milestone = this.getMilestone(admitted.milestoneId); + const mission = milestone ? this.getMission(milestone.missionId) : undefined; + if (mission?.autopilotEnabled === true || mission?.autoAdvance === true) { + try { + await this.triageSlice(admitted.id); + } catch (err) { + severityAuditLog.error(`[MissionStore] Auto-triage failed for slice ${admitted.id}:`, err); + } + } + this.emit("slice:activated", admitted); + return admitted; + } + async activateSlice(id: string): Promise { const slice = this.getSlice(id); if (!slice) { diff --git a/packages/core/src/missions/mission-types.ts b/packages/core/src/missions/mission-types.ts index 158a9f5f8c..2e0062b0a1 100644 --- a/packages/core/src/missions/mission-types.ts +++ b/packages/core/src/missions/mission-types.ts @@ -678,6 +678,41 @@ export interface SliceWithFeatures extends Slice { * A Mission with complete hierarchy loaded: * Mission → Milestones → Slices → Features */ +/** + * FNXC:MissionSliceAdmission 2026-08-08-03:30: + * Automatic mission progression is serial: duplicate completion and recovery + * signals may admit only the first pending slice after every earlier slice is + * complete. Dependencies can further block admission, never bypass ordering. + * A blocked, empty, or stale non-complete milestone is likewise an ordered + * gate, so a later milestone cannot silently start before reconciliation. + */ +export function selectNextSerialMissionSlice(mission: MissionWithHierarchy): Slice | undefined { + if (mission.status !== "active") return undefined; + const milestones = [...mission.milestones].sort((a, b) => a.orderIndex - b.orderIndex); + if (milestones.some((milestone) => milestone.slices.some((slice) => slice.status === "active"))) return undefined; + + for (const milestone of milestones) { + if (milestone.status === "blocked") return undefined; + const dependenciesMet = milestone.dependencies.every((dependencyId) => + milestones.some((candidate) => candidate.id === dependencyId && candidate.status === "complete"), + ); + if (!dependenciesMet) return undefined; + const slices = [...milestone.slices].sort((a, b) => a.orderIndex - b.orderIndex); + // An empty planning milestone has no completed slice evidence that permits + // crossing it; only an explicitly complete empty milestone may be passed. + if (slices.length === 0) { + if (milestone.status !== "complete") return undefined; + continue; + } + for (const slice of slices) { + if (slice.status === "complete") continue; + return slice.status === "pending" ? slice : undefined; + } + if (milestone.status !== "complete") return undefined; + } + return undefined; +} + export interface MissionWithHierarchy extends Mission { /** Goals linked to this mission */ linkedGoals?: Goal[]; diff --git a/packages/dashboard/src/mission-routes.ts b/packages/dashboard/src/mission-routes.ts index 7a69ba7ec0..bfd41627ef 100644 --- a/packages/dashboard/src/mission-routes.ts +++ b/packages/dashboard/src/mission-routes.ts @@ -3102,8 +3102,8 @@ export function createMissionRouter( throw conflict("Mission must be in 'planning' status to start"); } - const nextSlice = await missionStore.findNextPendingSlice(missionId); - if (!nextSlice) { + const initialHierarchy = await missionStore.getMissionWithHierarchy(missionId); + if (!initialHierarchy?.milestones.some((milestone) => milestone.slices.some((slice) => slice.status === "pending"))) { throw badRequest("No pending slices found"); } @@ -3115,8 +3115,9 @@ export function createMissionRouter( status: "active", }, { actor: DASHBOARD_MISSION_ACTOR }); - // Activate the first pending slice (triggers auto-triage via activateSlice) - await missionStore.activateSlice(nextSlice.id); + // Atomically admit the first serially eligible slice. A concurrent resume + // or recovery winner has already created the only permitted active slice. + await missionStore.tryActivateNextPendingSlice(missionId); // Return updated mission with hierarchy const hierarchy = await missionStore.getMissionWithHierarchy(missionId); diff --git a/packages/engine/src/__tests__/mission-autopilot-end-to-end.test.ts b/packages/engine/src/__tests__/mission-autopilot-end-to-end.test.ts index 587d41ff2a..c5a871a262 100644 --- a/packages/engine/src/__tests__/mission-autopilot-end-to-end.test.ts +++ b/packages/engine/src/__tests__/mission-autopilot-end-to-end.test.ts @@ -2,17 +2,20 @@ import { describe, it, expect, vi } from "vitest"; import { Scheduler } from "../scheduler.js"; import { MissionAutopilot } from "../missions/mission-autopilot.js"; import { MissionExecutionLoop } from "../missions/mission-execution-loop.js"; -import type { TaskStore } from "@fusion/core"; +import { selectNextSerialMissionSlice, type TaskStore } from "@fusion/core"; function makeHarness({ withAssertions = true, initialFeatureStatus = "in-progress", initialTaskColumn = "in-progress", + productionSequence = false, runValidationImpl, }: { withAssertions?: boolean; initialFeatureStatus?: string; initialTaskColumn?: string; + /** Mirror #3345: M1 → M2 (two slices) → M3 without dependencies. */ + productionSequence?: boolean; runValidationImpl?: () => Promise<{ status: "pass" | "error"; assertions: []; summary: string }>; } = {}) { const mission = { @@ -37,7 +40,15 @@ function makeHarness({ createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }; - const slices = new Map([ + const secondMilestone = { ...milestone, id: "MS-002", title: "Milestone 2", status: "planning", orderIndex: 1 }; + const thirdMilestone = { ...milestone, id: "MS-003", title: "Milestone 3", status: "planning", orderIndex: 2 }; + const milestones = new Map([[milestone.id, milestone], ...(productionSequence ? [[secondMilestone.id, secondMilestone], [thirdMilestone.id, thirdMilestone]] : [])]); + const slices = new Map(productionSequence ? [ + ["SL-001", { id: "SL-001", milestoneId: milestone.id, title: "M1 source", status: "active", planState: "not_started", orderIndex: 0, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }], + ["SL-002", { id: "SL-002", milestoneId: secondMilestone.id, title: "M2 first", status: "pending", planState: "not_started", orderIndex: 0, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }], + ["SL-003", { id: "SL-003", milestoneId: secondMilestone.id, title: "M2 second", status: "pending", planState: "not_started", orderIndex: 1, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }], + ["SL-004", { id: "SL-004", milestoneId: thirdMilestone.id, title: "M3 first", status: "pending", planState: "not_started", orderIndex: 0, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }], + ] : [ ["SL-001", { id: "SL-001", milestoneId: milestone.id, title: "Slice 1", status: "active", planState: "not_started", orderIndex: 0, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }], ["SL-002", { id: "SL-002", milestoneId: milestone.id, title: "Slice 2", status: "pending", planState: "not_started", orderIndex: 1, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }], ]); @@ -95,7 +106,7 @@ function makeHarness({ listMissions: vi.fn(() => [mission]), updateMission: vi.fn((_id: string, updates: any) => ({ ...mission, ...updates })), logMissionEvent: vi.fn(), - getMilestone: vi.fn((id: string) => (id === milestone.id ? milestone : undefined)), + getMilestone: vi.fn((id: string) => milestones.get(id)), getSlice: vi.fn((id: string) => slices.get(id)), updateSlice: vi.fn((id: string, updates: any) => { const next = { ...slices.get(id), ...updates, updatedAt: new Date().toISOString() }; @@ -104,7 +115,12 @@ function makeHarness({ }), getMissionWithHierarchy: vi.fn(() => ({ ...mission, - milestones: [{ ...milestone, slices: [...slices.values()].map((slice) => ({ ...slice, features: [...features.values()].filter((f) => f.sliceId === slice.id) })) }], + milestones: [...milestones.values()].map((currentMilestone) => ({ + ...currentMilestone, + slices: [...slices.values()] + .filter((slice) => slice.milestoneId === currentMilestone.id) + .map((slice) => ({ ...slice, features: [...features.values()].filter((f) => f.sliceId === slice.id) })), + })), })), listSlices: vi.fn(() => [...slices.values()]), listFeatures: vi.fn((sliceId?: string) => [...features.values()].filter((f) => !sliceId || f.sliceId === sliceId)), @@ -121,6 +137,13 @@ function makeHarness({ .every((f) => f.status === "done"); if (slice && done && slice.status !== "complete") { slices.set(slice.id, { ...slice, status: "complete", updatedAt: new Date().toISOString() }); + const sliceMilestone = milestones.get(slice.milestoneId); + const milestoneComplete = [...slices.values()] + .filter((candidate) => candidate.milestoneId === slice.milestoneId) + .every((candidate) => candidate.status === "complete"); + if (sliceMilestone && milestoneComplete) { + milestones.set(sliceMilestone.id, { ...sliceMilestone, status: "complete" }); + } } return next; }), @@ -154,11 +177,15 @@ function makeHarness({ missionExecutionLoop: loop, pollIntervalMs: 60_000, }); + const admittedIds: string[] = []; const activateSpy = vi.spyOn(scheduler, "activateNextPendingSlice").mockImplementation(async (missionId: string) => { if (missionId !== mission.id) return null; - const next = slices.get("SL-002"); - slices.set("SL-002", { ...next, status: "active", updatedAt: new Date().toISOString() }); - return slices.get("SL-002"); + const candidate = selectNextSerialMissionSlice(missionStore.getMissionWithHierarchy(missionId)); + if (!candidate) return null; + const activated = { ...slices.get(candidate.id), status: "active", updatedAt: new Date().toISOString() }; + slices.set(candidate.id, activated); + admittedIds.push(candidate.id); + return activated; }); const emitTaskMoved = async (to: string) => { @@ -169,7 +196,7 @@ function makeHarness({ scheduler.start(); - return { missionStore, loop, autopilot, scheduler, emitTaskMoved, slices, feature, activateSpy }; + return { missionStore, loop, autopilot, scheduler, emitTaskMoved, milestones, slices, feature, activateSpy, admittedIds }; } describe("mission autopilot end-to-end wiring", () => { @@ -190,6 +217,58 @@ describe("mission autopilot end-to-end wiring", () => { h.scheduler.stop(); }); + it("uses the watched-autopilot completion bridge only once for duplicate source completion", async () => { + const h = makeHarness({ withAssertions: false }); + await h.autopilot.watchMission("M-001"); + h.missionStore.updateFeatureStatus("F-001", "done"); + + // Drive the same scheduler bridge twice, matching a done/archived-equivalent + // duplicate completion callback after the source slice already completed. + await (h.scheduler as any).handleMissionTaskCompletion("FN-001", "SL-001"); + await (h.scheduler as any).handleMissionTaskCompletion("FN-001", "SL-001"); + + expect(h.autopilot.isWatching("M-001")).toBe(true); + expect(h.admittedIds).toEqual(["SL-002"]); + expect(h.slices.get("SL-002").status).toBe("active"); + h.scheduler.stop(); + }); + + it("keeps the #3345 M1 → M2 → M3 progression serial across duplicate completion and recovery", async () => { + const h = makeHarness({ withAssertions: false, productionSequence: true }); + await h.autopilot.watchMission("M-001"); + h.missionStore.updateFeatureStatus("F-001", "done"); + + await Promise.all([ + (h.scheduler as any).handleMissionTaskCompletion("FN-001", "SL-001"), + (h.scheduler as any).handleMissionTaskCompletion("FN-001", "SL-001"), + h.autopilot.recoverStaleMission("M-001"), + ]); + + expect(h.admittedIds).toEqual(["SL-002"]); + expect(h.slices.get("SL-002").status).toBe("active"); + expect(h.slices.get("SL-003").status).toBe("pending"); + expect(h.slices.get("SL-004").status).toBe("pending"); + + h.slices.set("SL-002", { ...h.slices.get("SL-002"), status: "complete" }); + await h.autopilot.advanceToNextSlice("M-001"); + expect(h.admittedIds).toEqual(["SL-002", "SL-003"]); + expect(h.slices.get("SL-003").status).toBe("active"); + expect(h.slices.get("SL-004").status).toBe("pending"); + + await Promise.all([ + h.autopilot.advanceToNextSlice("M-001"), + h.autopilot.recoverStaleMission("M-001"), + ]); + expect(h.slices.get("SL-004").status).toBe("pending"); + + h.slices.set("SL-003", { ...h.slices.get("SL-003"), status: "complete" }); + h.milestones.set("MS-002", { ...h.milestones.get("MS-002"), status: "complete" }); + await h.autopilot.advanceToNextSlice("M-001"); + expect(h.admittedIds).toEqual(["SL-002", "SL-003", "SL-004"]); + expect(h.slices.get("SL-004").status).toBe("active"); + h.scheduler.stop(); + }); + it("advances slices in no-assertions pass path", async () => { const h = makeHarness({ withAssertions: false }); diff --git a/packages/engine/src/__tests__/mission-autopilot.test.ts b/packages/engine/src/__tests__/mission-autopilot.test.ts index b655952245..c441ee5575 100644 --- a/packages/engine/src/__tests__/mission-autopilot.test.ts +++ b/packages/engine/src/__tests__/mission-autopilot.test.ts @@ -1329,6 +1329,28 @@ describe("MissionAutopilot", () => { expect(localScheduler.activateNextPendingSlice).toHaveBeenCalledWith(mission.id); }); + it("does not advance recovery when legacy corruption leaves another active slice incomplete", async () => { + const mission = createMockMission(); + const store = createMockMissionStore([mission]); + const localScheduler = createMockScheduler(); + const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler: localScheduler }); + store.getMissionWithHierarchy.mockReturnValue({ + ...mission, + milestones: [{ + ...createMockMilestone({ missionId: mission.id }), + slices: [ + { ...createMockSlice({ id: "SL-DONE", status: "active" }), features: [createMockFeature({ id: "F-DONE", status: "done" })] }, + { ...createMockSlice({ id: "SL-ACTIVE", status: "active" }), features: [createMockFeature({ id: "F-ACTIVE", status: "in-progress" })] }, + ], + }], + }); + + await ap.recoverMissions(store as any); + + expect(localScheduler.activateNextPendingSlice).not.toHaveBeenCalled(); + expect(store.updateSlice).not.toHaveBeenCalledWith("SL-DONE", { status: "complete" }); + }); + it("handles empty mission lists", async () => { const store = createMockMissionStore([]); const ap = new MissionAutopilot(taskStore as any, store as any, { scheduler }); diff --git a/packages/engine/src/__tests__/mission-scheduler.test.ts b/packages/engine/src/__tests__/mission-scheduler.test.ts index 1c14d3a784..e2e1fbc607 100644 --- a/packages/engine/src/__tests__/mission-scheduler.test.ts +++ b/packages/engine/src/__tests__/mission-scheduler.test.ts @@ -11,13 +11,14 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { Scheduler } from "../scheduler.js"; import { AgentSemaphore } from "../concurrency/concurrency.js"; -import type { TaskStore, MissionStore, Slice, Mission, Milestone, MissionFeature } from "@fusion/core"; +import { selectNextSerialMissionSlice, type TaskStore, type MissionStore, type Slice, type Mission, type Milestone, type MissionFeature } from "@fusion/core"; // Mock store factory function createMockMissionStore(): any { return { findNextPendingSlice: vi.fn(), activateSlice: vi.fn(), + tryActivateNextPendingSlice: vi.fn(), getSlice: vi.fn(), getMilestone: vi.fn(), getMission: vi.fn(), @@ -104,6 +105,36 @@ function createMockFeature(overrides: Partial = {}): MissionFeat } as MissionFeature; } +describe("serial mission slice admission decision", () => { + const slice = (id: string, status: Slice["status"], orderIndex = 0) => ({ + ...createMockSlice({ id, status, orderIndex }), + features: [], + }); + const milestone = (id: string, orderIndex: number, status: Milestone["status"], slices: ReturnType[], dependencies: string[] = []) => ({ + ...createMockMilestone({ id, orderIndex, status, dependencies }), + slices, + }); + + it.each([ + ["empty planning hierarchy", [milestone("MS-1", 0, "planning", [])], undefined], + ["all complete hierarchy", [milestone("MS-1", 0, "complete", [slice("SL-1", "complete")])], undefined], + ["an active slice lease", [milestone("MS-1", 0, "active", [slice("SL-1", "active"), slice("SL-2", "pending", 1)])], undefined], + ["the first pending slice", [milestone("MS-1", 0, "planning", [slice("SL-1", "pending")])], "SL-1"], + ["a blocked earlier milestone", [milestone("MS-1", 0, "blocked", [slice("SL-1", "complete")]), milestone("MS-2", 1, "planning", [slice("SL-2", "pending")])], undefined], + ["an unmet explicit dependency", [milestone("MS-1", 0, "planning", [slice("SL-1", "pending")], ["MS-missing"])], undefined], + ["an empty earlier planning milestone", [milestone("MS-1", 0, "planning", []), milestone("MS-2", 1, "planning", [slice("SL-2", "pending")])], undefined], + ["the next milestone after an earlier complete milestone", [milestone("MS-1", 0, "complete", [slice("SL-1", "complete")]), milestone("MS-2", 1, "planning", [slice("SL-2", "pending")])], "SL-2"], + ["a stale non-complete earlier milestone", [milestone("MS-1", 0, "active", [slice("SL-1", "complete")]), milestone("MS-2", 1, "planning", [slice("SL-2", "pending")])], undefined], + ])("returns %s", (_name, milestones, expectedId) => { + const result = selectNextSerialMissionSlice({ + ...createMockMission(), + milestones, + } as any); + + expect(result?.id).toBe(expectedId); + }); +}); + describe("Scheduler Mission Integration", () => { let taskStore: any; let missionStore: any; @@ -138,88 +169,29 @@ describe("Scheduler Mission Integration", () => { it("should find and activate next pending slice", async () => { const mockActivated = createMockSlice({ id: "SL-002", status: "active" }); - missionStore.getMissionWithHierarchy.mockReturnValue({ - id: "M-001", - status: "active", - milestones: [ - { - id: "MS-001", - orderIndex: 0, - dependencies: [], - slices: [ - { id: "SL-001", status: "complete", orderIndex: 0 }, - { id: "SL-002", status: "pending", orderIndex: 1 }, - ], - }, - ], - }); - missionStore.activateSlice.mockReturnValue(mockActivated); + missionStore.tryActivateNextPendingSlice.mockResolvedValue(mockActivated); const result = await scheduler.activateNextPendingSlice("M-001"); - expect(missionStore.getMissionWithHierarchy).toHaveBeenCalledWith("M-001"); - expect(missionStore.activateSlice).toHaveBeenCalledWith("SL-002"); + expect(missionStore.tryActivateNextPendingSlice).toHaveBeenCalledWith("M-001"); expect(result).toEqual(mockActivated); }); - it("skips milestones with unmet dependencies and activates the next eligible pending slice", async () => { - const mockActivated = createMockSlice({ id: "SL-ELIGIBLE", status: "active" }); - - missionStore.getMissionWithHierarchy.mockReturnValue({ - id: "M-001", - status: "active", - milestones: [ - { - id: "MS-BLOCKED", - orderIndex: 0, - dependencies: ["MS-DEP"], - status: "planning", - slices: [{ id: "SL-BLOCKED", status: "pending", orderIndex: 0 }], - }, - { - id: "MS-DEP", - orderIndex: 1, - dependencies: [], - status: "planning", - slices: [{ id: "SL-DEP", status: "complete", orderIndex: 0 }], - }, - { - id: "MS-ELIGIBLE", - orderIndex: 2, - dependencies: [], - status: "active", - slices: [{ id: "SL-ELIGIBLE", status: "pending", orderIndex: 0 }], - }, - ], - }); - missionStore.activateSlice.mockReturnValue(mockActivated); + it("returns a concurrency-safe no-op when no serial candidate is admitted", async () => { + missionStore.tryActivateNextPendingSlice.mockResolvedValue(undefined); const result = await scheduler.activateNextPendingSlice("M-001"); - expect(missionStore.activateSlice).toHaveBeenCalledWith("SL-ELIGIBLE"); - expect(result).toEqual(mockActivated); + expect(missionStore.tryActivateNextPendingSlice).toHaveBeenCalledWith("M-001"); + expect(result).toBeNull(); }); it("should return null when no pending slices", async () => { - missionStore.getMissionWithHierarchy.mockReturnValue({ - id: "M-001", - status: "active", - milestones: [ - { - id: "MS-001", - orderIndex: 0, - dependencies: [], - slices: [ - { id: "SL-001", status: "complete", orderIndex: 0 }, - { id: "SL-002", status: "complete", orderIndex: 1 }, - ], - }, - ], - }); + missionStore.tryActivateNextPendingSlice.mockResolvedValue(undefined); const result = await scheduler.activateNextPendingSlice("M-001"); - expect(missionStore.getMissionWithHierarchy).toHaveBeenCalledWith("M-001"); + expect(missionStore.tryActivateNextPendingSlice).toHaveBeenCalledWith("M-001"); expect(result).toBeNull(); }); @@ -237,7 +209,7 @@ describe("Scheduler Mission Integration", () => { }); it("should handle errors gracefully", async () => { - missionStore.getMissionWithHierarchy.mockImplementation(() => { + missionStore.tryActivateNextPendingSlice.mockImplementation(() => { throw new Error("Database error"); }); @@ -273,11 +245,11 @@ describe("Scheduler Mission Integration", () => { ], }], }); - missionStore.activateSlice.mockResolvedValue(createMockSlice({ id: "SL-002", status: "active" })); + missionStore.tryActivateNextPendingSlice.mockResolvedValue(createMockSlice({ id: "SL-002", status: "active" })); await scheduler.onSliceComplete(completedSlice); - expect(missionStore.activateSlice).toHaveBeenCalledWith("SL-002"); + expect(missionStore.tryActivateNextPendingSlice).toHaveBeenCalledWith("M-001"); }); it("auto-advances when autoAdvance is true (legacy compat)", async () => { @@ -305,11 +277,52 @@ describe("Scheduler Mission Integration", () => { ], }], }); - missionStore.activateSlice.mockResolvedValue(createMockSlice({ id: "SL-002", status: "active" })); + missionStore.tryActivateNextPendingSlice.mockResolvedValue(createMockSlice({ id: "SL-002", status: "active" })); await scheduler.onSliceComplete(completedSlice); - expect(missionStore.activateSlice).toHaveBeenCalledWith("SL-002"); + expect(missionStore.tryActivateNextPendingSlice).toHaveBeenCalledWith("M-001"); + }); + + it("keeps the #3345 M1 → M2 → M3 completion sequence serial across duplicate callbacks", async () => { + const source = createMockSlice({ id: "SL-M1", milestoneId: "MS-1", status: "complete" }); + const firstM2 = createMockSlice({ id: "SL-M2-A", milestoneId: "MS-2", status: "pending", orderIndex: 0 }); + const secondM2 = createMockSlice({ id: "SL-M2-B", milestoneId: "MS-2", status: "pending", orderIndex: 1 }); + const firstM3 = createMockSlice({ id: "SL-M3-A", milestoneId: "MS-3", status: "pending", orderIndex: 0 }); + const milestones = [ + { ...createMockMilestone({ id: "MS-1", missionId: "M-001", orderIndex: 0, status: "complete" }), slices: [source] }, + { ...createMockMilestone({ id: "MS-2", missionId: "M-001", orderIndex: 1 }), slices: [firstM2, secondM2] }, + { ...createMockMilestone({ id: "MS-3", missionId: "M-001", orderIndex: 2 }), slices: [firstM3] }, + ]; + const mission = createMockMission({ id: "M-001", autopilotEnabled: true, autoAdvance: true }); + const activated: string[] = []; + missionStore.getMilestone.mockImplementation((id: string) => milestones.find((milestone) => milestone.id === id)); + missionStore.getMission.mockReturnValue(mission); + missionStore.tryActivateNextPendingSlice.mockImplementation(async () => { + const candidate = selectNextSerialMissionSlice({ ...mission, milestones: milestones.map((milestone) => ({ ...milestone, slices: milestone.slices.map((slice) => ({ ...slice, features: [] })) })) } as any); + if (!candidate) return undefined; + const slice = milestones.flatMap((milestone) => milestone.slices).find((item) => item.id === candidate.id)!; + slice.status = "active"; + activated.push(slice.id); + return slice; + }); + + await scheduler.onSliceComplete(source); + await scheduler.onSliceComplete(source); // done/archived-equivalent duplicate callback + expect(activated).toEqual([firstM2.id]); + expect(secondM2.status).toBe("pending"); + expect(firstM3.status).toBe("pending"); + + firstM2.status = "complete"; + await scheduler.onSliceComplete(firstM2); + expect(activated).toEqual([firstM2.id, secondM2.id]); + await scheduler.onSliceComplete(source); + expect(firstM3.status).toBe("pending"); + + secondM2.status = "complete"; + milestones[1].status = "complete"; + await scheduler.onSliceComplete(secondM2); + expect(activated).toEqual([firstM2.id, secondM2.id, firstM3.id]); }); it("does not auto-advance when both autopilotEnabled and autoAdvance are false", async () => { diff --git a/packages/engine/src/missions/mission-autopilot.ts b/packages/engine/src/missions/mission-autopilot.ts index 20bb82959f..4a30d9ff7a 100644 --- a/packages/engine/src/missions/mission-autopilot.ts +++ b/packages/engine/src/missions/mission-autopilot.ts @@ -710,26 +710,22 @@ export class MissionAutopilot { const activeSlices = refreshedMission.milestones.flatMap((milestone) => milestone.slices) .filter((slice) => slice.status === "active"); + // The scheduler admission boundary treats every active slice as a lease. + // Recovery may reconcile completed leases, but one completed-looking lease + // must never bypass another active slice. + const allActiveSlicesComplete = activeSlices.length > 0 && activeSlices.every((slice) => + slice.features.length > 0 && slice.features.every((feature) => feature.status === "done"), + ); + if (allActiveSlicesComplete) { + await Promise.all(activeSlices.map((slice) => this.missionStore.updateSlice(slice.id, { status: "complete" }))); + } + // A stale snapshot may still show the slices as active after the status + // reconciliation above; only the all-complete case releases that lease. + const hasActiveSlice = activeSlices.length > 0 && !allActiveSlicesComplete; let advanced = false; - - if (activeSlices.length > 0) { - const hasCompletedActiveSlice = activeSlices.some((slice) => - slice.features.length > 0 && slice.features.every((feature) => feature.status === "done"), - ); - - if (hasCompletedActiveSlice) { - await this.advanceToNextSlice(missionId); - advanced = true; - } - } else { - const hasPendingSlice = refreshedMission.milestones.some((milestone) => - milestone.slices.some((slice) => slice.status === "pending"), - ); - - if (hasPendingSlice) { - await this.advanceToNextSlice(missionId); - advanced = true; - } + if (!hasActiveSlice) { + await this.advanceToNextSlice(missionId); + advanced = true; } await this.logMissionEventSafe( @@ -850,12 +846,18 @@ export class MissionAutopilot { continue; } - const hasCompletedActiveSlice = refreshedHierarchy.milestones + const activeSlices = refreshedHierarchy.milestones .flatMap((milestone) => milestone.slices) - .filter((slice) => slice.status === "active") - .some((slice) => slice.features.length > 0 && slice.features.every((feature) => feature.status === "done")); - - if (hasCompletedActiveSlice) { + .filter((slice) => slice.status === "active"); + const allActiveSlicesComplete = activeSlices.length > 0 && activeSlices.every((slice) => + slice.features.length > 0 && slice.features.every((feature) => feature.status === "done"), + ); + if (allActiveSlicesComplete) { + // Reconcile every finished lease before asking the shared admission + // boundary for the next serial slice. + await Promise.all(activeSlices.map((slice) => this.missionStore.updateSlice(slice.id, { status: "complete" }))); + } + if (activeSlices.length === 0 || allActiveSlicesComplete) { await this.advanceToNextSlice(mission.id); } } diff --git a/packages/engine/src/scheduler.ts b/packages/engine/src/scheduler.ts index d7eed65fe6..87ef941534 100644 --- a/packages/engine/src/scheduler.ts +++ b/packages/engine/src/scheduler.ts @@ -3256,17 +3256,6 @@ export class Scheduler { return; } - const missionHierarchy = await missionStore.getMissionWithHierarchy(mission.id); - const hasActiveSlice = missionHierarchy?.milestones.some((candidateMilestone) => - candidateMilestone.slices.some((candidateSlice) => - candidateSlice.id !== slice.id && candidateSlice.status === "active" - ) - ); - if (hasActiveSlice) { - schedulerLog.log(`Mission ${mission.id} already has an active slice; skipping auto-advance`); - return; - } - const nextSlice = await this.activateNextPendingSlice(mission.id); if (nextSlice) { schedulerLog.log(`Auto-advanced: activated slice ${nextSlice.id} for mission ${mission.id}`); @@ -3290,36 +3279,15 @@ export class Scheduler { const missionStore = this.options.missionStore; try { - const mission = await missionStore.getMissionWithHierarchy(missionId); - if (!mission || mission.status !== "active") { - schedulerLog.log(`Mission ${missionId}: not active, skipping slice activation`); - return null; - } - - const sortedMilestones = [...mission.milestones].sort((a, b) => a.orderIndex - b.orderIndex); - - for (const milestone of sortedMilestones) { - const dependenciesMet = milestone.dependencies.every((dependencyId) => { - const dependency = mission.milestones.find((candidate) => candidate.id === dependencyId); - return dependency?.status === "complete"; - }); - if (!dependenciesMet) { - continue; - } - - const pendingSlice = [...milestone.slices] - .sort((a, b) => a.orderIndex - b.orderIndex) - .find((slice) => slice.status === "pending"); - if (!pendingSlice) { - continue; - } - - const activated = await missionStore.activateSlice(pendingSlice.id); + // The store atomically re-reads the hierarchy and claims the candidate. + // A duplicate signal is an expected no-op, not a scheduler error. + const activated = await missionStore.tryActivateNextPendingSlice(missionId); + if (activated) { schedulerLog.log(`Activated slice ${activated.id} for mission ${missionId}`); return activated; } - schedulerLog.log(`Mission ${missionId}: no pending slices to activate`); + schedulerLog.log(`Mission ${missionId}: no serially eligible slice to activate`); return null; } catch (err) { schedulerLog.error(`Error activating next slice for mission ${missionId}:`, err);