From 90573e31b0c9f053d97027db267e61b39b7e2383 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Mon, 10 Aug 2026 21:41:47 -0700 Subject: [PATCH] FN-8963: prevent concurrent mission validation runs Prevent duplicate manual validation runs while preserving recovery from stale validator state. - Add atomic mission-store claims and release handling for manual validator runs. - Return a conflict response and disable the validation action while a run is active. - Cover concurrent claims, API behavior, UI state, and stale-window parity. Files changed: .changeset/fn-8963-validate-inflight-guard.md | 7 ++ docs/missions.md | 2 + .../__tests__/postgres/mission-store.pg.test.ts | 129 +++++++++++++++++++++ .../core/src/async-stores/async-mission-store.ts | 87 ++++++++++++-- packages/core/src/index.ts | 2 + packages/core/src/missions/mission-store.ts | 69 ++++++++++- packages/core/src/missions/mission-types.ts | 13 +++ packages/dashboard/app/api/legacy.ts | 2 + packages/dashboard/app/api/missions/missions.ts | 11 +- .../dashboard/app/components/MissionManager.tsx | 15 ++- .../MissionManager.validate-inflight.test.tsx | 77 ++++++++++++ .../mission-validate-inflight-guard.test.ts | 55 +++++++++ packages/dashboard/src/mission-routes.ts | 32 +--- .../src/__tests__/mission-execution-loop.test.ts | 61 +++++++++- .../validator-run-stale-window-parity.test.ts | 9 ++ 15 files changed, 549 insertions(+), 22 deletions(-) Fusion-Task-Id: FN-8963 Fusion-Task-Lineage: 7e5d0ae4-98ac-4f79-99ca-45fd1450362d Co-authored-by: Fusion (runfusion.ai) --- .changeset/fn-8963-validate-inflight-guard.md | 7 + docs/missions.md | 2 + .../postgres/mission-store.pg.test.ts | 129 ++++++++++++++++++ .../src/async-stores/async-mission-store.ts | 89 ++++++++++-- packages/core/src/index.ts | 2 + packages/core/src/missions/mission-store.ts | 69 +++++++++- packages/core/src/missions/mission-types.ts | 13 ++ packages/dashboard/app/api/legacy.ts | 2 + .../dashboard/app/api/missions/missions.ts | 11 +- .../app/components/MissionManager.tsx | 15 +- .../MissionManager.validate-inflight.test.tsx | 77 +++++++++++ .../mission-validate-inflight-guard.test.ts | 55 ++++++++ packages/dashboard/src/mission-routes.ts | 32 +++-- .../__tests__/mission-execution-loop.test.ts | 61 ++++++++- .../validator-run-stale-window-parity.test.ts | 9 ++ 15 files changed, 550 insertions(+), 23 deletions(-) create mode 100644 .changeset/fn-8963-validate-inflight-guard.md create mode 100644 packages/dashboard/app/components/__tests__/MissionManager.validate-inflight.test.tsx create mode 100644 packages/dashboard/src/__tests__/mission-validate-inflight-guard.test.ts create mode 100644 packages/engine/src/__tests__/validator-run-stale-window-parity.test.ts diff --git a/.changeset/fn-8963-validate-inflight-guard.md b/.changeset/fn-8963-validate-inflight-guard.md new file mode 100644 index 0000000000..2e477151cd --- /dev/null +++ b/.changeset/fn-8963-validate-inflight-guard.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Prevent duplicate manual mission validation runs for a feature. +category: fix +dev: Adds startManualValidatorRun and the 409 VALIDATION_ALREADY_RUNNING API contract. diff --git a/docs/missions.md b/docs/missions.md index 75990a6af3..e295a600c3 100644 --- a/docs/missions.md +++ b/docs/missions.md @@ -602,6 +602,8 @@ interface MissionValidatorRun { **Validation timeout:** 10 minutes (`VALIDATION_TIMEOUT_MS = 10 * 60 * 1000`). If session creation, auth/credit checks, prompting, or timeout fails, the run is marked `error` and emits a surfaced `validation_error` mission event instead of silently spawning a fix feature. +**Manual validation admission:** `POST /api/missions/features/:featureId/validate` admits manual runs atomically per feature. A fresh `running` run, including an engine-started run, returns `409` with `details: { code: "VALIDATION_ALREADY_RUNNING", runId, featureId, startedAt }` and does not mutate the feature. The guard ignores runs older than the stale window. Automatic `admitValidatorRun` remains fingerprint-scoped: a fingerprint-less manual run does not suppress a later automatic dispatch (tracked by FN-8976). + **Stale validator-run reaper:** startup recovery and periodic self-healing also sweep `MissionValidatorRun` rows stuck in `status="running"` longer than `VALIDATOR_RUN_STALE_MAX_AGE_MS` (currently 6 hours). Runs still owned by the live process (tracked in `activeValidations`) are skipped, so a slow-but-legitimate verification is never reaped while its session is in-flight. Ownerless stale runs are reaped to terminal `status="error"`, their reap reason is stored in `summary`, and live (non-`done`) mission features are moved to `loopState="needs_fix"` with `lastValidatorStatus="error"` so the loop can re-trigger. A *done* feature's loop state is intentionally left untouched (it keeps the `loopState="validating"` set when the run started) so the reaper does not rewrite a feature that already finished its task. Runs whose parent mission is already `complete`/`archived` are likewise terminated without touching feature state. Each successful reap emits a run-audit event with `mutationType: "mission:validator-run-reaped"`. **Verification wall-clock is bounded under the reaper window.** The aggregate verification budget — checkout materialization plus the test-suite command (`VERIFICATION_COMMAND_TIMEOUT_MS`, 10 min), including the optional pre-fix baseline run — is provably far shorter than the 6-hour reaper stale window, so a legitimate verification run completes long before it would be eligible for reaping. The reaper's `activeValidations` skip is the second line of defense: an in-flight run is never reaped regardless of wall-clock. diff --git a/packages/core/src/__tests__/postgres/mission-store.pg.test.ts b/packages/core/src/__tests__/postgres/mission-store.pg.test.ts index 6cdd5588c8..2e1c237531 100644 --- a/packages/core/src/__tests__/postgres/mission-store.pg.test.ts +++ b/packages/core/src/__tests__/postgres/mission-store.pg.test.ts @@ -865,6 +865,135 @@ pgTest("MissionStore (PostgreSQL backend mode)", () => { expect(fetched?.inputFingerprint).toBe("fingerprint-round-trip"); }); + it("atomically admits one fresh manual validator run and preserves rejected feature state", async () => { + const m = missions(); + const mission = await m.createMission({ title: "Manual admission" }); + const milestone = await m.addMilestone(mission.id, { title: "MS" }); + const slice = await m.addSlice(milestone.id, { title: "SL" }); + const feature = await m.addFeature(slice.id, { title: "F" }); + + const [first, second] = await Promise.all([ + m.startManualValidatorRun(feature.id), + m.startManualValidatorRun(feature.id), + ]); + const started = first.outcome === "started" ? first : second; + const blocked = first.outcome === "already-running" ? first : second; + expect(started).toMatchObject({ outcome: "started" }); + expect(blocked).toMatchObject({ outcome: "already-running", run: { id: started.run.id } }); + expect((await m.getValidatorRunsByFeature(feature.id)).filter((run) => run.status === "running")).toHaveLength(1); + const beforeRejected = await m.getFeature(feature.id); + await expect(m.startManualValidatorRun(feature.id)).resolves.toMatchObject({ outcome: "already-running", run: { id: started.run.id } }); + expect(await m.getFeature(feature.id)).toEqual(beforeRejected); + + await m.completeValidatorRun(started.run.id, "passed"); + await expect(m.startManualValidatorRun(feature.id)).resolves.toMatchObject({ outcome: "started" }); + }); + + it("serializes a manual trigger with the engine fallback validator creator", async () => { + /* + FNXC:MissionValidation 2026-08-11-04:27: + FN-8963's feature lock must cover the production non-memo engine fallback as well as the + manual route. Either contender may acquire it first, but their concurrent creation attempts + must leave exactly one live validator run. + */ + const m = missions(); + const mission = await m.createMission({ title: "Manual and fallback serialization" }); + const milestone = await m.addMilestone(mission.id, { title: "MS" }); + const slice = await m.addSlice(milestone.id, { title: "SL" }); + const feature = await m.addFeature(slice.id, { title: "F" }); + + const [manual, fallback] = await Promise.allSettled([ + m.startManualValidatorRun(feature.id), + m.startValidatorRun(feature.id, "task_completion"), + ]); + + expect((await m.getValidatorRunsByFeature(feature.id)).filter((run) => run.status === "running")).toHaveLength(1); + expect( + (manual.status === "fulfilled" && manual.value.outcome === "started") || fallback.status === "fulfilled", + ).toBe(true); + if (manual.status === "fulfilled" && manual.value.outcome === "started") { + expect(fallback).toMatchObject({ status: "rejected" }); + } else { + expect(manual).toMatchObject({ status: "fulfilled", value: { outcome: "already-running" } }); + } + }); + + it("reports the newest fresh run when legacy data has multiple running rows", async () => { + const m = missions(); + const mission = await m.createMission({ title: "Legacy concurrent validation rows" }); + const milestone = await m.addMilestone(mission.id, { title: "MS" }); + const slice = await m.addSlice(milestone.id, { title: "SL" }); + const feature = await m.addFeature(slice.id, { title: "F" }); + const older = await m.startValidatorRun(feature.id, "task_completion"); + await h.layer().transactionImmediate(async (tx) => { + await tx.update(schema.project.missionValidatorRuns) + .set({ startedAt: new Date(Date.now() - 60_000).toISOString() }) + .where(eq(schema.project.missionValidatorRuns.id, older.id)); + }); + const newest = await m.startValidatorRun(feature.id, "task_completion"); + + await expect(m.startManualValidatorRun(feature.id)) + .resolves.toMatchObject({ outcome: "already-running", run: { id: newest.id } }); + expect((await m.getValidatorRunsByFeature(feature.id)).filter((run) => run.status === "running")).toHaveLength(2); + }); + + it("manual admission blocks fresh automatic runs but ignores stale runs", async () => { + const m = missions(); + const mission = await m.createMission({ title: "Cross surface admission" }); + const milestone = await m.addMilestone(mission.id, { title: "MS" }); + const slice = await m.addSlice(milestone.id, { title: "SL" }); + const feature = await m.addFeature(slice.id, { title: "F" }); + const automatic = await m.admitValidatorRun(feature.id, { inputFingerprint: "fp-a", failureBudget: 3, reusePass: false }); + expect(automatic).toMatchObject({ outcome: "start" }); + await expect(m.startManualValidatorRun(feature.id)).resolves.toMatchObject({ outcome: "already-running", run: { id: automatic.run!.id } }); + await expect(m.admitValidatorRun(feature.id, { inputFingerprint: "fp-a", failureBudget: 3, reusePass: false })) + .resolves.toMatchObject({ outcome: "running", run: { id: automatic.run!.id } }); + + await m.completeValidatorRun(automatic.run!.id, "error"); + const stale = await m.startValidatorRun(feature.id, "task_completion"); + await h.layer().transactionImmediate(async (tx) => { + await tx.update(schema.project.missionValidatorRuns) + .set({ startedAt: new Date(Date.now() - 7 * 60 * 60 * 1000).toISOString() }) + .where(eq(schema.project.missionValidatorRuns.id, stale.id)); + }); + await expect(m.startManualValidatorRun(feature.id)).resolves.toMatchObject({ outcome: "started" }); + }); + + it("characterizes automatic admission's fingerprint-only boundary", async () => { + /* + FNXC:MissionValidation 2026-08-11-03:43: + FN-8976 owns this deliberately unchanged gap: a fingerprint-less manual run is not visible to + fingerprint-scoped automatic admission. Manual-after-automatic is guarded above; do not widen + admitValidatorRun in the manual endpoint fix. + */ + const m = missions(); + const mission = await m.createMission({ title: "Fingerprint boundary" }); + const milestone = await m.addMilestone(mission.id, { title: "MS" }); + const slice = await m.addSlice(milestone.id, { title: "SL" }); + const feature = await m.addFeature(slice.id, { title: "F" }); + await expect(m.startManualValidatorRun(feature.id)).resolves.toMatchObject({ outcome: "started" }); + await expect(m.admitValidatorRun(feature.id, { inputFingerprint: "fresh-fingerprint", failureBudget: 3, reusePass: false })) + .resolves.toMatchObject({ outcome: "start" }); + }); + + it("does not let reuse-pass or budget exhaustion block a manual validator run", async () => { + const m = missions(); + const mission = await m.createMission({ title: "Terminal automatic outcomes" }); + const milestone = await m.addMilestone(mission.id, { title: "MS" }); + const slice = await m.addSlice(milestone.id, { title: "SL" }); + const reuseFeature = await m.addFeature(slice.id, { title: "Reuse" }); + const passed = await m.startValidatorRun(reuseFeature.id, "task_completion", undefined, "reuse"); + await m.completeValidatorRun(passed.id, "passed"); + await expect(m.admitValidatorRun(reuseFeature.id, { inputFingerprint: "reuse", failureBudget: 3, reusePass: true })).resolves.toMatchObject({ outcome: "reuse-pass" }); + await expect(m.startManualValidatorRun(reuseFeature.id)).resolves.toMatchObject({ outcome: "started" }); + + const budgetFeature = await m.addFeature(slice.id, { title: "Budget" }); + const failed = await m.startValidatorRun(budgetFeature.id, "task_completion", undefined, "budget"); + await m.completeValidatorRun(failed.id, "failed"); + await expect(m.admitValidatorRun(budgetFeature.id, { inputFingerprint: "budget", failureBudget: 1, reusePass: false })).resolves.toMatchObject({ outcome: "budget-exhausted" }); + await expect(m.startManualValidatorRun(budgetFeature.id)).resolves.toMatchObject({ outcome: "started" }); + }); + it("clears all validation-budget provenance when a changed fingerprint is admitted", async () => { const m = missions(); const mission = await m.createMission({ title: "Changed validation input" }); diff --git a/packages/core/src/async-stores/async-mission-store.ts b/packages/core/src/async-stores/async-mission-store.ts index c53244ea1b..ee6f5c6f92 100644 --- a/packages/core/src/async-stores/async-mission-store.ts +++ b/packages/core/src/async-stores/async-mission-store.ts @@ -14,13 +14,14 @@ import { EventEmitter } from "node:events"; import { and, desc, eq, inArray, notInArray, sql } from "drizzle-orm"; import * as schema from "../postgres/schema/index.js"; import type { AsyncDataLayer } from "../postgres/data-layer.js"; -import { boundMissionEventReason, classifyMissionResumeBlockers, FEATURE_LOOP_REPAIR_TRANSITIONS, buildMissionStatusEventMetadata, featureValidationRepairEligibility, FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType, normalizeMissionTransitionActorForEvent, renderValidationCause, selectNextSerialMissionSlice } from "../missions/mission-types.js"; +import { boundMissionEventReason, classifyMissionResumeBlockers, FEATURE_LOOP_REPAIR_TRANSITIONS, buildMissionStatusEventMetadata, featureValidationRepairEligibility, FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType, normalizeMissionTransitionActorForEvent, renderValidationCause, selectNextSerialMissionSlice, VALIDATION_INFLIGHT_STALE_MAX_AGE_MS } from "../missions/mission-types.js"; import type { Mission, Milestone, Slice, MissionFeature, MissionValidatorRun, + MissionManualValidatorRunAdmission, ValidatorRunAdmission, ValidatorRunAdmissionInput, MissionAssertionFailureRecord, @@ -1822,17 +1823,83 @@ export class AsyncMissionStore extends EventEmitter { taskId, inputFingerprint, startedAt: now, createdAt: now, updatedAt: now }; } - async startValidatorRun(featureId: string, triggerType?: string, taskId?: string, inputFingerprint?: string): Promise { - const feature = await getFeature(this.db, featureId); - if (!feature) throw new Error(`Feature ${featureId} not found`); - const run = await this.buildValidatorRun(this.db, feature, triggerType, taskId, inputFingerprint); - await createValidatorRun(this.db, run); - this.emit("validator-run:started", run); - await this.updateFeature(featureId, { - validatorAttemptCount: run.validatorAttempt, - lastValidatorRunId: run.id, - loopState: "validating", + /* + FNXC:MissionValidation 2026-08-11-03:43: + Manual validation previously had no in-flight guard: automatic admission is fingerprint-scoped + and FN-8947 guarded only repair re-runs. This feature-scoped transaction observes engine-started + runs, while runs beyond the reaper window do not wedge the button. A fingerprint-less manual run + intentionally remains invisible to fingerprint-scoped automatic admission; FN-8976 owns that + tested boundary rather than widening admitValidatorRun here. + */ + async startManualValidatorRun( + featureId: string, + input: { triggerType?: string; taskId?: string } = {}, + ): Promise { + const admission = await this.layer.transactionImmediate(async (tx) => { + const locked = await tx.select().from(schema.project.missionFeatures).where(and( + eq(schema.project.missionFeatures.projectId, missionProjectId()), + eq(schema.project.missionFeatures.id, featureId), + )).for("update"); + const feature = locked[0] ? await getFeature(tx, featureId) : undefined; + if (!feature) throw new Error(`Feature ${featureId} not found`); + const cutoff = Date.now() - VALIDATION_INFLIGHT_STALE_MAX_AGE_MS; + const blockingRun = (await listValidatorRunsByFeature(tx, featureId)).find( + (run) => run.status === "running" && Date.parse(run.startedAt) >= cutoff, + ); + if (blockingRun) return { outcome: "already-running", run: blockingRun }; + const run = await this.buildValidatorRun(tx, feature, input.triggerType ?? "manual", input.taskId); + await createValidatorRun(tx, run); + await updateFeature(tx, { + ...feature, + validatorAttemptCount: run.validatorAttempt, + lastValidatorRunId: run.id, + loopState: "validating", + updatedAt: run.startedAt, + }); + return { outcome: "started", run }; }); + if (admission.outcome === "started") this.emit("validator-run:started", admission.run); + return admission; + } + + /* + FNXC:MissionValidation 2026-08-11-04:27: + The engine's non-memo fallback still calls this low-level creator, so it must share the feature + row lock with manual admission. Preserve unrestricted automatic seeding and fingerprint behavior, + but refuse an engine fallback that arrives after a fresh manual run; otherwise the two paths can + serialize as manual-create then fallback-create and leave two running rows. + */ + async startValidatorRun(featureId: string, triggerType?: string, taskId?: string, inputFingerprint?: string): Promise { + const run = await this.layer.transactionImmediate(async (tx) => { + const locked = await tx.select().from(schema.project.missionFeatures).where(and( + eq(schema.project.missionFeatures.projectId, missionProjectId()), + eq(schema.project.missionFeatures.id, featureId), + )).for("update"); + const feature = locked[0] ? await getFeature(tx, featureId) : undefined; + if (!feature) throw new Error(`Feature ${featureId} not found`); + + if (triggerType === "task_completion") { + const cutoff = Date.now() - VALIDATION_INFLIGHT_STALE_MAX_AGE_MS; + const manualRun = (await listValidatorRunsByFeature(tx, featureId)).find( + (candidate) => candidate.status === "running" + && candidate.triggerType === "manual" + && Date.parse(candidate.startedAt) >= cutoff, + ); + if (manualRun) throw new Error(`Validator run ${manualRun.id} is already running for feature ${featureId}`); + } + + const created = await this.buildValidatorRun(tx, feature, triggerType, taskId, inputFingerprint); + await createValidatorRun(tx, created); + await updateFeature(tx, { + ...feature, + validatorAttemptCount: created.validatorAttempt, + lastValidatorRunId: created.id, + loopState: "validating", + updatedAt: created.startedAt, + }); + return created; + }); + this.emit("validator-run:started", run); return run; } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index aa9327b9af..77c745dbae 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1778,6 +1778,7 @@ export { FEATURE_LOOP_STATES, FEATURE_LOOP_REPAIR_TRANSITIONS, VALIDATOR_RUN_STATUSES, + VALIDATION_INFLIGHT_STALE_MAX_AGE_MS, VALIDATION_DIAGNOSTICS_MAX_EVIDENCE_PER_ASSERTION, VALIDATION_DIAGNOSTICS_MAX_TEXT_BYTES, MISSION_EVENT_REASON_MAX_BYTES, @@ -1806,6 +1807,7 @@ export type { SlicePlanState, FeatureLoopState, MissionFeatureRepairGroundTruth, + MissionManualValidatorRunAdmission, ValidatorRunStatus, ValidationAssertionVerdict, ValidationEvidenceReference, diff --git a/packages/core/src/missions/mission-store.ts b/packages/core/src/missions/mission-store.ts index c3b0883bcf..7dbd0dfda7 100644 --- a/packages/core/src/missions/mission-store.ts +++ b/packages/core/src/missions/mission-store.ts @@ -17,7 +17,7 @@ const severityAuditLog = createLogger("core-mission-store"); import { EventEmitter } from "node:events"; import type { Database } from "../db/db.js"; import { fromJson, toJson, toJsonNullable } from "../db/db.js"; -import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionOrigin, normalizeMissionAssertionScope, normalizeMissionAssertionType, renderValidationCause, selectNextSerialMissionSlice } from "./mission-types.js"; +import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionOrigin, normalizeMissionAssertionScope, normalizeMissionAssertionType, renderValidationCause, selectNextSerialMissionSlice, VALIDATION_INFLIGHT_STALE_MAX_AGE_MS } from "./mission-types.js"; import type { Goal, GoalStatus } from "../goals/goal-types.js"; import type { Mission, @@ -26,6 +26,7 @@ import type { Slice, MissionFeature, MissionValidatorRun, + MissionManualValidatorRunAdmission, MissionAssertionFailureRecord, MissionFixFeatureLineage, MissionFeatureLoopSnapshot, @@ -2804,6 +2805,64 @@ export class MissionStore extends EventEmitter { * @returns The created validator run * @throws Error if feature not found */ + /* + FNXC:MissionValidation 2026-08-11-03:43: + SQLite keeps the same feature-scoped manual admission contract as PostgreSQL. It blocks fresh + engine-started runs but lets runs older than the reaper window expire; FN-8976 tracks the known + fingerprint-less manual-to-automatic boundary without changing automatic admission here. + */ + startManualValidatorRun( + featureId: string, + input: { triggerType?: string; taskId?: string } = {}, + ): MissionManualValidatorRunAdmission { + let admission: MissionManualValidatorRunAdmission | undefined; + this.db.transaction(() => { + const feature = this.getFeature(featureId); + if (!feature) throw new Error(`Feature ${featureId} not found`); + const cutoff = new Date(Date.now() - VALIDATION_INFLIGHT_STALE_MAX_AGE_MS).toISOString(); + const rows = this.db.prepare( + "SELECT * FROM mission_validator_runs WHERE featureId = ? AND status = 'running' AND startedAt >= ? ORDER BY startedAt DESC, createdAt DESC, id DESC" + ).all(featureId, cutoff) as unknown as ValidatorRunRow[]; + const blockingRun = rows[0] ? this.rowToValidatorRun(rows[0]) : undefined; + if (blockingRun) { + admission = { outcome: "already-running", run: blockingRun }; + return; + } + const slice = this.getSlice(feature.sliceId); + if (!slice) throw new Error(`Slice ${feature.sliceId} not found`); + const milestone = this.getMilestone(slice.milestoneId); + if (!milestone) throw new Error(`Milestone ${slice.milestoneId} not found`); + const now = new Date().toISOString(); + const run: MissionValidatorRun = { + id: this.generateValidatorRunId(), featureId, milestoneId: milestone.id, sliceId: slice.id, + status: "running", triggerType: input.triggerType ?? "manual", + implementationAttempt: feature.implementationAttemptCount ?? 0, + validatorAttempt: (feature.validatorAttemptCount ?? 0) + 1, + taskId: input.taskId, startedAt: now, createdAt: now, updatedAt: now, + }; + this.db.prepare(` + INSERT INTO mission_validator_runs (id, featureId, milestoneId, sliceId, status, triggerType, implementationAttempt, validatorAttempt, taskId, inputFingerprint, startedAt, createdAt, updatedAt) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run(run.id, run.featureId, run.milestoneId, run.sliceId, run.status, run.triggerType ?? "auto", run.implementationAttempt, run.validatorAttempt, run.taskId ?? null, null, run.startedAt, run.createdAt, run.updatedAt); + this.updateFeature(featureId, { + validatorAttemptCount: run.validatorAttempt, + lastValidatorRunId: run.id, + loopState: "validating", + }); + admission = { outcome: "started", run }; + }); + if (!admission) throw new Error(`Manual validator admission did not resolve for ${featureId}`); + this.db.bumpLastModified(); + if (admission.outcome === "started") this.emit("validator-run:started", admission.run); + return admission; + } + + /* + FNXC:MissionValidation 2026-08-11-04:27: + Keep SQLite's non-memo engine fallback aligned with PostgreSQL: it shares manual admission's + transaction and cannot append a task-completion run behind a fresh manual run. This does not + change automatic fingerprint admission or unrestricted legacy automatic-run seeding. + */ startValidatorRun(featureId: string, triggerType?: string, taskId?: string, inputFingerprint?: string): MissionValidatorRun { const feature = this.getFeature(featureId); if (!feature) { @@ -2844,6 +2903,14 @@ export class MissionStore extends EventEmitter { }; this.db.transaction(() => { + if (triggerType === "task_completion") { + const cutoff = new Date(Date.now() - VALIDATION_INFLIGHT_STALE_MAX_AGE_MS).toISOString(); + const manualRun = this.db.prepare( + "SELECT id FROM mission_validator_runs WHERE featureId = ? AND status = 'running' AND triggerType = 'manual' AND startedAt >= ? LIMIT 1" + ).get(featureId, cutoff) as { id: string } | undefined; + if (manualRun) throw new Error(`Validator run ${manualRun.id} is already running for feature ${featureId}`); + } + // Insert the validator run this.db.prepare(` INSERT INTO mission_validator_runs (id, featureId, milestoneId, sliceId, status, triggerType, implementationAttempt, validatorAttempt, taskId, inputFingerprint, startedAt, createdAt, updatedAt) diff --git a/packages/core/src/missions/mission-types.ts b/packages/core/src/missions/mission-types.ts index 4073e0d36e..b080063ddb 100644 --- a/packages/core/src/missions/mission-types.ts +++ b/packages/core/src/missions/mission-types.ts @@ -665,6 +665,19 @@ export interface MissionFeature { // ── Validator Run & Loop Types ────────────────────────────────────── /** Atomic admission result for an automatic content-addressed validator dispatch. */ +/* +FNXC:MissionValidation 2026-08-11-03:43: +Manual validation must share the reaper's six-hour liveness window so a stranded run cannot +permanently wedge an operator control. Engine parity is pinned outside core because core must not +import the engine's healing constants. +*/ +export const VALIDATION_INFLIGHT_STALE_MAX_AGE_MS = 6 * 60 * 60 * 1000; + +/** Atomic admission result for a feature-scoped manual validator dispatch. */ +export type MissionManualValidatorRunAdmission = + | { outcome: "started"; run: MissionValidatorRun } + | { outcome: "already-running"; run: MissionValidatorRun }; + export type ValidatorRunAdmissionOutcome = "start" | "running" | "reuse-pass" | "budget-exhausted"; export interface ValidatorRunAdmission { outcome: ValidatorRunAdmissionOutcome; diff --git a/packages/dashboard/app/api/legacy.ts b/packages/dashboard/app/api/legacy.ts index 85e8c00e83..1834e120b4 100644 --- a/packages/dashboard/app/api/legacy.ts +++ b/packages/dashboard/app/api/legacy.ts @@ -1098,6 +1098,7 @@ export { triageAllSliceFeatures, triageFeature, triggerValidation, + VALIDATION_ALREADY_RUNNING, repairFeatureValidation, unlinkFeatureFromAssertion, unlinkFeatureFromTask, @@ -1137,6 +1138,7 @@ export type { SliceStatus, SliceWithFeatures, ValidationRunsResponse, + ValidationAlreadyRunningDetail, } from "./missions/missions.js"; /* * FNXC:CodeOrganization 2026-07-20-14:00: diff --git a/packages/dashboard/app/api/missions/missions.ts b/packages/dashboard/app/api/missions/missions.ts index ac1edbb962..e98016d06f 100644 --- a/packages/dashboard/app/api/missions/missions.ts +++ b/packages/dashboard/app/api/missions/missions.ts @@ -584,7 +584,16 @@ export interface MissionValidatorRun { updatedAt: string; } -/** Trigger validation for a feature */ +/** API conflict detail returned when a fresh validator run already owns a feature. */ +export const VALIDATION_ALREADY_RUNNING = "VALIDATION_ALREADY_RUNNING" as const; +export interface ValidationAlreadyRunningDetail { + code: typeof VALIDATION_ALREADY_RUNNING; + runId: string; + featureId: string; + startedAt: string; +} + +/** Trigger validation for a feature; may reject with ApiRequestError 409 and ValidationAlreadyRunningDetail. */ export function triggerValidation(featureId: string, projectId?: string): Promise<{ runId: string; featureId: string; status: string; triggerType: string; implementationAttempt: number; validatorAttempt: number; startedAt: string }> { return api(withProjectId(`/missions/features/${encodeURIComponent(featureId)}/validate`, projectId), { method: "POST", diff --git a/packages/dashboard/app/components/MissionManager.tsx b/packages/dashboard/app/components/MissionManager.tsx index 853db9f884..2f0e3c2795 100644 --- a/packages/dashboard/app/components/MissionManager.tsx +++ b/packages/dashboard/app/components/MissionManager.tsx @@ -104,6 +104,7 @@ import { fetchMilestoneValidation, fetchMilestoneValidationTelemetry, triggerValidation, + VALIDATION_ALREADY_RUNNING, repairFeatureValidation, fetchValidationLoopState, fetchValidationRuns, @@ -2529,7 +2530,19 @@ export function MissionManager({ isOpen, isInline = false, onClose, addToast, pr return next; }); } catch (err) { - addToast(getErrorMessage(err) || t("missions.validationTriggerFailed", "Failed to trigger validation"), "error"); + if (err instanceof ApiRequestError + && err.status === 409 + && (err.details as { code?: string } | undefined)?.code === VALIDATION_ALREADY_RUNNING) { + addToast(t("missions.validationAlreadyRunning", "Validation is already running for this feature"), "info"); + try { + const snapshot = await fetchValidationLoopState(featureId, projectId); + setFeatureLoopStates((prev) => new Map(prev).set(featureId, snapshot)); + } catch { + // FNXC:MissionValidation 2026-08-11-03:43: Preserve the specific conflict message when the live-state refresh races its owning validator. + } + } else { + addToast(getErrorMessage(err) || t("missions.validationTriggerFailed", "Failed to trigger validation"), "error"); + } } finally { setValidatingFeatures((prev) => { const next = new Set(prev); diff --git a/packages/dashboard/app/components/__tests__/MissionManager.validate-inflight.test.tsx b/packages/dashboard/app/components/__tests__/MissionManager.validate-inflight.test.tsx new file mode 100644 index 0000000000..0be8f3ddb8 --- /dev/null +++ b/packages/dashboard/app/components/__tests__/MissionManager.validate-inflight.test.tsx @@ -0,0 +1,77 @@ +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { ApiRequestError } from "../../api/client/client"; +import { MissionManager } from "../MissionManager"; +import { ConfirmDialogProvider } from "../../hooks/useConfirm"; + +const fetchMissions = vi.fn(); const fetchMission = vi.fn(); const fetchMissionsHealth = vi.fn(); +const fetchMissionEvents = vi.fn(); const fetchAssertions = vi.fn(); const fetchMilestoneValidation = vi.fn(); +const fetchMilestoneValidationTelemetry = vi.fn(); const fetchValidationLoopState = vi.fn(); const fetchValidationRuns = vi.fn(); +const fetchAiSessions = vi.fn(); const fetchAiSession = vi.fn(); const fetchMissionInterviewDrafts = vi.fn(); const triggerValidation = vi.fn(); +vi.mock("../../hooks/useNavigationHistory", () => ({ useNavigationHistoryContext: () => ({ pushNav: vi.fn(), replaceCurrent: vi.fn() }) })); +vi.mock("../../sse-bus", () => ({ subscribeSse: vi.fn(() => vi.fn()) })); +vi.mock("../MissionInterviewModal", () => ({ MissionInterviewModal: () => null })); +vi.mock("../MilestoneSliceInterviewModal", () => ({ MilestoneSliceInterviewModal: () => null })); +vi.mock("../../api", async (original) => ({ ...(await original()), + fetchMissions: (...args: unknown[]) => fetchMissions(...args), fetchMission: (...args: unknown[]) => fetchMission(...args), + fetchMissionsHealth: (...args: unknown[]) => fetchMissionsHealth(...args), fetchMissionEvents: (...args: unknown[]) => fetchMissionEvents(...args), + fetchAssertions: (...args: unknown[]) => fetchAssertions(...args), fetchMilestoneValidation: (...args: unknown[]) => fetchMilestoneValidation(...args), + fetchMilestoneValidationTelemetry: (...args: unknown[]) => fetchMilestoneValidationTelemetry(...args), fetchValidationLoopState: (...args: unknown[]) => fetchValidationLoopState(...args), + fetchValidationRuns: (...args: unknown[]) => fetchValidationRuns(...args), fetchAiSessions: (...args: unknown[]) => fetchAiSessions(...args), fetchAiSession: (...args: unknown[]) => fetchAiSession(...args), + fetchMissionInterviewDrafts: (...args: unknown[]) => fetchMissionInterviewDrafts(...args), triggerValidation: (...args: unknown[]) => triggerValidation(...args), + fetchModels: vi.fn().mockResolvedValue({ models: [], favoriteProviders: [], favoriteModels: [] }), +})); + +const feature = { id: "F-1", title: "Ready feature", status: "in-progress", loopState: "implementing", taskId: "FN-1", createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z" }; +const currentMission = () => ({ id: "M-1", title: "Mission", description: "", status: "active", interviewState: "completed", createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", milestones: [{ id: "MS-1", missionId: "M-1", title: "Milestone", status: "active", interviewState: "completed", orderIndex: 0, dependencies: [], createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", slices: [{ id: "SL-1", milestoneId: "MS-1", title: "Slice", status: "active", orderIndex: 0, dependencies: [], createdAt: "2026-01-01T00:00:00Z", updatedAt: "2026-01-01T00:00:00Z", features: [feature] }] }] }); + +/* +FNXC:MissionValidation 2026-08-11-04:17: +FN-8963 requires the manual validation conflict to preserve the single existing validate affordance +at desktop and mobile widths. A 409 must refresh live state, show an informative message, and release +the spinner rather than leaving an empty or disabled button shell. +*/ +describe("MissionManager manual validation in-flight conflict", () => { + afterEach(cleanup); + beforeEach(() => { + vi.clearAllMocks(); + fetchMissions.mockResolvedValue([{ ...currentMission(), milestones: [] }]); fetchMission.mockImplementation(async () => currentMission()); + fetchMissionsHealth.mockResolvedValue({}); fetchMissionEvents.mockResolvedValue([]); fetchAssertions.mockResolvedValue([]); fetchMilestoneValidation.mockResolvedValue(null); + fetchMilestoneValidationTelemetry.mockResolvedValue({ rollup: { milestoneId: "MS-1", state: "not_started" }, validationTelemetry: { validationRounds: [], totalRuns: 0 }, validationContract: null, fixFeatures: [] }); + fetchValidationLoopState.mockResolvedValue(null); fetchValidationRuns.mockResolvedValue([]); fetchAiSessions.mockResolvedValue([]); fetchAiSession.mockResolvedValue(null); fetchMissionInterviewDrafts.mockResolvedValue([]); + triggerValidation.mockResolvedValue({ runId: "VR-1" }); + }); + + function renderManager(addToast = vi.fn()) { + const rendered = render( {}} addToast={addToast} projectId="p1" targetMissionId="M-1" />); + return { ...rendered, addToast }; + } + + it.each([1280, 640])("shows the specific conflict and restores one enabled validate button at %ipx", async (width) => { + Object.defineProperty(window, "innerWidth", { configurable: true, value: width }); + window.dispatchEvent(new Event("resize")); + triggerValidation.mockRejectedValueOnce(new ApiRequestError("Validation is already running for this feature", 409, { code: "VALIDATION_ALREADY_RUNNING" })); + const { container, addToast } = renderManager(); + await screen.findByText("Ready feature"); + const button = container.querySelector('[title="Validate feature"]') as HTMLButtonElement; + expect(button).not.toBeNull(); + fireEvent.click(button); + + await waitFor(() => expect(addToast).toHaveBeenCalledWith("Validation is already running for this feature", "info")); + expect(addToast).not.toHaveBeenCalledWith("Failed to trigger validation", "error"); + await waitFor(() => expect((container.querySelector('[title="Validate feature"]') as HTMLButtonElement).disabled).toBe(false)); + expect(container.querySelectorAll('[title="Validate feature"]')).toHaveLength(1); + expect(fetchValidationLoopState).toHaveBeenCalledWith("F-1", "p1"); + }); + + it("preserves successful and generic validation feedback", async () => { + const { container, addToast } = renderManager(); + await screen.findByText("Ready feature"); + fireEvent.click(container.querySelector('[title="Validate feature"]')!); + await waitFor(() => expect(addToast).toHaveBeenCalledWith("Validation triggered", "success")); + + triggerValidation.mockRejectedValueOnce(new Error("network unavailable")); + fireEvent.click(container.querySelector('[title="Validate feature"]')!); + await waitFor(() => expect(addToast).toHaveBeenCalledWith("network unavailable", "error")); + }); +}); diff --git a/packages/dashboard/src/__tests__/mission-validate-inflight-guard.test.ts b/packages/dashboard/src/__tests__/mission-validate-inflight-guard.test.ts new file mode 100644 index 0000000000..95d3802a97 --- /dev/null +++ b/packages/dashboard/src/__tests__/mission-validate-inflight-guard.test.ts @@ -0,0 +1,55 @@ +// @vitest-environment node +import { describe, expect, it, vi } from "vitest"; +import express from "express"; +import type { TaskStore } from "@fusion/core"; +import { createMissionRouter } from "../mission-routes.js"; +import { request } from "../test-request.js"; + +const feature = { id: "F-TEST", sliceId: "SL-TEST", status: "defined", loopState: "idle" }; +const run = { id: "VR-FIRST", featureId: feature.id, status: "running", triggerType: "manual", implementationAttempt: 0, validatorAttempt: 1, startedAt: "2026-08-11T03:43:00.000Z" }; + +function fixture(options: { manual?: boolean; assertions?: unknown[]; exists?: boolean } = {}) { + const startManualValidatorRun = options.manual === false ? undefined : vi.fn() + .mockResolvedValueOnce({ outcome: "started", run }) + .mockResolvedValue({ outcome: "already-running", run }); + const missionStore = { + getFeature: vi.fn(async () => options.exists === false ? undefined : feature), + listAssertionsForFeature: vi.fn(async () => options.assertions ?? [{ id: "CA-1" }]), + startManualValidatorRun, + startValidatorRun: vi.fn(async () => run), + updateFeature: vi.fn(), on: vi.fn(), off: vi.fn(), + }; + const store = { + getMissionStore: () => missionStore, + getGoalStore: () => ({ getGoal: vi.fn(), listGoals: vi.fn() }), + getRootDir: () => "/tmp/mission-validate-inflight-guard", getSettings: vi.fn(async () => ({})), backendMode: true, + } as unknown as TaskStore; + const app = express(); + app.use(express.json()); + app.use("/api/missions", createMissionRouter(store)); + return { app, missionStore }; +} + +describe("manual mission validation in-flight guard", () => { + it("returns a precise conflict without a feature write after the first start", async () => { + const { app, missionStore } = fixture(); + expect((await request(app, "POST", "/api/missions/features/F-TEST/validate")).status).toBe(202); + const second = await request(app, "POST", "/api/missions/features/F-TEST/validate"); + expect(second.status).toBe(409); + expect(second.body).toMatchObject({ error: "Validation is already running for this feature", details: { code: "VALIDATION_ALREADY_RUNNING", runId: run.id, featureId: feature.id, startedAt: run.startedAt } }); + expect(missionStore.updateFeature).not.toHaveBeenCalled(); + }); + + it("preserves missing-feature and assertion precedence", async () => { + expect((await request(fixture({ exists: false }).app, "POST", "/api/missions/features/F-TEST/validate")).status).toBe(404); + const noAssertions = fixture({ assertions: [] }); + expect((await request(noAssertions.app, "POST", "/api/missions/features/F-TEST/validate")).status).toBe(400); + expect(noAssertions.missionStore.startManualValidatorRun).not.toHaveBeenCalled(); + }); + + it("uses the legacy start method when the store lacks the new capability", async () => { + const legacy = fixture({ manual: false }); + expect((await request(legacy.app, "POST", "/api/missions/features/F-TEST/validate")).status).toBe(202); + expect(legacy.missionStore.startValidatorRun).toHaveBeenCalledWith(feature.id, "manual"); + }); +}); diff --git a/packages/dashboard/src/mission-routes.ts b/packages/dashboard/src/mission-routes.ts index 9e2f425947..60e52af349 100644 --- a/packages/dashboard/src/mission-routes.ts +++ b/packages/dashboard/src/mission-routes.ts @@ -51,7 +51,6 @@ import type { FeatureStatus, InterviewState, MissionAssertionStatus, - FeatureLoopState, ValidatorRunStatus, ContractAssertionCreateInput, ContractAssertionUpdateInput, @@ -2441,13 +2440,30 @@ export function createMissionRouter( throw badRequest("Feature has no linked assertions. Link assertions before triggering validation."); } - // Transition feature to validating state - await missionStore.updateFeature(featureId, { - loopState: "validating" as FeatureLoopState, - }); - - // Start a validator run - const run = await missionStore.startValidatorRun(featureId, "manual"); + /* + FNXC:MissionValidation 2026-08-11-03:43: + A route pre-check races other tabs and engine validation. The admission transaction owns both + the feature mutation and the feature-scoped liveness check, returning this stable 409 contract + without touching the feature when a fresh run already exists. + */ + const manualAdmissionStore = missionStore as typeof missionStore & { + startManualValidatorRun?: (id: string, input?: { triggerType?: string; taskId?: string }) => Promise< + | { outcome: "started"; run: { id: string; featureId: string; status: string; triggerType?: string; implementationAttempt: number; validatorAttempt: number; startedAt: string } } + | { outcome: "already-running"; run: { id: string; startedAt: string } } + >; + }; + const admission = typeof manualAdmissionStore.startManualValidatorRun === "function" + ? await manualAdmissionStore.startManualValidatorRun(featureId) + : { outcome: "started" as const, run: await missionStore.startValidatorRun(featureId, "manual") }; + if (admission.outcome === "already-running") { + throw conflict("Validation is already running for this feature", { + code: "VALIDATION_ALREADY_RUNNING", + runId: admission.run.id, + featureId, + startedAt: admission.run.startedAt, + }); + } + const run = admission.run; res.status(202).json({ runId: run.id, diff --git a/packages/engine/src/__tests__/mission-execution-loop.test.ts b/packages/engine/src/__tests__/mission-execution-loop.test.ts index c2eb76105c..1381b0992d 100644 --- a/packages/engine/src/__tests__/mission-execution-loop.test.ts +++ b/packages/engine/src/__tests__/mission-execution-loop.test.ts @@ -49,7 +49,7 @@ vi.mock("../logger.js", () => ({ })), })); -vi.mock("../agent-session-helpers.js", async (importOriginal) => { +vi.mock("../agents/agent-session-helpers.js", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, @@ -1040,6 +1040,65 @@ describe("MissionExecutionLoop", () => { expect(missionStore.startValidatorRun).toHaveBeenCalledWith("F-001", "task_completion", "FN-001"); }); + it.each(["running", "budget-exhausted"] as const)("preserves automatic %s admission short-circuit without invoking manual admission", async (outcome) => { + /* + FNXC:MissionValidation 2026-08-11-04:17: + FN-8963 changes only manual admission. The automatic loop must retain its existing + fingerprint-scoped running and budget-exhausted short-circuits and never call the manual + primitive; FN-8976 owns widening automatic admission to observe manual runs. + */ + const feature = createMockFeature({ loopState: "implementing", taskId: "FN-001" }); + missionStore._setFeature(feature); + missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature); + missionStore._setAssertionsForFeature(feature.id, makeAssertions(1)); + taskStore._setTask({ id: "FN-001", title: "Test", description: "Test task", log: [], mergeDetails: { commitSha: "landed-sha" } } as any); + taskStore.getSettings.mockResolvedValue({ + missionStaleThresholdMs: 600_000, + missionMaxTaskRetries: 3, + defaultProvider: "memo-provider", + defaultModelId: "memo-model", + }); + const admitValidatorRun = vi.fn().mockResolvedValue({ outcome }); + const startManualValidatorRun = vi.fn(); + Object.assign(missionStore, { admitValidatorRun, startManualValidatorRun }); + const dispose = vi.fn().mockResolvedValue(undefined); + loop = new MissionExecutionLoop({ + taskStore: taskStore as any, + missionStore: missionStore as any, + rootDir: "/tmp", + checkoutMaterializer: { materialize: vi.fn().mockResolvedValue({ dir: "/inspection/landed", dispose }), assertSourceClean: vi.fn() }, + }); + loop.start(); + + await loop.processTaskOutcome("FN-001"); + + expect(admitValidatorRun).toHaveBeenCalledOnce(); + expect(startManualValidatorRun).not.toHaveBeenCalled(); + expect(missionStore.startValidatorRun).not.toHaveBeenCalled(); + expect(createResolvedAgentSession).not.toHaveBeenCalled(); + expect(dispose).toHaveBeenCalledOnce(); + }); + + it("keeps the non-memo automatic fallback separate from manual admission", async () => { + const feature = createMockFeature({ loopState: "implementing", taskId: "FN-001" }); + missionStore._setFeature(feature); + missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature); + taskStore._setTask({ id: "FN-001", title: "Test", description: "Test task", log: [] }); + const startManualValidatorRun = vi.fn(); + Object.assign(missionStore, { startManualValidatorRun }); + loop = new MissionExecutionLoop({ taskStore: taskStore as any, missionStore: missionStore as any, rootDir: "/tmp" }); + vi.spyOn(loop as any, "runValidation").mockResolvedValue({ + result: { status: "pass", summary: "ok" }, + inspection: { inspectionRoot: "/tmp", landedSha: undefined, fallbackUsed: true, workspaceStale: false }, + }); + loop.start(); + + await loop.processTaskOutcome("FN-001"); + + expect(missionStore.startValidatorRun).toHaveBeenCalledWith("F-001", "task_completion", "FN-001"); + expect(startManualValidatorRun).not.toHaveBeenCalled(); + }); + it("requeues needs_fix features back through validation", async () => { const assertions = makeAssertions(1); const response = JSON.stringify({ diff --git a/packages/engine/src/__tests__/validator-run-stale-window-parity.test.ts b/packages/engine/src/__tests__/validator-run-stale-window-parity.test.ts new file mode 100644 index 0000000000..57cbb3cdec --- /dev/null +++ b/packages/engine/src/__tests__/validator-run-stale-window-parity.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, it } from "vitest"; +import { VALIDATION_INFLIGHT_STALE_MAX_AGE_MS } from "@fusion/core"; +import { VALIDATOR_RUN_STALE_MAX_AGE_MS } from "../healing/self-healing-constants.js"; + +describe("validator stale-window parity", () => { + it("keeps manual admission and the stale-run reaper on the same window", () => { + expect(VALIDATION_INFLIGHT_STALE_MAX_AGE_MS).toBe(VALIDATOR_RUN_STALE_MAX_AGE_MS); + }); +});