FN-5902: make mission validation AI-run all criteria

Route every mission feature through validator-backed completion checks.

- lazily restore a managed feature assertion before validation instead of auto-passing zero-assertion features
- thread milestone acceptance criteria into validator prompts and system instructions as enforced requirements
- update MissionManager copy/tests to present criteria as AI-validated runtime gates and remove informational-only/zero-assertion warnings
- document the all-criteria AI-run contract and add a changeset for @runfusion/fusion

Files changed:
 .changeset/fn-5902-mission-validation-ai-run.md    |   5 +
 AGENTS.md                                          |   2 +-
 docs/architecture.md                               |   2 +-
 docs/missions-completion-contract.md               | 198 ++++++---------------
 docs/missions.md                                   |   5 +-
 packages/core/src/__tests__/mission-store.test.ts  |  23 ++-
 packages/core/src/mission-store.ts                 |  10 ++
 packages/dashboard/app/components/MissionManager.css    |  31 ----
 packages/dashboard/app/components/MissionManager.tsx    |  86 +++------
 packages/dashboard/app/components/__tests__/MissionManager.test.tsx   |  60 +++++--
 packages/engine/src/__tests__/mission-execution-loop.test.ts   | 111 +++++++++---
 packages/engine/src/__tests__/reliability-interactions/mission-validation-trigger-gap.test.ts         |  57 +++---
 packages/engine/src/mission-execution-loop.ts      |  78 ++++----
 13 files changed, 318 insertions(+), 350 deletions(-)

Fusion-Task-Id: FN-5902

Fusion-Task-Lineage: 5f25caad-33c9-42ff-822b-1ea092afc29f
This commit is contained in:
gsxdsm
2026-06-02 18:11:26 -07:00
parent abbeaec0a8
commit cc18206bc5
13 changed files with 318 additions and 350 deletions

View File

@@ -204,6 +204,16 @@ function createMockMissionStore() {
return updated;
}),
listAssertionsForFeature: vi.fn((featureId: string) => assertionsByFeature.get(featureId) ?? []),
ensureFeatureAssertionLinked: vi.fn((featureId: string) => {
const feature = features.get(featureId);
if (!feature) {
throw new Error(`Feature ${featureId} not found`);
}
if ((assertionsByFeature.get(featureId) ?? []).length === 0) {
store._addFeatureWithManagedAssertion(feature);
}
return assertionsByFeature.get(featureId) ?? [];
}),
getAssertionsForFeature: vi.fn((featureId: string) => assertionsByFeature.get(featureId) ?? []),
getSlice: vi.fn((id: string) => {
// Return a mock slice with milestoneId for the hierarchy
@@ -764,12 +774,21 @@ describe("MissionExecutionLoop", () => {
expect(missionStore.completeValidatorRun).toHaveBeenCalledWith(expect.any(String), "passed", "Recovered validation passed");
});
it("should auto-pass if feature has no linked assertions", async () => {
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-001" });
it("lazy-ensures a managed assertion and routes zero-assertion features through validation", async () => {
const feature = createMockFeature({
id: "F-001",
loopState: "implementing",
taskId: "FN-001",
title: "Feature from prose",
acceptanceCriteria: "Feature must validate through AI",
});
missionStore._setFeature(feature);
taskStore._setTask({ id: "FN-001", title: "Test", description: "Test task", log: [] });
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue([]);
missionStore.listAssertionsForFeature = vi
.fn()
.mockReturnValueOnce([])
.mockImplementation((featureId: string) => (missionStore as any).getAssertionsForFeature(featureId));
loop = new MissionExecutionLoop({
taskStore: taskStore as any,
@@ -777,41 +796,39 @@ describe("MissionExecutionLoop", () => {
rootDir: "/tmp",
});
const emitSpy = vi.spyOn(loop, "emit");
vi.spyOn(loop as any, "runValidation").mockResolvedValue({ status: "pass", summary: "ok" });
loop.start();
await loop.processTaskOutcome("FN-001");
// When there are no assertions, we skip starting a validator run
expect(missionStore.startValidatorRun).not.toHaveBeenCalled();
// But the passed event should be emitted
expect(missionStore.ensureFeatureAssertionLinked).toHaveBeenCalledWith("F-001");
expect(missionStore.startValidatorRun).toHaveBeenCalledWith("F-001", "task_completion");
expect(emitSpy).toHaveBeenCalledWith(
"validation:passed",
expect.objectContaining({ featureId: "F-001" }),
);
expect(missionStore.updateFeature).toHaveBeenCalledWith(
"F-001",
expect.objectContaining({ loopState: "passed", lastValidatorStatus: "passed" }),
);
expect(missionStore.logMissionEvent).toHaveBeenCalledWith(
expect.any(String),
"warning",
expect.stringContaining("auto-passed"),
expect.objectContaining({
code: "validation_auto_passed_no_assertions",
featureId: "F-001",
reason: "No assertions linked",
taskId: "FN-001",
}),
const noAssertionEvents = missionStore.logMissionEvent.mock.calls.filter(
([, , , payload]) => payload?.code === "validation_auto_passed_no_assertions",
);
expect(noAssertionEvents).toHaveLength(0);
expectNoValidationBoardTaskMutation(taskStore);
});
it("emits no-assertions auto-pass event exactly once across re-entry", async () => {
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-001" });
it("does not emit auto-pass evidence across re-entry after lazy assertion ensure", async () => {
const feature = createMockFeature({
id: "F-001",
loopState: "implementing",
taskId: "FN-001",
title: "Feature from prose",
acceptanceCriteria: "Feature must validate through AI",
});
missionStore._setFeature(feature);
taskStore._setTask({ id: "FN-001", title: "Test", description: "Test task", log: [] });
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue([]);
missionStore.listAssertionsForFeature = vi
.fn()
.mockReturnValueOnce([])
.mockImplementation((featureId: string) => (missionStore as any).getAssertionsForFeature(featureId));
loop = new MissionExecutionLoop({
taskStore: taskStore as any,
@@ -823,10 +840,11 @@ describe("MissionExecutionLoop", () => {
await loop.processTaskOutcome("FN-001");
await loop.processTaskOutcome("FN-001");
expect(missionStore.ensureFeatureAssertionLinked).toHaveBeenCalledTimes(1);
const noAssertionEvents = missionStore.logMissionEvent.mock.calls.filter(
([, , , payload]) => payload?.code === "validation_auto_passed_no_assertions",
);
expect(noAssertionEvents).toHaveLength(1);
expect(noAssertionEvents).toHaveLength(0);
});
it("uses validator path for later-added feature with managed assertion", async () => {
@@ -854,6 +872,44 @@ describe("MissionExecutionLoop", () => {
expect(missionStore.startValidatorRun).toHaveBeenCalledWith("F-LATER", "task_completion");
});
it("threads milestone acceptance criteria into validator prompts", () => {
const feature = createMockFeature({
id: "F-MILESTONE",
title: "Feature under milestone",
acceptanceCriteria: "Feature criteria",
});
const milestone = createMockMilestone({
id: "MS-MILESTONE",
acceptanceCriteria: "Milestone pass bar text",
});
const assertions = [
{
id: "CA-1",
milestoneId: milestone.id,
title: "Managed assertion",
assertion: "Feature criteria",
status: "pending" as const,
orderIndex: 0,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
},
];
loop = new MissionExecutionLoop({
taskStore: taskStore as any,
missionStore: missionStore as any,
rootDir: "/tmp",
});
const prompt = (loop as any).buildValidationPrompt(feature, assertions, milestone);
const systemPrompt = (loop as any).buildValidationSystemPrompt(feature, assertions, "Task context", milestone);
expect(prompt).toContain("Milestone pass bar text");
expect(prompt).toContain("must also be satisfied for this feature to pass");
expect(systemPrompt).toContain("Milestone pass bar text");
expect(systemPrompt).toContain("validator-executed requirements");
});
it("does NOT create a board task for single-feature validation", async () => {
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-001", sliceId: "SL-001" });
missionStore._setFeature(feature);
@@ -1464,7 +1520,7 @@ describe("MissionExecutionLoop", () => {
});
missionStore._setFeature(feature);
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue([]); // No assertions = auto-pass
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue([]);
taskStore._setTask({ id: "FN-001", title: "Test", description: "Test", log: [] });
const notifySpy = vi.fn();
@@ -1477,12 +1533,13 @@ describe("MissionExecutionLoop", () => {
},
});
const emitSpy = vi.spyOn(loop, "emit");
vi.spyOn(loop as any, "runValidation").mockResolvedValue({ status: "pass", summary: "ok" });
loop.start();
await loop.processTaskOutcome("FN-001");
// No validator run started (no assertions)
expect(missionStore.startValidatorRun).not.toHaveBeenCalled();
expect(missionStore.ensureFeatureAssertionLinked).toHaveBeenCalledWith("F-001");
expect(missionStore.startValidatorRun).toHaveBeenCalledWith("F-001", "task_completion");
// validation:passed event emitted
expect(emitSpy).toHaveBeenCalledWith(

View File

@@ -186,9 +186,10 @@ describe("FN-5715 reliability: mission validation trigger gap", () => {
loop.stop();
});
it("periodic recovery pass replays implementing done tasks with zero assertions and advances loop state", async () => {
const feature = makeFeature({ status: "done", lastValidatorStatus: undefined, loopState: "implementing" });
it("periodic recovery lazily ensures assertions and AI-validates zero-link legacy features", async () => {
const feature = makeFeature({ status: "done", lastValidatorStatus: undefined, loopState: "implementing", acceptanceCriteria: "must pass" });
const currentFeature = { ...feature };
const linkedAssertions: Array<{ id: string }> = [];
const missionStore = {
listMissions: vi.fn(() => [{ id: "M-001", status: "active" }]),
getMissionWithHierarchy: vi.fn(() => ({
@@ -203,11 +204,21 @@ describe("FN-5715 reliability: mission validation trigger gap", () => {
Object.assign(currentFeature, patch);
return { ...currentFeature };
}),
listAssertionsForFeature: vi.fn(() => []),
listAssertionsForFeature: vi.fn(() => linkedAssertions),
ensureFeatureAssertionLinked: vi.fn(() => {
if (linkedAssertions.length === 0) {
linkedAssertions.push({ id: "CA-ENSURED" });
}
return linkedAssertions;
}),
startValidatorRun: vi.fn(() => ({ id: "VR-001", featureId: "F-001" })),
completeValidatorRun: vi.fn(),
getSlice: vi.fn(() => ({ id: "SL-001", milestoneId: "MS-001", status: "active" })),
getMilestone: vi.fn(() => ({ id: "MS-001", missionId: "M-001" })),
logMissionEvent: vi.fn(),
transitionLoopState: vi.fn(),
setFeatureCurrentTaskRunId: vi.fn(),
getFailuresForRun: vi.fn(() => []),
};
const taskStore = {
getTask: vi.fn(async () => ({ id: "FN-001", column: "done", status: "done" })),
@@ -220,25 +231,23 @@ describe("FN-5715 reliability: mission validation trigger gap", () => {
taskStore: taskStore as any,
rootDir: process.cwd(),
});
vi.spyOn(loop as any, "runValidation").mockResolvedValue({ status: "pass", summary: "ok" });
loop.start();
const periodicMaintenancePass = async () => loop.recoverActiveMissions();
await periodicMaintenancePass();
await periodicMaintenancePass();
expect(missionStore.updateFeature).toHaveBeenCalledTimes(1);
expect(missionStore.updateFeature).toHaveBeenCalledWith(
"F-001",
expect.objectContaining({ loopState: "passed", lastValidatorStatus: "passed" }),
);
expect(missionStore.ensureFeatureAssertionLinked).toHaveBeenCalledWith("F-001");
expect(missionStore.startValidatorRun).toHaveBeenCalledWith("F-001", "task_completion");
const noAssertionEvents = missionStore.logMissionEvent.mock.calls.filter(
([, type, , payload]) => type === "warning" && payload?.code === "validation_auto_passed_no_assertions",
);
expect(noAssertionEvents).toHaveLength(1);
expect(noAssertionEvents).toHaveLength(0);
loop.stop();
});
it("routes through validator after assertion backfill instead of no-assertion auto-pass", async () => {
it("keeps backfill optional because runtime lazy-ensure routes through validator", async () => {
const feature = makeFeature({ status: "done", acceptanceCriteria: "must pass", loopState: "implementing" });
const currentFeature = { ...feature };
const linkedAssertions: Array<{ id: string }> = [];
@@ -258,6 +267,12 @@ describe("FN-5715 reliability: mission validation trigger gap", () => {
return { ...currentFeature };
}),
listAssertionsForFeature: vi.fn(() => linkedAssertions),
ensureFeatureAssertionLinked: vi.fn(() => {
if (linkedAssertions.length === 0) {
linkedAssertions.push({ id: "CA-001" });
}
return linkedAssertions;
}),
startValidatorRun: vi.fn(() => ({ id: "VR-001", featureId: "F-001" })),
completeValidatorRun: vi.fn(),
getSlice: vi.fn(() => ({ id: "SL-001", milestoneId: "MS-001", status: "active" })),
@@ -279,27 +294,13 @@ describe("FN-5715 reliability: mission validation trigger gap", () => {
await loop.recoverActiveMissions();
const noAssertionEventsBefore = missionStore.logMissionEvent.mock.calls.filter(
([, type, , payload]) => type === "warning" && payload?.code === "validation_auto_passed_no_assertions",
);
expect(noAssertionEventsBefore).toHaveLength(1);
expect(missionStore.startValidatorRun).not.toHaveBeenCalled();
linkedAssertions.push({ id: "CA-001" });
currentFeature.loopState = "implementing";
currentFeature.lastValidatorStatus = undefined;
await loop.processTaskOutcome("FN-001");
expect(missionStore.ensureFeatureAssertionLinked).toHaveBeenCalledWith("F-001");
expect(missionStore.startValidatorRun).toHaveBeenCalledWith("F-001", "task_completion");
const noAssertionEventsAfter = missionStore.logMissionEvent.mock.calls.filter(
const noAssertionEvents = missionStore.logMissionEvent.mock.calls.filter(
([, type, , payload]) => type === "warning" && payload?.code === "validation_auto_passed_no_assertions",
);
expect(noAssertionEventsAfter).toHaveLength(1);
expect(missionStore.updateFeature).toHaveBeenCalledWith(
"F-001",
expect.objectContaining({ loopState: "passed", lastValidatorStatus: "passed" }),
);
expect(noAssertionEvents).toHaveLength(0);
expect(missionStore.completeValidatorRun).toHaveBeenCalledWith("VR-001", "passed", "ok");
loop.stop();
});

View File

@@ -20,6 +20,7 @@ import type {
MissionValidatorRun,
AgentStore,
Settings,
Milestone,
} from "@fusion/core";
import {
TEST_MODE_RESOLVED,
@@ -353,13 +354,12 @@ export class MissionExecutionLoop extends EventEmitter {
return;
}
// Get linked assertions for this feature
const assertions = this.missionStore.listAssertionsForFeature(feature.id);
// Lazily guarantee a linked assertion before validation so every feature
// is evaluated by the validator even when legacy data is missing links.
let assertions = this.missionStore.listAssertionsForFeature(feature.id);
if (assertions.length === 0) {
loopLog.log(`Feature ${feature.id} has no linked assertions; marking as passed`);
// No assertions = automatically pass
await this.handleValidationPass(feature.id, undefined, "No assertions linked");
return;
loopLog.log(`Feature ${feature.id} has no linked assertions; lazily ensuring store-managed assertion linkage`);
assertions = this.missionStore.ensureFeatureAssertionLinked(feature.id);
}
// Mark feature as being validated
@@ -408,8 +408,10 @@ export class MissionExecutionLoop extends EventEmitter {
): Promise<ValidationResult> {
loopLog.log(`Running validation for feature ${feature.id} with ${assertions.length} assertions`);
const milestone = this.resolveFeatureMilestone(feature);
// Build the validation prompt
const prompt = this.buildValidationPrompt(feature, assertions);
const prompt = this.buildValidationPrompt(feature, assertions, milestone);
// Get task context for validation
const task = feature.taskId ? await this.taskStore.getTask(feature.taskId) : null;
@@ -441,7 +443,7 @@ export class MissionExecutionLoop extends EventEmitter {
runtimeHint: validationRuntimeHint,
pluginRunner: this.pluginRunner,
cwd: this.rootDir,
systemPrompt: this.buildValidationSystemPrompt(feature, assertions, taskContext),
systemPrompt: this.buildValidationSystemPrompt(feature, assertions, taskContext, milestone),
tools: "readonly",
defaultProvider: validationSessionModel.provider,
defaultModelId: validationSessionModel.modelId,
@@ -796,19 +798,27 @@ export class MissionExecutionLoop extends EventEmitter {
/**
* Build the validation prompt sent to the AI agent.
*/
private buildValidationPrompt(feature: MissionFeature, assertions: MissionContractAssertion[]): string {
private buildValidationPrompt(
feature: MissionFeature,
assertions: MissionContractAssertion[],
milestone?: Milestone,
): string {
const assertionTexts = assertions
.map((a, i) => `${i + 1}. **${a.title}**: ${a.assertion}`)
.join("\n");
const milestoneAcceptanceCriteria = milestone?.acceptanceCriteria?.trim();
const milestoneContext = milestoneAcceptanceCriteria
? `\nMilestone acceptance criteria (must also be satisfied for this feature to pass):\n${milestoneAcceptanceCriteria}\n`
: "";
return `Evaluate the implementation for feature "${feature.title}" against the following contract assertions:
${assertionTexts}
${assertionTexts}${milestoneContext}
For each assertion:
- Determine if the implementation satisfies the assertion (pass/fail/blocked)
- If failed, explain what was expected vs what was actually observed
- If blocked, explain what external factor prevented validation
- Also verify that the implementation satisfies any milestone acceptance criteria provided above
Respond with a JSON object in this format:
{
@@ -833,22 +843,25 @@ Be thorough and objective. If any assertion fails, the overall status should be
* Build the system prompt for the validation agent.
*/
private buildValidationSystemPrompt(
feature: MissionFeature,
_feature: MissionFeature,
_assertions: MissionContractAssertion[],
taskContext: string,
milestone?: Milestone,
): string {
const milestoneAcceptanceCriteria = milestone?.acceptanceCriteria?.trim();
return `You are a validation agent responsible for evaluating whether an implementation satisfies its contract assertions.
You will receive:
1. A feature description with its acceptance criteria
2. Contract assertions to evaluate against
3. Task context including the implementation details
3. Task context including the implementation details${milestoneAcceptanceCriteria ? `\n4. Milestone acceptance criteria text that also applies to this feature: ${milestoneAcceptanceCriteria}` : ""}
Your job is to:
1. Carefully review the implementation as described in the task context
2. Evaluate each contract assertion objectively
3. Determine if the implementation fully satisfies each assertion
4. Return a structured JSON response with your findings
4. Verify the implementation also satisfies any milestone acceptance criteria provided for the parent milestone
5. Return a structured JSON response with your findings
Be thorough and precise. A contract assertion represents a commitment made during planning - the implementation must fully satisfy it or it is considered failed.
@@ -857,6 +870,7 @@ Evaluation guidance:
- "fail" means one or more assertions are unmet or only partially satisfied.
- "blocked" means you cannot evaluate due to missing/insufficient evidence or external constraints.
- Partial satisfaction must be marked as failed with clear expected vs actual details.
- Milestone acceptance criteria are validator-executed requirements, not informational context.
Response format: Return ONLY a JSON object (no additional text) with this structure:
{
@@ -898,6 +912,15 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
return lines.join("\n");
}
private resolveFeatureMilestone(feature: MissionFeature): Milestone | undefined {
const slice = this.missionStore.getSlice(feature.sliceId);
if (!slice) {
return undefined;
}
return this.missionStore.getMilestone(slice.milestoneId);
}
private completeValidatorRunIfStillRunning(
runId: string | undefined,
status: "passed" | "failed" | "blocked" | "error",
@@ -938,33 +961,6 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
this.missionStore.updateFeatureStatus(featureId, "done");
}
if (!runId && feature) {
const alreadyAutoPassed =
feature.status === "done" &&
feature.loopState === "passed" &&
feature.lastValidatorStatus === "passed";
if (!alreadyAutoPassed) {
// Auto-pass path has no validator run, so we must advance loop bookkeeping here.
if (feature.loopState !== "passed" || feature.lastValidatorStatus !== "passed") {
this.missionStore.updateFeature(featureId, {
loopState: "passed",
lastValidatorStatus: "passed",
});
}
this.logFeatureWarningEvent(
featureId,
"validation_auto_passed_no_assertions",
`Feature ${featureId} auto-passed because no assertions were linked.`,
{
taskId: feature.taskId,
reason: "No assertions linked",
},
);
}
}
loopLog.log(`Feature ${featureId} passed validation`);
// Notify autopilot if configured