feat(FN-1569): add mission validation telemetry APIs and UI
- Emit fix-feature mission events and forward validator telemetry through mission execution - Add core and dashboard API types plus a validation telemetry route in mission-routes - Update SSE mission event handling so validation telemetry updates stream reliably to clients - Add telemetry surfaces in MissionManager with corresponding mission-specific UI styling - Expand MissionManager and mission e2e test coverage for validation telemetry flows
This commit is contained in:
Fusion
@@ -22,6 +22,8 @@ import type {
|
||||
MissionHealth,
|
||||
MissionContractAssertion,
|
||||
ContractAssertionCreateInput,
|
||||
MissionValidatorRun,
|
||||
MissionAssertionFailureRecord,
|
||||
} from "@fusion/core";
|
||||
import type { AiSessionRow } from "./ai-session-store.js";
|
||||
import {
|
||||
@@ -45,6 +47,8 @@ function createMockMissionStore() {
|
||||
const missionEvents: Map<string, MissionEvent[]> = new Map();
|
||||
const assertions: Map<string, MissionContractAssertion> = new Map();
|
||||
const assertionLinks: Array<{ featureId: string; assertionId: string }> = [];
|
||||
const validatorRuns: Map<string, MissionValidatorRun> = new Map();
|
||||
const runFailures: Map<string, MissionAssertionFailureRecord[]> = new Map();
|
||||
|
||||
let missionCounter = 1;
|
||||
let milestoneCounter = 1;
|
||||
@@ -421,6 +425,39 @@ function createMockMissionStore() {
|
||||
assertionLinks.push({ featureId, assertionId });
|
||||
}),
|
||||
|
||||
listAssertionsForFeature: vi.fn((featureId: string) =>
|
||||
assertionLinks
|
||||
.filter((link) => link.featureId === featureId)
|
||||
.map((link) => assertions.get(link.assertionId))
|
||||
.filter((assertion): assertion is MissionContractAssertion => Boolean(assertion))
|
||||
),
|
||||
|
||||
listFeaturesForAssertion: vi.fn((assertionId: string) =>
|
||||
assertionLinks
|
||||
.filter((link) => link.assertionId === assertionId)
|
||||
.map((link) => features.get(link.featureId))
|
||||
.filter((feature): feature is MissionFeature => Boolean(feature))
|
||||
),
|
||||
|
||||
getValidatorRunsByFeature: vi.fn((featureId: string) =>
|
||||
Array.from(validatorRuns.values())
|
||||
.filter((run) => run.featureId === featureId)
|
||||
.sort((a, b) => b.startedAt.localeCompare(a.startedAt))
|
||||
),
|
||||
|
||||
getFailuresForRun: vi.fn((runId: string) => runFailures.get(runId) ?? []),
|
||||
|
||||
getMilestoneValidationRollup: vi.fn((milestoneId: string) => ({
|
||||
milestoneId,
|
||||
totalAssertions: 0,
|
||||
passedAssertions: 0,
|
||||
failedAssertions: 0,
|
||||
blockedAssertions: 0,
|
||||
pendingAssertions: 0,
|
||||
unlinkedAssertions: 0,
|
||||
state: "not_started",
|
||||
})),
|
||||
|
||||
reorderMilestones: vi.fn((missionId: string, orderedIds: string[]) => {
|
||||
orderedIds.forEach((id, index) => {
|
||||
const milestone = milestones.get(id);
|
||||
@@ -4316,6 +4353,172 @@ describe("Mission API", () => {
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
});
|
||||
|
||||
describe("GET /milestones/:milestoneId/validation-telemetry", () => {
|
||||
it("returns empty grouped telemetry for milestones without assertions or runs", async () => {
|
||||
const { app, missionStore } = buildApp();
|
||||
const ms = missionStore as ReturnType<typeof createMockMissionStore>;
|
||||
|
||||
const mission = ms.createMission({ title: "Telemetry Mission" });
|
||||
const milestone = ms.addMilestone(mission.id, { title: "Milestone A" });
|
||||
|
||||
const res = await get(app, `/api/missions/milestones/${milestone.id}/validation-telemetry`);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.validationContract.assertions).toEqual([]);
|
||||
expect(res.body.validationContract.featureFulfillment).toEqual({});
|
||||
expect(res.body.validationTelemetry.validationRounds).toEqual([]);
|
||||
expect(res.body.validationTelemetry.lastValidatorStatus).toBeNull();
|
||||
expect(res.body.validationTelemetry.totalRuns).toBe(0);
|
||||
expect(res.body.fixFeatures).toEqual([]);
|
||||
expect(res.body.rollup.state).toBe("not_started");
|
||||
});
|
||||
|
||||
it("returns contract assertions and feature fulfillment links", async () => {
|
||||
const { app, missionStore } = buildApp();
|
||||
const ms = missionStore as ReturnType<typeof createMockMissionStore>;
|
||||
|
||||
const mission = ms.createMission({ title: "Contract Mission" });
|
||||
const milestone = ms.addMilestone(mission.id, { title: "Milestone B" });
|
||||
const slice = ms.addSlice(milestone.id, { title: "Slice B" });
|
||||
const featureOne = ms.addFeature(slice.id, { title: "Feature One" });
|
||||
const featureTwo = ms.addFeature(slice.id, { title: "Feature Two" });
|
||||
const assertionOne = ms.addContractAssertion(milestone.id, {
|
||||
title: "Assertion One",
|
||||
assertion: "Feature one must pass",
|
||||
});
|
||||
const assertionTwo = ms.addContractAssertion(milestone.id, {
|
||||
title: "Assertion Two",
|
||||
assertion: "Feature two must pass",
|
||||
});
|
||||
|
||||
ms.linkFeatureToAssertion(featureOne.id, assertionOne.id);
|
||||
ms.linkFeatureToAssertion(featureTwo.id, assertionTwo.id);
|
||||
|
||||
const res = await get(app, `/api/missions/milestones/${milestone.id}/validation-telemetry`);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.validationContract.assertions).toHaveLength(2);
|
||||
expect(res.body.validationContract.assertions).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: assertionOne.id,
|
||||
title: assertionOne.title,
|
||||
assertion: assertionOne.assertion,
|
||||
status: assertionOne.status,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: assertionTwo.id,
|
||||
title: assertionTwo.title,
|
||||
assertion: assertionTwo.assertion,
|
||||
status: assertionTwo.status,
|
||||
}),
|
||||
])
|
||||
);
|
||||
expect(res.body.validationContract.featureFulfillment[featureOne.id]).toEqual({
|
||||
assertionIds: [assertionOne.id],
|
||||
featureTitle: featureOne.title,
|
||||
featureStatus: featureOne.status,
|
||||
});
|
||||
expect(res.body.validationContract.featureFulfillment[featureTwo.id]).toEqual({
|
||||
assertionIds: [assertionTwo.id],
|
||||
featureTitle: featureTwo.title,
|
||||
featureStatus: featureTwo.status,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns validator rounds and generated fix-feature lineage", async () => {
|
||||
const { app, missionStore } = buildApp();
|
||||
const ms = missionStore as ReturnType<typeof createMockMissionStore>;
|
||||
|
||||
const mission = ms.createMission({ title: "Validation Mission" });
|
||||
const milestone = ms.addMilestone(mission.id, { title: "Milestone C" });
|
||||
const slice = ms.addSlice(milestone.id, { title: "Slice C" });
|
||||
const sourceFeature = ms.addFeature(slice.id, { title: "Source Feature" });
|
||||
const fixFeature = ms.addFeature(slice.id, { title: "Fix Feature" });
|
||||
const assertion = ms.addContractAssertion(milestone.id, {
|
||||
title: "Fails assertion",
|
||||
assertion: "Must not regress",
|
||||
});
|
||||
|
||||
ms.updateFeature(fixFeature.id, {
|
||||
generatedFromFeatureId: sourceFeature.id,
|
||||
generatedFromRunId: "VR-FAILED-001",
|
||||
});
|
||||
|
||||
(missionStore.getValidatorRunsByFeature as ReturnType<typeof vi.fn>).mockImplementation((featureId: string) => {
|
||||
if (featureId !== sourceFeature.id) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
id: "VR-FAILED-001",
|
||||
featureId: sourceFeature.id,
|
||||
milestoneId: milestone.id,
|
||||
sliceId: slice.id,
|
||||
status: "failed",
|
||||
implementationAttempt: 2,
|
||||
validatorAttempt: 2,
|
||||
startedAt: "2026-04-16T12:00:00.000Z",
|
||||
completedAt: "2026-04-16T12:02:00.000Z",
|
||||
createdAt: "2026-04-16T12:00:00.000Z",
|
||||
updatedAt: "2026-04-16T12:02:00.000Z",
|
||||
},
|
||||
] as MissionValidatorRun[];
|
||||
});
|
||||
|
||||
(missionStore.getFailuresForRun as ReturnType<typeof vi.fn>).mockImplementation((runId: string) => {
|
||||
if (runId !== "VR-FAILED-001") {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
id: "VAF-001",
|
||||
runId: "VR-FAILED-001",
|
||||
featureId: sourceFeature.id,
|
||||
assertionId: assertion.id,
|
||||
message: "Assertion failed",
|
||||
createdAt: "2026-04-16T12:01:00.000Z",
|
||||
},
|
||||
] as MissionAssertionFailureRecord[];
|
||||
});
|
||||
|
||||
const res = await get(app, `/api/missions/milestones/${milestone.id}/validation-telemetry`);
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.validationTelemetry.validationRounds).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
roundId: "VR-FAILED-001",
|
||||
validatorStatus: "failed",
|
||||
failedAssertionIds: [assertion.id],
|
||||
generatedFixFeatureIds: [fixFeature.id],
|
||||
}),
|
||||
])
|
||||
);
|
||||
expect(res.body.fixFeatures).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: fixFeature.id,
|
||||
sourceFeatureId: sourceFeature.id,
|
||||
runId: "VR-FAILED-001",
|
||||
failedAssertionIds: [assertion.id],
|
||||
}),
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
it("returns 404 when milestone does not exist", async () => {
|
||||
const { app } = buildApp();
|
||||
|
||||
const res = await get(app, "/api/missions/milestones/MS-MISSING-TST/validation-telemetry");
|
||||
|
||||
expect(res.status).toBe(404);
|
||||
expect(res.body.error).toBe("Milestone not found");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
|
||||
@@ -32,6 +32,7 @@ import type {
|
||||
InterviewState,
|
||||
MissionAssertionStatus,
|
||||
FeatureLoopState,
|
||||
ValidatorRunStatus,
|
||||
ContractAssertionCreateInput,
|
||||
ContractAssertionUpdateInput,
|
||||
} from "@fusion/core";
|
||||
@@ -1869,6 +1870,150 @@ export function createMissionRouter(
|
||||
})
|
||||
);
|
||||
|
||||
/**
|
||||
* GET /api/missions/milestones/:milestoneId/validation-telemetry
|
||||
*
|
||||
* Returns grouped milestone validation data by combining:
|
||||
* - Contract assertions (`listContractAssertions`) and per-feature assertion links (`listAssertionsForFeature`)
|
||||
* - Validator run rounds (`getValidatorRunsByFeature`) and failures (`getFailuresForRun`)
|
||||
* - Generated fix feature lineage (features with `generatedFromFeatureId` and `generatedFromRunId`)
|
||||
* - Milestone validation rollup (`getMilestoneValidationRollup`)
|
||||
*/
|
||||
router.get(
|
||||
"/milestones/:milestoneId/validation-telemetry",
|
||||
catchTypedHandler(async (req, res) => {
|
||||
const { milestoneId } = req.params;
|
||||
|
||||
if (!validateMilestoneId(milestoneId)) {
|
||||
throw badRequest("Invalid milestone ID format");
|
||||
}
|
||||
|
||||
const milestone = missionStore.getMilestone(milestoneId);
|
||||
if (!milestone) {
|
||||
throw notFound("Milestone not found");
|
||||
}
|
||||
|
||||
const assertions = missionStore.listContractAssertions(milestoneId);
|
||||
const slices = missionStore.listSlices(milestoneId);
|
||||
const allFeatures: MissionFeature[] = slices.flatMap((slice) => missionStore.listFeatures(slice.id));
|
||||
|
||||
const featureFulfillment: Record<string, {
|
||||
assertionIds: string[];
|
||||
featureTitle: string;
|
||||
featureStatus: string;
|
||||
}> = {};
|
||||
|
||||
for (const feature of allFeatures) {
|
||||
const linkedAssertions = missionStore.listAssertionsForFeature(feature.id);
|
||||
featureFulfillment[feature.id] = {
|
||||
assertionIds: linkedAssertions.map((assertion) => assertion.id),
|
||||
featureTitle: feature.title,
|
||||
featureStatus: feature.status,
|
||||
};
|
||||
}
|
||||
|
||||
const generatedFixFeatureIdsByRunId = new Map<string, string[]>();
|
||||
for (const feature of allFeatures) {
|
||||
const runId = feature.generatedFromRunId;
|
||||
if (!runId) continue;
|
||||
const existing = generatedFixFeatureIdsByRunId.get(runId) ?? [];
|
||||
existing.push(feature.id);
|
||||
generatedFixFeatureIdsByRunId.set(runId, existing);
|
||||
}
|
||||
|
||||
const failedAssertionIdsByRunId = new Map<string, string[]>();
|
||||
const validationRounds = [] as Array<{
|
||||
roundId: string;
|
||||
featureId: string;
|
||||
featureTitle: string;
|
||||
validatorStatus: ValidatorRunStatus;
|
||||
implementationAttempt: number;
|
||||
validatorAttempt: number;
|
||||
failedAssertionIds: string[];
|
||||
generatedFixFeatureIds: string[];
|
||||
blockedReason?: string;
|
||||
startedAt: string;
|
||||
completedAt?: string;
|
||||
}>;
|
||||
|
||||
for (const feature of allFeatures) {
|
||||
const runs = missionStore.getValidatorRunsByFeature(feature.id);
|
||||
for (const run of runs) {
|
||||
let failedAssertionIds: string[] = [];
|
||||
if (run.status === "failed") {
|
||||
failedAssertionIds = missionStore
|
||||
.getFailuresForRun(run.id)
|
||||
.map((failure) => failure.assertionId);
|
||||
failedAssertionIdsByRunId.set(run.id, failedAssertionIds);
|
||||
}
|
||||
|
||||
validationRounds.push({
|
||||
roundId: run.id,
|
||||
featureId: run.featureId,
|
||||
featureTitle: feature.title,
|
||||
validatorStatus: run.status,
|
||||
implementationAttempt: run.implementationAttempt,
|
||||
validatorAttempt: run.validatorAttempt,
|
||||
failedAssertionIds,
|
||||
generatedFixFeatureIds: generatedFixFeatureIdsByRunId.get(run.id) ?? [],
|
||||
blockedReason: run.blockedReason,
|
||||
startedAt: run.startedAt,
|
||||
completedAt: run.completedAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
validationRounds.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
|
||||
const lastValidatorStatus = validationRounds[0]?.validatorStatus ?? null;
|
||||
|
||||
const fixFeatures = allFeatures
|
||||
.filter((feature) => feature.generatedFromFeatureId && feature.generatedFromRunId)
|
||||
.map((feature) => {
|
||||
const runId = feature.generatedFromRunId as string;
|
||||
let failedAssertionIds = failedAssertionIdsByRunId.get(runId);
|
||||
|
||||
if (!failedAssertionIds) {
|
||||
failedAssertionIds = missionStore
|
||||
.getFailuresForRun(runId)
|
||||
.map((failure) => failure.assertionId);
|
||||
failedAssertionIdsByRunId.set(runId, failedAssertionIds);
|
||||
}
|
||||
|
||||
return {
|
||||
id: feature.id,
|
||||
title: feature.title,
|
||||
sourceFeatureId: feature.generatedFromFeatureId as string,
|
||||
runId,
|
||||
failedAssertionIds,
|
||||
status: feature.status,
|
||||
loopState: feature.loopState,
|
||||
};
|
||||
});
|
||||
|
||||
const rollup = missionStore.getMilestoneValidationRollup(milestoneId);
|
||||
|
||||
res.json({
|
||||
validationContract: {
|
||||
assertions: assertions.map((assertion) => ({
|
||||
id: assertion.id,
|
||||
title: assertion.title,
|
||||
assertion: assertion.assertion,
|
||||
status: assertion.status,
|
||||
orderIndex: assertion.orderIndex,
|
||||
})),
|
||||
featureFulfillment,
|
||||
},
|
||||
validationTelemetry: {
|
||||
validationRounds,
|
||||
lastValidatorStatus,
|
||||
totalRuns: validationRounds.length,
|
||||
},
|
||||
fixFeatures,
|
||||
rollup,
|
||||
});
|
||||
})
|
||||
);
|
||||
|
||||
// ── Validation & Loop State Endpoints ─────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,5 +1,15 @@
|
||||
import type { Request, Response } from "express";
|
||||
import type { TaskStore, MissionStore, PluginStore, PluginInstallation, PluginState, AgentStore, MessageStore } from "@fusion/core";
|
||||
import type {
|
||||
TaskStore,
|
||||
MissionStore,
|
||||
PluginStore,
|
||||
PluginInstallation,
|
||||
PluginState,
|
||||
AgentStore,
|
||||
MessageStore,
|
||||
MissionValidatorRun,
|
||||
FixFeatureCreatedPayload,
|
||||
} from "@fusion/core";
|
||||
import type { AiSessionStore } from "./ai-session-store.js";
|
||||
|
||||
let activeConnections = 0;
|
||||
@@ -289,6 +299,18 @@ export function createSSE(
|
||||
send(`event: milestone:validation:updated\ndata: ${JSON.stringify(data)}\n\n`);
|
||||
};
|
||||
|
||||
const onValidatorRunStarted = (run: MissionValidatorRun) => {
|
||||
send(`event: validator-run:started\ndata: ${JSON.stringify(run)}\n\n`);
|
||||
};
|
||||
|
||||
const onValidatorRunCompleted = (run: MissionValidatorRun) => {
|
||||
send(`event: validator-run:completed\ndata: ${JSON.stringify(run)}\n\n`);
|
||||
};
|
||||
|
||||
const onFixFeatureCreated = (payload: FixFeatureCreatedPayload) => {
|
||||
send(`event: fix-feature:created\ndata: ${JSON.stringify(payload)}\n\n`);
|
||||
};
|
||||
|
||||
const onAiSessionUpdated = (data: any) => {
|
||||
send(`event: ai_session:updated\ndata: ${JSON.stringify(data)}\n\n`);
|
||||
};
|
||||
@@ -402,6 +424,9 @@ export function createSSE(
|
||||
missionStore.off("assertion:unlinked", onAssertionUnlinked);
|
||||
missionStore.off("mission:event", onMissionEvent);
|
||||
missionStore.off("milestone:validation:updated", onMilestoneValidationUpdated);
|
||||
missionStore.off("validator-run:started", onValidatorRunStarted);
|
||||
missionStore.off("validator-run:completed", onValidatorRunCompleted);
|
||||
missionStore.off("fix-feature:created", onFixFeatureCreated);
|
||||
}
|
||||
if (aiSessionStore) {
|
||||
aiSessionStore.off("ai_session:updated", onAiSessionUpdated);
|
||||
@@ -459,6 +484,9 @@ export function createSSE(
|
||||
missionStore.on("assertion:unlinked", onAssertionUnlinked);
|
||||
missionStore.on("mission:event", onMissionEvent);
|
||||
missionStore.on("milestone:validation:updated", onMilestoneValidationUpdated);
|
||||
missionStore.on("validator-run:started", onValidatorRunStarted);
|
||||
missionStore.on("validator-run:completed", onValidatorRunCompleted);
|
||||
missionStore.on("fix-feature:created", onFixFeatureCreated);
|
||||
}
|
||||
|
||||
if (aiSessionStore) {
|
||||
|
||||
Reference in New Issue
Block a user