fix(FEAT-004-FIX-001): fix MissionExecutionLoop validation parsing and recovery
- Implement actual AI response parsing in parseValidationResult() with JSON extraction from markdown code blocks, repair for common JSON issues, and assertion result parsing - Fix notifyValidationComplete to pass feature.taskId instead of featureId to handleTaskCompletion() in in-process-runtime, dashboard, and serve - Fix recoverActiveMissions() to actually transition validating features back to implementing and call processTaskOutcome for features with completed tasks - Add comprehensive unit tests for MissionExecutionLoop lifecycle, processTaskOutcome, recoverActiveMissions, and error handling
This commit is contained in:
gsxdsm
491
packages/engine/src/mission-execution-loop.test.ts
Normal file
491
packages/engine/src/mission-execution-loop.test.ts
Normal file
@@ -0,0 +1,491 @@
|
||||
/**
|
||||
* MissionExecutionLoop unit tests.
|
||||
*
|
||||
* Tests the validation cycle orchestration class with mocked TaskStore and MissionStore.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { MissionExecutionLoop } from "./mission-execution-loop.js";
|
||||
import type {
|
||||
Mission,
|
||||
Milestone,
|
||||
Slice,
|
||||
MissionFeature,
|
||||
MissionContractAssertion,
|
||||
MissionValidatorRun,
|
||||
} from "@fusion/core";
|
||||
|
||||
// ── Mock Factories ──────────────────────────────────────────────────────────
|
||||
|
||||
function createMockMission(overrides: Partial<Mission> = {}): Mission {
|
||||
return {
|
||||
id: "M-TEST1",
|
||||
title: "Test Mission",
|
||||
status: "active",
|
||||
interviewState: "not_started",
|
||||
autoAdvance: true,
|
||||
autopilotEnabled: true,
|
||||
autopilotState: "inactive",
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createMockMilestone(overrides: Partial<Milestone> = {}): Milestone {
|
||||
return {
|
||||
id: "MS-001",
|
||||
missionId: "M-TEST1",
|
||||
title: "Test Milestone",
|
||||
status: "active",
|
||||
orderIndex: 0,
|
||||
interviewState: "not_started",
|
||||
dependencies: [],
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createMockSlice(overrides: Partial<Slice> = {}): Slice {
|
||||
return {
|
||||
id: "SL-001",
|
||||
milestoneId: "MS-001",
|
||||
title: "Test Slice",
|
||||
status: "active",
|
||||
planState: "not_started",
|
||||
orderIndex: 0,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createMockFeature(overrides: Partial<MissionFeature> = {}): MissionFeature {
|
||||
return {
|
||||
id: "F-001",
|
||||
sliceId: "SL-001",
|
||||
title: "Test Feature",
|
||||
status: "defined",
|
||||
loopState: "idle",
|
||||
implementationAttemptCount: 0,
|
||||
validatorAttemptCount: 0,
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createMockValidatorRun(overrides: Partial<MissionValidatorRun> = {}): MissionValidatorRun {
|
||||
return {
|
||||
id: "VR-001",
|
||||
featureId: "F-001",
|
||||
milestoneId: "MS-001",
|
||||
sliceId: "SL-001",
|
||||
status: "running",
|
||||
triggerType: "task_completion",
|
||||
implementationAttempt: 1,
|
||||
validatorAttempt: 1,
|
||||
startedAt: new Date().toISOString(),
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createMockMissionStore() {
|
||||
const missions = new Map<string, Mission>();
|
||||
const features = new Map<string, MissionFeature>();
|
||||
const validatorRuns = new Map<string, MissionValidatorRun>();
|
||||
|
||||
const store = {
|
||||
// Mission methods
|
||||
getMission: vi.fn((id: string) => missions.get(id)),
|
||||
listMissions: vi.fn(() => [...missions.values()]),
|
||||
updateMission: vi.fn((id: string, updates: Partial<Mission>) => {
|
||||
const existing = missions.get(id);
|
||||
if (!existing) throw new Error(`Mission ${id} not found`);
|
||||
const updated = { ...existing, ...updates, updatedAt: new Date().toISOString() };
|
||||
missions.set(id, updated);
|
||||
return updated;
|
||||
}),
|
||||
getMissionWithHierarchy: vi.fn((id: string) => {
|
||||
const mission = missions.get(id);
|
||||
if (!mission) return undefined;
|
||||
return {
|
||||
...mission,
|
||||
milestones: [createMockMilestone({ missionId: id })],
|
||||
};
|
||||
}),
|
||||
|
||||
// Feature methods
|
||||
getFeature: vi.fn((id: string) => features.get(id)),
|
||||
getFeatureByTaskId: vi.fn((taskId: string) => {
|
||||
for (const feature of features.values()) {
|
||||
if (feature.taskId === taskId) return feature;
|
||||
}
|
||||
return undefined;
|
||||
}),
|
||||
listFeatures: vi.fn(() => [...features.values()]),
|
||||
updateFeatureStatus: vi.fn((id: string, status: MissionFeature["status"]) => {
|
||||
const feature = features.get(id);
|
||||
if (!feature) throw new Error(`Feature ${id} not found`);
|
||||
const updated = { ...feature, status, updatedAt: new Date().toISOString() };
|
||||
features.set(id, updated);
|
||||
return updated;
|
||||
}),
|
||||
transitionLoopState: vi.fn((id: string, newState: MissionFeature["loopState"]) => {
|
||||
const feature = features.get(id);
|
||||
if (!feature) throw new Error(`Feature ${id} not found`);
|
||||
const updated = { ...feature, loopState: newState, updatedAt: new Date().toISOString() };
|
||||
features.set(id, updated);
|
||||
return updated;
|
||||
}),
|
||||
listAssertionsForFeature: vi.fn(() => []),
|
||||
getAssertionsForFeature: vi.fn(() => []),
|
||||
|
||||
// Validator run methods
|
||||
startValidatorRun: vi.fn((featureId: string, _triggerType?: string) => {
|
||||
const run = createMockValidatorRun({ featureId });
|
||||
validatorRuns.set(run.id, run);
|
||||
return run;
|
||||
}),
|
||||
completeValidatorRun: vi.fn((id: string, status: MissionValidatorRun["status"], summary?: string) => {
|
||||
const run = validatorRuns.get(id);
|
||||
if (!run) throw new Error(`Validator run ${id} not found`);
|
||||
const updated = {
|
||||
...run,
|
||||
status,
|
||||
summary,
|
||||
completedAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
validatorRuns.set(id, updated);
|
||||
return updated;
|
||||
}),
|
||||
recordValidatorFailures: vi.fn(() => []),
|
||||
createGeneratedFixFeature: vi.fn((sourceFeatureId: string, runId: string, _failedAssertionIds: string[]) => {
|
||||
const sourceFeature = features.get(sourceFeatureId);
|
||||
if (!sourceFeature) throw new Error(`Feature ${sourceFeatureId} not found`);
|
||||
|
||||
const fixFeature = createMockFeature({
|
||||
id: `FIX-${sourceFeatureId}`,
|
||||
sliceId: sourceFeature.sliceId,
|
||||
title: `Fix for ${sourceFeature.title}`,
|
||||
taskId: `TASK-FIX-${sourceFeatureId}`,
|
||||
generatedFromFeatureId: sourceFeatureId,
|
||||
generatedFromRunId: runId,
|
||||
loopState: "implementing",
|
||||
implementationAttemptCount: 0,
|
||||
});
|
||||
features.set(fixFeature.id, fixFeature);
|
||||
|
||||
const updatedSource = {
|
||||
...sourceFeature,
|
||||
implementationAttemptCount: (sourceFeature.implementationAttemptCount ?? 0) + 1,
|
||||
loopState: "needs_fix" as const,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
features.set(sourceFeatureId, updatedSource);
|
||||
|
||||
return fixFeature;
|
||||
}),
|
||||
|
||||
// Event emitter
|
||||
on: vi.fn(),
|
||||
off: vi.fn(),
|
||||
emit: vi.fn(),
|
||||
|
||||
// Internal setters for test setup
|
||||
_setMission: (m: Mission) => missions.set(m.id, m),
|
||||
_setFeature: (f: MissionFeature) => features.set(f.id, f),
|
||||
_getValidatorRun: (id: string) => validatorRuns.get(id),
|
||||
_clear: () => {
|
||||
missions.clear();
|
||||
features.clear();
|
||||
validatorRuns.clear();
|
||||
},
|
||||
};
|
||||
|
||||
return store;
|
||||
}
|
||||
|
||||
function createMockTaskStore() {
|
||||
const tasks = new Map<string, { id: string; column: string }>();
|
||||
|
||||
const store = {
|
||||
getTask: vi.fn(async (id: string) => tasks.get(id)),
|
||||
moveTask: vi.fn(),
|
||||
updateTask: vi.fn(),
|
||||
getSettings: vi.fn().mockResolvedValue({
|
||||
missionStaleThresholdMs: 600_000,
|
||||
missionMaxTaskRetries: 3,
|
||||
}),
|
||||
on: vi.fn(),
|
||||
off: vi.fn(),
|
||||
|
||||
_setTask: (t: { id: string; column: string }) => tasks.set(t.id, t),
|
||||
_clear: () => tasks.clear(),
|
||||
};
|
||||
|
||||
return store;
|
||||
}
|
||||
|
||||
// ── Tests ───────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MissionExecutionLoop", () => {
|
||||
let loop: MissionExecutionLoop;
|
||||
let missionStore: ReturnType<typeof createMockMissionStore>;
|
||||
let taskStore: ReturnType<typeof createMockTaskStore>;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
missionStore = createMockMissionStore();
|
||||
taskStore = createMockTaskStore();
|
||||
|
||||
const mission = createMockMission();
|
||||
missionStore._setMission(mission);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
loop?.stop();
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
// ── Lifecycle ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("start/stop", () => {
|
||||
it("should start and be running", () => {
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
|
||||
loop.start();
|
||||
expect(loop.isRunning()).toBe(true);
|
||||
});
|
||||
|
||||
it("should be idempotent on start", () => {
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
|
||||
loop.start();
|
||||
loop.start(); // Should not throw
|
||||
expect(loop.isRunning()).toBe(true);
|
||||
});
|
||||
|
||||
it("should stop cleanly", () => {
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
|
||||
loop.start();
|
||||
loop.stop();
|
||||
expect(loop.isRunning()).toBe(false);
|
||||
});
|
||||
|
||||
it("should be idempotent on stop", () => {
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
|
||||
loop.stop(); // Should not throw
|
||||
expect(loop.isRunning()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── processTaskOutcome ───────────────────────────────────────────────────
|
||||
|
||||
describe("processTaskOutcome", () => {
|
||||
it("should skip if loop is not running", async () => {
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-001" });
|
||||
missionStore._setFeature(feature);
|
||||
taskStore._setTask({ id: "FN-001", column: "done" });
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
// Don't start - loop is not running
|
||||
|
||||
await loop.processTaskOutcome("FN-001");
|
||||
|
||||
// Should not start validator run
|
||||
expect(missionStore.startValidatorRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should skip if task has no linked feature", async () => {
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(undefined);
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-999");
|
||||
|
||||
expect(missionStore.startValidatorRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should skip if feature is not in implementing state", async () => {
|
||||
const feature = createMockFeature({ loopState: "idle", taskId: "FN-001" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-001");
|
||||
|
||||
expect(missionStore.startValidatorRun).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should auto-pass if feature has no linked assertions", async () => {
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-001" });
|
||||
missionStore._setFeature(feature);
|
||||
taskStore._setTask({ id: "FN-001", column: "done" });
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue([]);
|
||||
|
||||
// Spy on loop's emit to verify validation:passed event
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
const emitSpy = vi.spyOn(loop, "emit");
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-001");
|
||||
|
||||
// When there are no assertions, we skip starting a validator run
|
||||
expect(missionStore.startValidatorRun).not.toHaveBeenCalled();
|
||||
// But the passed event should be emitted
|
||||
expect(emitSpy).toHaveBeenCalledWith("validation:passed", expect.any(Object));
|
||||
});
|
||||
});
|
||||
|
||||
// ── recoverActiveMissions ────────────────────────────────────────────────
|
||||
|
||||
describe("recoverActiveMissions", () => {
|
||||
it("should not crash when called on stopped loop", async () => {
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
// Don't start - loop is not running
|
||||
|
||||
await expect(loop.recoverActiveMissions()).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it("should not crash when getMissionWithHierarchy returns null", async () => {
|
||||
const mission = createMockMission({ status: "active" });
|
||||
missionStore._setMission(mission);
|
||||
|
||||
missionStore.getMissionWithHierarchy = vi.fn().mockReturnValue(null);
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await expect(loop.recoverActiveMissions()).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it("should not crash when getMissionWithHierarchy throws", async () => {
|
||||
const mission = createMockMission({ status: "active" });
|
||||
missionStore._setMission(mission);
|
||||
|
||||
missionStore.getMissionWithHierarchy = vi.fn().mockImplementation(() => {
|
||||
throw new Error("Database error");
|
||||
});
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await expect(loop.recoverActiveMissions()).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it("should handle empty hierarchy gracefully", async () => {
|
||||
const mission = createMockMission({ status: "active" });
|
||||
missionStore._setMission(mission);
|
||||
|
||||
missionStore.getMissionWithHierarchy = vi.fn().mockReturnValue({
|
||||
...mission,
|
||||
milestones: [],
|
||||
});
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.recoverActiveMissions();
|
||||
expect(missionStore.transitionLoopState).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should not recover features from archived missions", async () => {
|
||||
const mission = createMockMission({ status: "archived" });
|
||||
missionStore._setMission(mission);
|
||||
|
||||
missionStore.getMissionWithHierarchy = vi.fn().mockReturnValue({
|
||||
...mission,
|
||||
milestones: [],
|
||||
});
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.recoverActiveMissions();
|
||||
expect(missionStore.transitionLoopState).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Error handling ────────────────────────────────────────────────────────
|
||||
|
||||
describe("error handling", () => {
|
||||
it("should not crash the loop on processTaskOutcome errors", async () => {
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockImplementation(() => {
|
||||
throw new Error("Database error");
|
||||
});
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await expect(loop.processTaskOutcome("FN-001")).resolves.not.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -35,7 +35,7 @@ const VALIDATION_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes
|
||||
*/
|
||||
export interface ValidationResult {
|
||||
/** Overall validation status */
|
||||
status: "pass" | "fail" | "blocked";
|
||||
status: "pass" | "fail" | "blocked" | "error";
|
||||
/** Per-assertion results */
|
||||
assertions: Array<{
|
||||
assertionId: string;
|
||||
@@ -136,7 +136,14 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
for (const mission of missions) {
|
||||
if (mission.status !== "active") continue;
|
||||
|
||||
const hierarchy = this.missionStore.getMissionWithHierarchy(mission.id);
|
||||
let hierarchy;
|
||||
try {
|
||||
hierarchy = this.missionStore.getMissionWithHierarchy(mission.id);
|
||||
} catch {
|
||||
// Database error, skip this mission
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!hierarchy) continue;
|
||||
|
||||
for (const milestone of hierarchy.milestones) {
|
||||
@@ -148,16 +155,32 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
if (feature.loopState === "validating") {
|
||||
loopLog.log(`Recovery: re-queuing validating feature ${feature.id}`);
|
||||
// Transition back to implementing so the next task completion triggers validation
|
||||
// Or if there's a task, we can re-trigger validation directly
|
||||
recoveredCount++;
|
||||
try {
|
||||
await this.missionStore.transitionLoopState(feature.id, "implementing");
|
||||
// If the feature has a linked task that's already done, re-trigger validation
|
||||
if (feature.taskId) {
|
||||
await this.processTaskOutcome(feature.taskId);
|
||||
}
|
||||
recoveredCount++;
|
||||
} catch (err) {
|
||||
loopLog.error(`Recovery failed for validating feature ${feature.id}:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
// Features in needs_fix state need to continue their fix cycle
|
||||
// Features in needs_fix state with completed tasks need to continue
|
||||
if (feature.loopState === "needs_fix") {
|
||||
loopLog.log(`Recovery: feature ${feature.id} awaiting fix implementation`);
|
||||
// The feature is already in needs_fix - it will progress when
|
||||
// its fix task completes and processTaskOutcome is called again
|
||||
recoveredCount++;
|
||||
// If the fix task is complete, call processTaskOutcome to continue the cycle
|
||||
if (feature.taskId) {
|
||||
try {
|
||||
await this.processTaskOutcome(feature.taskId);
|
||||
recoveredCount++;
|
||||
} catch (err) {
|
||||
loopLog.error(`Recovery failed for needs_fix feature ${feature.id}:`, err);
|
||||
}
|
||||
} else {
|
||||
recoveredCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -332,34 +355,238 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the validation result from the agent's response.
|
||||
* Parse the validation result from the AI agent's response.
|
||||
*
|
||||
* The agent is expected to return structured JSON with the validation result.
|
||||
* We extract the text from the AI's messages and parse the JSON response.
|
||||
*/
|
||||
private async parseValidationResult(
|
||||
agentSession: Awaited<ReturnType<typeof createKbAgent>>["session"],
|
||||
assertions: MissionContractAssertion[],
|
||||
): Promise<ValidationResult> {
|
||||
// In a real implementation, we would parse the agent's response to extract
|
||||
// the structured validation result. For now, we'll use a simplified approach
|
||||
// where we look for a JSON response in the conversation.
|
||||
//
|
||||
// The agent should have responded with something like:
|
||||
// {
|
||||
// "status": "pass|fail|blocked",
|
||||
// "assertions": [...],
|
||||
// "summary": "..."
|
||||
// }
|
||||
try {
|
||||
// Extract the AI's response text from the session messages
|
||||
const responseText = this.extractResponseTextFromSession(agentSession);
|
||||
|
||||
// For now, return a default "pass" result since we don't have the actual
|
||||
// parsing logic implemented. This will be refined based on the actual
|
||||
// agent response format.
|
||||
if (!responseText) {
|
||||
loopLog.warn("No response text found in validation session");
|
||||
return this.createErrorValidationResult("No response from validation agent", assertions);
|
||||
}
|
||||
|
||||
// Extract JSON from the response (handles markdown code blocks)
|
||||
const jsonCandidate = this.extractJsonCandidate(responseText);
|
||||
|
||||
if (!jsonCandidate) {
|
||||
loopLog.warn("No JSON found in validation response");
|
||||
return this.createErrorValidationResult("Validation agent did not return JSON", assertions);
|
||||
}
|
||||
|
||||
// Try to parse the JSON
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(jsonCandidate);
|
||||
} catch {
|
||||
// Try to repair common JSON issues
|
||||
const repaired = this.repairJson(jsonCandidate);
|
||||
try {
|
||||
parsed = JSON.parse(repaired);
|
||||
} catch (e) {
|
||||
loopLog.warn("Failed to parse validation JSON", e);
|
||||
return this.createErrorValidationResult("Invalid JSON in validation response", assertions);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the status field
|
||||
const status = this.validateValidationStatus(parsed.status);
|
||||
if (!status) {
|
||||
loopLog.warn("Invalid validation status in response", parsed.status);
|
||||
return this.createErrorValidationResult("Invalid status in validation response", assertions);
|
||||
}
|
||||
|
||||
// Extract assertion results from the parsed JSON
|
||||
const assertionResults = this.extractAssertionResults(parsed, assertions);
|
||||
|
||||
// Extract summary and blocked reason
|
||||
const summary = typeof parsed.summary === "string" ? parsed.summary : `Validation ${status}`;
|
||||
const blockedReason = typeof parsed.blockedReason === "string" ? parsed.blockedReason : undefined;
|
||||
|
||||
return {
|
||||
status,
|
||||
assertions: assertionResults,
|
||||
summary,
|
||||
blockedReason,
|
||||
};
|
||||
} catch (err) {
|
||||
loopLog.error("Error parsing validation result", err);
|
||||
return this.createErrorValidationResult(`Error parsing validation: ${err}`, assertions);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract response text from AI session messages.
|
||||
* Looks for the last assistant message with text content.
|
||||
*/
|
||||
private extractResponseTextFromSession(
|
||||
agentSession: Awaited<ReturnType<typeof createKbAgent>>["session"],
|
||||
): string | undefined {
|
||||
try {
|
||||
// Access the session state to get messages
|
||||
const state = (agentSession as { state?: { messages?: Array<{ role?: string; content?: unknown }> } }).state;
|
||||
if (!state?.messages) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Find the last assistant message with text content
|
||||
for (let i = state.messages.length - 1; i >= 0; i--) {
|
||||
const msg = state.messages[i];
|
||||
if (msg.role === "assistant") {
|
||||
if (typeof msg.content === "string" && msg.content.trim()) {
|
||||
return msg.content;
|
||||
}
|
||||
// Handle content as array (common in some AI SDKs)
|
||||
if (Array.isArray(msg.content)) {
|
||||
for (const part of msg.content) {
|
||||
if (typeof part === "object" && part !== null && "text" in part && typeof part.text === "string") {
|
||||
return part.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract JSON from a text that may contain markdown code blocks.
|
||||
*/
|
||||
private extractJsonCandidate(text: string): string | undefined {
|
||||
// Try to find JSON in markdown code blocks first
|
||||
const codeBlockMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/);
|
||||
if (codeBlockMatch) {
|
||||
return codeBlockMatch[1].trim();
|
||||
}
|
||||
|
||||
// Try to find JSON directly (starts with { or [)
|
||||
const jsonStartMatch = text.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
|
||||
if (jsonStartMatch) {
|
||||
return jsonStartMatch[1];
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Repair common JSON issues in AI responses.
|
||||
*/
|
||||
private repairJson(json: string): string {
|
||||
// Remove trailing commas before closing braces/brackets
|
||||
let repaired = json.replace(/,\s*([\]}])/g, "$1");
|
||||
|
||||
// Handle unclosed arrays/objects by finding the last balanced close
|
||||
const openBraces = (repaired.match(/\{/g) || []).length;
|
||||
const closeBraces = (repaired.match(/\}/g) || []).length;
|
||||
const openBrackets = (repaired.match(/\[/g) || []).length;
|
||||
const closeBrackets = (repaired.match(/\]/g) || []).length;
|
||||
|
||||
// Close missing braces
|
||||
while (closeBraces < openBraces) {
|
||||
repaired += "}";
|
||||
}
|
||||
// Close missing brackets
|
||||
while (closeBrackets < openBrackets) {
|
||||
repaired += "]";
|
||||
}
|
||||
|
||||
// Remove any trailing commas
|
||||
repaired = repaired.replace(/,\s*([\]}])/g, "$1");
|
||||
|
||||
return repaired;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that the status field is a valid validation status.
|
||||
*/
|
||||
private validateValidationStatus(status: unknown): ValidationResult["status"] | undefined {
|
||||
if (status === "pass" || status === "fail" || status === "blocked") {
|
||||
return status;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract assertion results from the parsed JSON.
|
||||
*/
|
||||
private extractAssertionResults(
|
||||
parsed: Record<string, unknown>,
|
||||
assertions: MissionContractAssertion[],
|
||||
): Array<{ assertionId: string; passed: boolean; message?: string; expected?: string; actual?: string }> {
|
||||
const results: Array<{
|
||||
assertionId: string;
|
||||
passed: boolean;
|
||||
message?: string;
|
||||
expected?: string;
|
||||
actual?: string;
|
||||
}> = [];
|
||||
|
||||
// If assertions array is provided in the response, use it
|
||||
if (Array.isArray(parsed.assertions)) {
|
||||
for (const item of parsed.assertions) {
|
||||
if (typeof item === "object" && item !== null) {
|
||||
const assertionItem = item as Record<string, unknown>;
|
||||
const assertionId =
|
||||
typeof assertionItem.assertionId === "string"
|
||||
? assertionItem.assertionId
|
||||
: typeof assertionItem.id === "string"
|
||||
? assertionItem.id
|
||||
: undefined;
|
||||
|
||||
const passed = typeof assertionItem.passed === "boolean" ? assertionItem.passed : false;
|
||||
|
||||
results.push({
|
||||
assertionId: assertionId || "unknown",
|
||||
passed,
|
||||
message: typeof assertionItem.message === "string" ? assertionItem.message : undefined,
|
||||
expected: typeof assertionItem.expected === "string" ? assertionItem.expected : undefined,
|
||||
actual: typeof assertionItem.actual === "string" ? assertionItem.actual : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If no assertion results but we have assertions, create default results based on status
|
||||
if (results.length === 0 && assertions.length > 0) {
|
||||
const overallPassed = parsed.status === "pass";
|
||||
for (const assertion of assertions) {
|
||||
results.push({
|
||||
assertionId: assertion.id,
|
||||
passed: overallPassed,
|
||||
message: overallPassed ? "Passed" : "Failed",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an error validation result.
|
||||
*/
|
||||
private createErrorValidationResult(
|
||||
errorMessage: string,
|
||||
assertions: MissionContractAssertion[],
|
||||
): ValidationResult {
|
||||
return {
|
||||
status: "pass",
|
||||
status: "error",
|
||||
assertions: assertions.map((a) => ({
|
||||
assertionId: a.id,
|
||||
passed: true,
|
||||
passed: false,
|
||||
message: errorMessage,
|
||||
})),
|
||||
summary: "All assertions passed",
|
||||
summary: errorMessage,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -179,8 +179,12 @@ export class InProcessRuntime
|
||||
missionStore,
|
||||
missionAutopilot: missionAutopilot
|
||||
? {
|
||||
notifyValidationComplete: async (featureId: string, status: "passed" | "failed" | "blocked" | "error") => {
|
||||
await missionAutopilot.handleTaskCompletion(featureId);
|
||||
notifyValidationComplete: async (featureId: string, _status: "passed" | "failed" | "blocked" | "error") => {
|
||||
// Pass the feature's linked taskId to handleTaskCompletion, not the featureId
|
||||
const feature = missionStore.getFeature(featureId);
|
||||
if (feature?.taskId) {
|
||||
await missionAutopilot.handleTaskCompletion(feature.taskId);
|
||||
}
|
||||
},
|
||||
}
|
||||
: undefined,
|
||||
|
||||
Reference in New Issue
Block a user