Files
fusion/packages/engine/src/mission-execution-loop.ts
gsxdsm 4ea9d66b4f FN-6219: prefer fresh agent model settings for automatic runs
Automatic agent runs now resolve lane-specific task and settings models before falling back to durable runtime defaults.

- Prefer fresh execution, planning, heartbeat, merger, and validator model resolution over stale assigned-agent runtime config when complete settings are available.
- Centralize validator session model resolution and align mission validation with the shared helper.
- Add regression coverage for automatic run model precedence and test-mode handling.
- Add a patch changeset for the published CLI package.

Files changed:
 .changeset/fuzzy-agents-run.md                     |   5 +
 .../core/src/__tests__/model-resolution.test.ts    |  40 +++++
 .../agent-session-helpers-test-mode.test.ts        |  35 ++--
 .../src/__tests__/agent-session-helpers.test.ts    | 199 ++++++++++++++++++---
 .../src/__tests__/heartbeat-executor.test.ts       |  10 +-
 .../src/__tests__/mission-execution-loop.test.ts   |   6 +-
 packages/engine/src/__tests__/triage.test.ts       |   4 +-
 packages/engine/src/agent-session-helpers.ts       | 111 +++++++-----
 packages/engine/src/mission-execution-loop.ts      |  33 +---
 packages/engine/src/triage.ts                      |   6 +-
 10 files changed, 326 insertions(+), 123 deletions(-)

Fusion-Task-Id: FN-6219
Fusion-Task-Lineage: c3ef6d91-234f-4ed1-97c7-9be8408cd86c
2026-06-11 09:21:23 -07:00

1187 lines
43 KiB
TypeScript

/**
* MissionExecutionLoop — Orchestrates the validation cycle for mission features.
*
* After a task completes, the loop:
* 1. Transitions the feature from "implementing" to "validating"
* 2. Runs an AI agent to evaluate the implementation against contract assertions
* 3. Based on the validation result:
* - pass: marks feature as "passed", enables slice advancement
* - fail: creates a fix feature with failure context, decrements retry budget
* - blocked: marks feature as "blocked" (external blocker)
* - error: keeps feature in "validating" for retry
*/
import { EventEmitter } from "node:events";
import type {
TaskStore,
MissionStore,
MissionContractAssertion,
MissionFeature,
MissionValidatorRun,
AgentStore,
Settings,
Milestone,
} from "@fusion/core";
import { createFnAgent, promptWithFallback, type AgentResult } from "./pi.js";
import { mergeEffectiveSettings } from "./effective-settings.js";
import {
createResolvedAgentSession,
extractRuntimeHint,
resolveValidatorSessionModel,
} from "./agent-session-helpers.js";
import { createLogger } from "./logger.js";
import { createFallbackModelObserver } from "./fallback-model-observer.js";
import { createRunAuditor, generateSyntheticRunId } from "./run-audit.js";
/** Logger for the mission execution loop subsystem. */
export const loopLog = createLogger("mission-loop");
/** Maximum time (ms) to wait for a validation session to complete. */
const VALIDATION_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes
/**
* Validation result returned by the AI agent.
* The agent evaluates each linked assertion and returns pass/fail/blocked
* per assertion plus an overall status.
*/
export interface ValidationResult {
/** Overall validation status */
status: "pass" | "fail" | "blocked" | "error";
/** Per-assertion results */
assertions: Array<{
assertionId: string;
passed: boolean;
message?: string;
expected?: string;
actual?: string;
}>;
/** Summary message for overall result */
summary: string;
/** If blocked, the reason for the block */
blockedReason?: string;
}
export interface MissionExecutionLoopOptions {
/** Task store for accessing task data */
taskStore: TaskStore;
/** Mission store for accessing mission/feature data */
missionStore: MissionStore;
/** Optional MissionAutopilot for notifying on loop state changes */
missionAutopilot?: {
notifyValidationComplete?: (featureId: string, status: "passed" | "failed" | "blocked" | "error") => void | Promise<void>;
};
/** Root directory for worktree operations */
rootDir: string;
/** Maximum implementation retry budget (default: 3) */
maxRetryBudget?: number;
/** Plugin runner for runtime selection. When provided, enables plugin runtime lookup. */
pluginRunner?: import("./plugin-runner.js").PluginRunner;
/** Optional agent store for resolving assigned-agent runtime hints. */
agentStore?: AgentStore;
}
export class MissionExecutionLoop extends EventEmitter {
private running = false;
private taskStore: TaskStore;
private missionStore: MissionStore;
private rootDir: string;
private maxRetryBudget: number;
private missionAutopilot?: MissionExecutionLoopOptions["missionAutopilot"];
private pluginRunner?: MissionExecutionLoopOptions["pluginRunner"];
private agentStore?: MissionExecutionLoopOptions["agentStore"];
private activeValidations = new Set<string>(); // feature IDs currently being validated
constructor(options: MissionExecutionLoopOptions) {
super();
this.taskStore = options.taskStore;
this.missionStore = options.missionStore;
this.rootDir = options.rootDir;
this.maxRetryBudget = options.maxRetryBudget ?? 3;
this.missionAutopilot = options.missionAutopilot;
this.pluginRunner = options.pluginRunner;
this.agentStore = options.agentStore;
loopLog.log("MissionExecutionLoop created");
}
/**
* Start the execution loop.
* Currently a no-op since the loop is event-driven, but may be used
* for future background processing.
*/
start(): void {
if (this.running) return;
this.running = true;
loopLog.log("MissionExecutionLoop started");
}
/**
* Stop the execution loop.
* Aborts any in-progress validations.
*/
stop(): void {
if (!this.running) return;
this.running = false;
// Abort any active validations
for (const featureId of this.activeValidations) {
loopLog.warn(`Aborting in-progress validation for feature ${featureId}`);
}
this.activeValidations.clear();
loopLog.log("MissionExecutionLoop stopped");
}
/**
* Check if the loop is currently running.
*/
isRunning(): boolean {
return this.running;
}
/**
* Reap validator runs that have been left in status='running' beyond the stale window.
*
* Runs still actively owned by this process are skipped so live validations are never
* terminated by maintenance while their session is still in-flight.
*/
async reapStaleValidatorRuns(maxAgeMs: number): Promise<{ reapedCount: number }> {
const staleRuns = this.missionStore.listStaleRunningValidatorRuns(maxAgeMs);
let reapedCount = 0;
for (const run of staleRuns) {
if (this.activeValidations.has(run.featureId)) {
continue;
}
try {
const reapedRun = this.missionStore.reapValidatorRun(
run.id,
`Validator run reaped after exceeding stale threshold (${maxAgeMs}ms) without a live owner.`,
);
reapedCount += 1;
try {
const milestone = this.missionStore.getMilestone(reapedRun.milestoneId);
const missionId = milestone ? this.missionStore.getMission(milestone.missionId)?.id : undefined;
const elapsedMs = Math.max(0, Date.now() - new Date(run.startedAt).getTime());
this.taskStore.recordRunAuditEvent({
agentId: "store",
runId: "validator-run-reaper",
domain: "database",
mutationType: "mission:validator-run-reaped",
target: reapedRun.id,
metadata: {
runId: reapedRun.id,
featureId: reapedRun.featureId,
missionId,
triggerType: reapedRun.triggerType,
elapsedMs,
},
});
} catch (auditErr) {
loopLog.warn(`Failed to record validator-run reaper audit for ${run.id}:`, auditErr);
}
} catch (err) {
loopLog.warn(`Failed to reap stale validator run ${run.id}:`, err);
}
}
return { reapedCount };
}
/**
* Recover active missions on startup.
*
* Finds all features in "validating" or "needs_fix" state and re-enqueues
* them for validation or fix implementation respectively.
*
* This handles the case where the engine was shut down mid-validation
* or mid-fix, ensuring those features continue their loop progression.
*/
async recoverActiveMissions(): Promise<{ recoveredCount: number }> {
loopLog.log("Starting active mission recovery...");
if (!this.running) {
loopLog.warn("recoverActiveMissions called while loop is stopped; starting loop for recovery");
this.start();
}
try {
const missions = this.missionStore.listMissions();
let recoveredCount = 0;
for (const mission of missions) {
if (mission.status !== "active") continue;
let hierarchy;
try {
hierarchy = this.missionStore.getMissionWithHierarchy(mission.id);
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
loopLog.warn(`getMissionWithHierarchy failed for mission ${mission.id}: ${errorMessage} — skipping`);
// Database error, skip this mission
continue;
}
if (!hierarchy) continue;
for (const milestone of hierarchy.milestones) {
for (const slice of milestone.slices) {
if (slice.status !== "active") continue;
for (const feature of slice.features) {
// Features in validating state need to be re-validated
if (feature.loopState === "validating") {
loopLog.log(`Recovery: re-queuing validating feature ${feature.id}`);
// Transition back to implementing so the next task completion triggers validation
try {
await this.missionStore.transitionLoopState(feature.id, "implementing");
// If the feature has a linked task that's already done, re-trigger validation
if (feature.taskId) {
const linkedTask = await this.taskStore.getTask(feature.taskId).catch(() => null);
if (linkedTask && (linkedTask.column === "done" || linkedTask.column === "archived")) {
await this.processTaskOutcome(feature.taskId);
}
}
recoveredCount++;
} catch (err) {
loopLog.error(`Recovery failed for validating feature ${feature.id}:`, err);
}
}
// Features in needs_fix state with completed tasks need to continue
if (feature.loopState === "needs_fix") {
loopLog.log(`Recovery: feature ${feature.id} awaiting fix implementation`);
// If the fix task is complete, call processTaskOutcome to continue the cycle
if (feature.taskId) {
try {
const linkedTask = await this.taskStore.getTask(feature.taskId).catch(() => null);
if (linkedTask && (linkedTask.column === "done" || linkedTask.column === "archived")) {
await this.processTaskOutcome(feature.taskId);
}
recoveredCount++;
} catch (err) {
loopLog.error(`Recovery failed for needs_fix feature ${feature.id}:`, err);
}
} else {
recoveredCount++;
}
}
// Features that remained implementing while their linked task already finished
// can be stranded after restart; recover by re-triggering task outcome.
if (feature.loopState === "implementing" && feature.taskId) {
const currentFeature = this.missionStore.getFeature(feature.id) ?? feature;
if (
this.activeValidations.has(feature.id)
|| currentFeature.loopState === "passed"
|| currentFeature.lastValidatorStatus === "passed"
) {
continue;
}
try {
const linkedTask = await this.taskStore.getTask(feature.taskId).catch(() => null);
if (linkedTask && (linkedTask.column === "done" || linkedTask.column === "archived")) {
loopLog.log(`Recovery: re-triggering implementing feature ${feature.id} from completed task ${feature.taskId}`);
await this.processTaskOutcome(feature.taskId);
recoveredCount++;
}
} catch (err) {
loopLog.error(`Recovery failed for implementing feature ${feature.id}:`, err);
}
}
// Features marked "done" but stranded in "implementing" with no
// linked task can never validate on their own: the branches above
// only re-drive features that still carry a taskId. Meanwhile the
// slice-completion gate (MissionStore.computeSliceStatus) refuses
// to count an assertion-linked "done" feature until its validator
// passes — so the slice, milestone, and mission can never
// auto-progress. Re-drive validation directly so the gate can
// resolve. Validation is a read-only judge (no board task, no code
// changes); on pass the feature becomes legitimately complete, on
// fail the normal fix-feature flow takes over.
if (
feature.loopState === "implementing"
&& !feature.taskId
&& feature.status === "done"
&& feature.lastValidatorStatus !== "passed"
&& !this.activeValidations.has(feature.id)
) {
const currentFeature = this.missionStore.getFeature(feature.id) ?? feature;
if (
currentFeature.loopState === "passed"
|| currentFeature.lastValidatorStatus === "passed"
) {
continue;
}
try {
loopLog.warn(
`Recovery: re-validating stranded "done" feature ${feature.id} `
+ `(loopState=${feature.loopState}, no linked task) so its slice can complete`,
);
recoveredCount++;
await this.runFeatureValidation(currentFeature);
} catch (err) {
loopLog.error(`Recovery failed for stranded done feature ${feature.id}:`, err);
}
}
}
}
}
}
loopLog.log(`Active mission recovery complete: recovered ${recoveredCount} features`);
return { recoveredCount };
} catch (err) {
loopLog.error("Error during active mission recovery:", err);
return { recoveredCount: 0 };
}
}
/**
* Process the outcome of a completed mission-linked task.
*
* Called by the Scheduler when a task with a sliceId moves to "done".
* Triggers the validation cycle for the linked feature.
*
* @param taskId - The completed task ID
*/
async processTaskOutcome(taskId: string): Promise<void> {
if (!this.running) {
loopLog.warn(`processTaskOutcome called but loop is not running; ignoring ${taskId}`);
return;
}
loopLog.log(`Processing task outcome for ${taskId}`);
try {
// Find the feature linked to this task
const feature = this.missionStore.getFeatureByTaskId(taskId);
if (!feature) {
loopLog.log(`Task ${taskId} has no linked feature; skipping validation`);
return;
}
if (feature.loopState === "needs_fix") {
this.missionStore.transitionLoopState(feature.id, "implementing");
feature.loopState = "implementing";
}
// Only validate features in "implementing" state
if (feature.loopState !== "implementing") {
loopLog.log(`Feature ${feature.id} loopState is "${feature.loopState}"; skipping validation`);
this.logFeatureWarningEvent(feature.id, "validation_skipped_loop_state", `Validation skipped: feature ${feature.id} is in loopState "${feature.loopState}" (expected "implementing").`, {
taskId,
loopState: feature.loopState,
});
return;
}
if (this.activeValidations.has(feature.id)) {
loopLog.log(`Feature ${feature.id} already has an active validation; skipping duplicate trigger`);
this.logFeatureWarningEvent(feature.id, "validation_deduplicated", `Validation already running for feature ${feature.id}; duplicate trigger ignored.`, {
taskId,
});
return;
}
await this.runFeatureValidation(feature);
} catch (err) {
loopLog.error(`Error processing task outcome for ${taskId}:`, err);
// Don't crash the loop - log and continue
}
}
/**
* Run assertion validation for a feature and apply the outcome.
*
* Shared by processTaskOutcome (task-triggered) and recoverActiveMissions
* (self-healing for features stranded mid-loop with no board task). Callers
* are responsible for confirming the feature is eligible to validate; this
* method handles lazy assertion linkage, validator run bookkeeping, and
* dispatch of the validation result.
*/
private async runFeatureValidation(feature: MissionFeature): Promise<void> {
// Lazily guarantee a linked assertion before validation so every feature
// is evaluated by the validator even when legacy data is missing links.
let assertions = this.missionStore.listAssertionsForFeature(feature.id);
if (assertions.length === 0) {
loopLog.log(`Feature ${feature.id} has no linked assertions; lazily ensuring store-managed assertion linkage`);
assertions = this.missionStore.ensureFeatureAssertionLinked(feature.id);
}
// Mark feature as being validated
this.activeValidations.add(feature.id);
try {
loopLog.log(`Running internal validation for feature ${feature.id} — no board task created (policy: docs/missions.md)`);
// Start the validator run (no board task per docs/missions.md)
const run = this.missionStore.startValidatorRun(feature.id, "task_completion");
loopLog.log(`Started validator run ${run.id} for feature ${feature.id}`);
// Run the validation
const result = await this.runValidation(feature, assertions, run);
// Handle the result
if (result.status === "pass") {
await this.handleValidationPass(feature.id, run.id, result.summary);
} else if (result.status === "fail") {
await this.handleValidationFail(feature.id, run.id, result);
} else if (result.status === "blocked") {
await this.handleValidationBlocked(feature.id, run.id, result.blockedReason);
} else if (result.status === "error") {
await this.handleValidationError(feature.id, run.id, result.summary);
}
} finally {
this.activeValidations.delete(feature.id);
}
}
/**
* Run the validation AI session for a feature.
*
* Creates a fresh AI agent session with a validation system prompt,
* evaluates the implementation against the linked assertions, and
* returns the structured validation result.
*/
private async runValidation(
feature: MissionFeature,
assertions: MissionContractAssertion[],
_run: MissionValidatorRun,
): Promise<ValidationResult> {
loopLog.log(`Running validation for feature ${feature.id} with ${assertions.length} assertions`);
const milestone = this.resolveFeatureMilestone(feature);
// Build the validation prompt
const prompt = this.buildValidationPrompt(feature, assertions, milestone);
// Get task context for validation
const task = feature.taskId ? await this.taskStore.getTask(feature.taskId) : null;
const taskContext = task ? this.buildTaskContext(task) : "";
const assignedAgent = task?.assignedAgentId && this.agentStore
? await this.agentStore.getAgent(task.assignedAgentId).catch(() => null)
: null;
const validationRuntimeHint = extractRuntimeHint(assignedAgent?.runtimeConfig);
// Merge per-task effective workflow settings (U3, KTD-3) so the validator
// model-lane reads pick up workflow values; skip when there is no task in
// scope (mission-level validation has no per-task workflow). Behavior-inert by
// default.
const baseSettings = await this.taskStore.getSettings().catch(() => undefined);
const settings = task && baseSettings
? await mergeEffectiveSettings(this.taskStore, task, baseSettings)
: baseSettings;
const validationSessionModel = this.resolveValidationSessionModel(
task,
settings,
assignedAgent?.runtimeConfig,
);
let session: AgentResult | null = null;
try {
// Create validation agent session
const runAuditor = createRunAuditor(this.taskStore, {
runId: generateSyntheticRunId("mission", feature.taskId ?? feature.id),
agentId: "reviewer",
taskId: task?.id,
phase: "mission",
source: "mission-execution-loop",
});
const sessionResult = await createResolvedAgentSession({
sessionPurpose: "validation",
runtimeHint: validationRuntimeHint,
pluginRunner: this.pluginRunner,
cwd: this.rootDir,
systemPrompt: this.buildValidationSystemPrompt(feature, assertions, taskContext, milestone),
tools: "readonly",
defaultProvider: validationSessionModel.provider,
defaultModelId: validationSessionModel.modelId,
fallbackProvider: settings?.fallbackProvider,
fallbackModelId: settings?.fallbackModelId,
defaultThinkingLevel: "medium",
runAuditor,
settings,
onText: (_delta) => {
// Could stream this to a log entry if needed
},
taskId: task?.id,
taskTitle: task?.title,
onFallbackModelUsed: createFallbackModelObserver({
agent: "reviewer",
label: "mission validator",
store: this.taskStore,
taskId: task?.id,
taskTitle: task?.title,
}),
});
session = { session: sessionResult.session, sessionFile: sessionResult.sessionFile };
loopLog.log(`Validation session created for feature ${feature.id}`);
// Run the validation with timeout
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error("Validation timeout")), VALIDATION_TIMEOUT_MS);
});
const validationPromise = this.runValidationSession(session.session, prompt);
await Promise.race([validationPromise, timeoutPromise]);
// Get the validation result from the session
// The agent should have returned structured JSON in its response
const result = await this.parseValidationResult(session.session, assertions);
loopLog.log(`Validation completed for feature ${feature.id}: ${result.status}`);
return result;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
loopLog.error(`Validation error for feature ${feature.id}:`, message);
// Return an error result - the loop will handle it
return {
status: "error",
assertions: assertions.map((a) => ({
assertionId: a.id,
passed: false,
message: `Validation error: ${message}`,
})),
summary: `Validation failed due to error: ${message}`,
};
} finally {
// Always dispose the session
if (session) {
try {
session.session.dispose();
loopLog.log(`Validation session disposed for feature ${feature.id}`);
} catch (disposeErr) {
loopLog.warn(`Error disposing validation session for ${feature.id}:`, disposeErr);
}
}
}
}
private resolveValidationSessionModel(
task: Awaited<ReturnType<TaskStore["getTask"]>> | null,
settings: Partial<Settings> | undefined,
assignedAgentRuntimeConfig?: Record<string, unknown>,
): { provider: string | undefined; modelId: string | undefined } {
return resolveValidatorSessionModel(
task?.validatorModelProvider,
task?.validatorModelId,
settings,
assignedAgentRuntimeConfig,
);
}
/**
* Run the actual validation session with the AI agent.
*/
private async runValidationSession(
agentSession: Awaited<ReturnType<typeof createFnAgent>>["session"],
prompt: string,
): Promise<void> {
// Use promptWithFallback for resilience - if the primary model fails,
// it will automatically try the fallback model
await promptWithFallback(
agentSession as Parameters<typeof promptWithFallback>[0],
prompt,
);
}
/**
* Parse the validation result from the AI agent's response.
*
* The agent is expected to return structured JSON with the validation result.
* We extract the text from the AI's messages and parse the JSON response.
*/
private async parseValidationResult(
agentSession: Awaited<ReturnType<typeof createFnAgent>>["session"],
assertions: MissionContractAssertion[],
): Promise<ValidationResult> {
try {
// Extract the AI's response text from the session messages
const responseText = this.extractResponseTextFromSession(agentSession);
if (!responseText) {
loopLog.warn("No response text found in validation session");
return this.createErrorValidationResult("No response from validation agent", assertions);
}
// Extract JSON from the response (handles markdown code blocks)
const jsonCandidate = this.extractJsonCandidate(responseText);
if (!jsonCandidate) {
loopLog.warn("No JSON found in validation response");
return this.createErrorValidationResult("Validation agent did not return JSON", assertions);
}
// Try to parse the JSON
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(jsonCandidate);
} catch {
// Intentional fallback: initial parse can fail on malformed JSON; try repairJson() next.
const repaired = this.repairJson(jsonCandidate);
try {
parsed = JSON.parse(repaired);
} catch (e) {
loopLog.warn("Failed to parse validation JSON", e);
return this.createErrorValidationResult("Invalid JSON in validation response", assertions);
}
}
// Validate the status field
const status = this.validateValidationStatus(parsed.status);
if (!status) {
loopLog.warn("Invalid validation status in response", parsed.status);
return this.createErrorValidationResult("Invalid status in validation response", assertions);
}
// Extract assertion results from the parsed JSON
const assertionResults = this.extractAssertionResults(parsed, assertions);
// Extract summary and blocked reason
const summary = typeof parsed.summary === "string" ? parsed.summary : `Validation ${status}`;
const blockedReason = typeof parsed.blockedReason === "string" ? parsed.blockedReason : undefined;
return {
status,
assertions: assertionResults,
summary,
blockedReason,
};
} catch (err) {
loopLog.error("Error parsing validation result", err);
return this.createErrorValidationResult(`Error parsing validation: ${err}`, assertions);
}
}
/**
* Extract response text from AI session messages.
* Looks for the last assistant message with text content.
*/
private extractResponseTextFromSession(
agentSession: Awaited<ReturnType<typeof createFnAgent>>["session"],
): string | undefined {
try {
// Access the session state to get messages
const state = (agentSession as { state?: { messages?: Array<{ role?: string; content?: unknown }> } }).state;
if (!state?.messages) {
return undefined;
}
// Find the last assistant message with text content
for (let i = state.messages.length - 1; i >= 0; i--) {
const msg = state.messages[i];
if (msg.role === "assistant") {
if (typeof msg.content === "string" && msg.content.trim()) {
return msg.content;
}
// Handle content as array (common in some AI SDKs)
if (Array.isArray(msg.content)) {
for (const part of msg.content) {
if (typeof part === "object" && part !== null && "text" in part && typeof part.text === "string") {
return part.text;
}
}
}
}
}
return undefined;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
loopLog.warn(`AI response JSON extraction failed: ${errorMessage}`);
return undefined;
}
}
/**
* Extract JSON from a text that may contain markdown code blocks.
*/
private extractJsonCandidate(text: string): string | undefined {
// Try to find JSON in markdown code blocks first
const codeBlockMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/);
if (codeBlockMatch) {
return codeBlockMatch[1].trim();
}
// Try to find JSON directly (starts with { or [)
const jsonStartMatch = text.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
if (jsonStartMatch) {
return jsonStartMatch[1];
}
return undefined;
}
/**
* Repair common JSON issues in AI responses.
*/
private repairJson(json: string): string {
// Remove trailing commas before closing braces/brackets
let repaired = json.replace(/,\s*([\]}])/g, "$1");
// Handle unclosed arrays/objects by finding the last balanced close
const openBraces = (repaired.match(/\{/g) || []).length;
const closeBraces = (repaired.match(/\}/g) || []).length;
const openBrackets = (repaired.match(/\[/g) || []).length;
const closeBrackets = (repaired.match(/\]/g) || []).length;
// Close missing braces
while (closeBraces < openBraces) {
repaired += "}";
}
// Close missing brackets
while (closeBrackets < openBrackets) {
repaired += "]";
}
// Remove any trailing commas
repaired = repaired.replace(/,\s*([\]}])/g, "$1");
return repaired;
}
/**
* Validate that the status field is a valid validation status.
*/
private validateValidationStatus(status: unknown): ValidationResult["status"] | undefined {
if (status === "pass" || status === "fail" || status === "blocked") {
return status;
}
return undefined;
}
/**
* Extract assertion results from the parsed JSON.
*/
private extractAssertionResults(
parsed: Record<string, unknown>,
assertions: MissionContractAssertion[],
): Array<{ assertionId: string; passed: boolean; message?: string; expected?: string; actual?: string }> {
const results: Array<{
assertionId: string;
passed: boolean;
message?: string;
expected?: string;
actual?: string;
}> = [];
// If assertions array is provided in the response, use it
if (Array.isArray(parsed.assertions)) {
for (const item of parsed.assertions) {
if (typeof item === "object" && item !== null) {
const assertionItem = item as Record<string, unknown>;
const assertionId =
typeof assertionItem.assertionId === "string"
? assertionItem.assertionId
: typeof assertionItem.id === "string"
? assertionItem.id
: undefined;
const passed = typeof assertionItem.passed === "boolean" ? assertionItem.passed : false;
results.push({
assertionId: assertionId || "unknown",
passed,
message: typeof assertionItem.message === "string" ? assertionItem.message : undefined,
expected: typeof assertionItem.expected === "string" ? assertionItem.expected : undefined,
actual: typeof assertionItem.actual === "string" ? assertionItem.actual : undefined,
});
}
}
}
// If no assertion results but we have assertions, create default results based on status
if (results.length === 0 && assertions.length > 0) {
const overallPassed = parsed.status === "pass";
for (const assertion of assertions) {
results.push({
assertionId: assertion.id,
passed: overallPassed,
message: overallPassed ? "Passed" : "Failed",
});
}
}
return results;
}
/**
* Create an error validation result.
*/
private createErrorValidationResult(
errorMessage: string,
assertions: MissionContractAssertion[],
): ValidationResult {
return {
status: "error",
assertions: assertions.map((a) => ({
assertionId: a.id,
passed: false,
message: errorMessage,
})),
summary: errorMessage,
};
}
/**
* Build the validation prompt sent to the AI agent.
*/
private buildValidationPrompt(
feature: MissionFeature,
assertions: MissionContractAssertion[],
milestone?: Milestone,
): string {
const assertionTexts = assertions
.map((a, i) => `${i + 1}. **${a.title}**: ${a.assertion}`)
.join("\n");
const milestoneAcceptanceCriteria = milestone?.acceptanceCriteria?.trim();
const milestoneContext = milestoneAcceptanceCriteria
? `\nMilestone acceptance criteria (must also be satisfied for this feature to pass):\n${milestoneAcceptanceCriteria}\n`
: "";
return `Evaluate the implementation for feature "${feature.title}" against the following contract assertions:
${assertionTexts}${milestoneContext}
For each assertion:
- Determine if the implementation satisfies the assertion (pass/fail/blocked)
- If failed, explain what was expected vs what was actually observed
- If blocked, explain what external factor prevented validation
- Also verify that the implementation satisfies any milestone acceptance criteria provided above
Respond with a JSON object in this format:
{
"status": "pass|fail|blocked",
"assertions": [
{
"assertionId": "CA-...",
"passed": true|false,
"message": "Explanation if failed",
"expected": "What was expected",
"actual": "What was observed"
}
],
"summary": "Overall summary of validation",
"blockedReason": "Reason if status is blocked"
}
Be thorough and objective. If any assertion fails, the overall status should be "fail".`;
}
/**
* Build the system prompt for the validation agent.
*/
private buildValidationSystemPrompt(
_feature: MissionFeature,
_assertions: MissionContractAssertion[],
taskContext: string,
milestone?: Milestone,
): string {
const milestoneAcceptanceCriteria = milestone?.acceptanceCriteria?.trim();
return `You are a validation agent responsible for evaluating whether an implementation satisfies its contract assertions.
You will receive:
1. A feature description with its acceptance criteria
2. Contract assertions to evaluate against
3. Task context including the implementation details${milestoneAcceptanceCriteria ? `\n4. Milestone acceptance criteria text that also applies to this feature: ${milestoneAcceptanceCriteria}` : ""}
Your job is to:
1. Carefully review the implementation as described in the task context
2. Evaluate each contract assertion objectively
3. Determine if the implementation fully satisfies each assertion
4. Verify the implementation also satisfies any milestone acceptance criteria provided for the parent milestone
5. Return a structured JSON response with your findings
Be thorough and precise. A contract assertion represents a commitment made during planning - the implementation must fully satisfy it or it is considered failed.
Evaluation guidance:
- "pass" means all required assertions are fully satisfied.
- "fail" means one or more assertions are unmet or only partially satisfied.
- "blocked" means you cannot evaluate due to missing/insufficient evidence or external constraints.
- Partial satisfaction must be marked as failed with clear expected vs actual details.
- Milestone acceptance criteria are validator-executed requirements, not informational context.
Response format: Return ONLY a JSON object (no additional text) with this structure:
{
"status": "pass|fail|blocked",
"assertions": [
{
"assertionId": "The assertion ID",
"passed": true|false,
"message": "Explanation of your evaluation",
"expected": "What the assertion required",
"actual": "What you observed in the implementation"
}
],
"summary": "A concise summary of your overall evaluation",
"blockedReason": "If blocked, explain what external factor prevented validation"
}
${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
}
/**
* Build task context string for validation.
*/
private buildTaskContext(task: { id: string; title?: string; description?: string; log?: Array<{ action?: string }> }): string {
const lines: string[] = [];
lines.push(`Task: ${task.title || task.id}`);
if (task.description) {
lines.push(`Description: ${task.description}`);
}
if (task.log && task.log.length > 0) {
lines.push("\nRecent actions:");
const recentLogs = task.log.slice(-10);
for (const entry of recentLogs) {
if (entry.action) {
lines.push(` - ${entry.action}`);
}
}
}
return lines.join("\n");
}
private resolveFeatureMilestone(feature: MissionFeature): Milestone | undefined {
const slice = this.missionStore.getSlice(feature.sliceId);
if (!slice) {
return undefined;
}
return this.missionStore.getMilestone(slice.milestoneId);
}
private completeValidatorRunIfStillRunning(
runId: string | undefined,
status: "passed" | "failed" | "blocked" | "error",
summaryOrReason?: string,
): boolean {
if (!runId) {
return false;
}
if (typeof this.missionStore.getValidatorRun !== "function") {
this.missionStore.completeValidatorRun(runId, status, summaryOrReason);
return true;
}
const run = this.missionStore.getValidatorRun(runId);
if (!run || run.status !== "running") {
loopLog.warn(`Validator run ${runId} is no longer running; skipping ${status} completion.`);
return false;
}
this.missionStore.completeValidatorRun(runId, status, summaryOrReason);
return true;
}
/**
* Handle a successful validation (pass).
*/
private async handleValidationPass(
featureId: string,
runId: string | undefined,
summary: string,
): Promise<void> {
try {
this.completeValidatorRunIfStillRunning(runId, "passed", summary);
const feature = this.missionStore.getFeature(featureId);
if (feature && feature.status !== "done") {
this.missionStore.updateFeatureStatus(featureId, "done");
}
loopLog.log(`Feature ${featureId} passed validation`);
// Notify autopilot if configured
if (this.missionAutopilot?.notifyValidationComplete) {
await this.missionAutopilot.notifyValidationComplete(featureId, "passed");
}
this.emit("validation:passed", { featureId, runId, summary });
} catch (err) {
loopLog.error(`Error handling validation pass for ${featureId}:`, err);
}
}
/**
* Handle a failed validation.
*/
private async handleValidationFail(
featureId: string,
runId: string | undefined,
result: ValidationResult,
): Promise<void> {
try {
// Record the failures
const failures = result.assertions
.filter((a) => !a.passed)
.map((a) => ({
featureId,
assertionId: a.assertionId,
message: a.message || "Assertion failed",
expected: a.expected,
actual: a.actual,
}));
const canCompleteRun = runId
? typeof this.missionStore.getValidatorRun !== "function" || this.missionStore.getValidatorRun(runId)?.status === "running"
: false;
if (runId && failures.length > 0 && canCompleteRun) {
this.missionStore.recordValidatorFailures(runId, failures);
}
this.completeValidatorRunIfStillRunning(runId, "failed", result.summary);
loopLog.log(`Feature ${featureId} failed validation with ${failures.length} failures`);
// Create fix feature
try {
const fixFeature = this.missionStore.createGeneratedFixFeature(
featureId,
runId || "unknown",
failures.map((f) => f.assertionId),
);
loopLog.log(`Created fix feature ${fixFeature.id} for ${featureId}`);
// Auto-triage the fix feature so the retry loop can continue
try {
await this.missionStore.triageFeature(fixFeature.id);
loopLog.log(`Auto-triaged fix feature ${fixFeature.id}`);
} catch (triageErr) {
const triageMessage = triageErr instanceof Error ? triageErr.message : String(triageErr);
loopLog.error(`Error triaging fix feature ${fixFeature.id}:`, triageMessage);
// Continue even if triage fails - the fix feature was created and can be triaged manually
}
this.emit("validation:failed", {
featureId,
runId,
failures,
fixFeatureId: fixFeature.id,
});
} catch (fixErr) {
const message = fixErr instanceof Error ? fixErr.message : String(fixErr);
if (message.includes("retry budget exhausted")) {
loopLog.warn(`Feature ${featureId} retry budget exhausted; marking as blocked`);
// completeValidatorRun already handles the blocked transition when budget is exhausted
this.emit("validation:budget_exhausted", { featureId, runId });
} else {
loopLog.error(`Error creating fix feature for ${featureId}:`, message);
}
}
// Notify autopilot if configured
if (this.missionAutopilot?.notifyValidationComplete) {
await this.missionAutopilot.notifyValidationComplete(featureId, "failed");
}
} catch (err) {
loopLog.error(`Error handling validation fail for ${featureId}:`, err);
}
}
/**
* Handle a blocked validation.
*/
private async handleValidationBlocked(
featureId: string,
runId: string | undefined,
blockedReason: string | undefined,
): Promise<void> {
try {
this.completeValidatorRunIfStillRunning(runId, "blocked", blockedReason);
loopLog.log(`Feature ${featureId} blocked: ${blockedReason}`);
this.logFeatureErrorEvent(featureId, "validation_blocked", `Validation blocked for feature ${featureId}: ${blockedReason ?? "no reason provided"}`, {
runId,
blockedReason: blockedReason ?? null,
});
// Notify autopilot if configured
if (this.missionAutopilot?.notifyValidationComplete) {
await this.missionAutopilot.notifyValidationComplete(featureId, "blocked");
}
this.emit("validation:blocked", { featureId, runId, reason: blockedReason });
} catch (err) {
loopLog.error(`Error handling validation blocked for ${featureId}:`, err);
}
}
/**
* Handle a validation error (AI session failure, etc).
*/
private async handleValidationError(
featureId: string,
runId: string | undefined,
error: string,
): Promise<void> {
try {
this.completeValidatorRunIfStillRunning(runId, "error", error);
loopLog.error(`Feature ${featureId} validation error: ${error}`);
this.logFeatureErrorEvent(featureId, "validation_error", `Validation error for feature ${featureId}: ${error}`, {
runId,
error,
});
// Notify autopilot if configured
if (this.missionAutopilot?.notifyValidationComplete) {
await this.missionAutopilot.notifyValidationComplete(featureId, "error");
}
this.emit("validation:error", { featureId, runId, error });
} catch (err) {
loopLog.error(`Error handling validation error for ${featureId}:`, err);
}
}
private logFeatureWarningEvent(
featureId: string,
code: string,
description: string,
metadata: Record<string, unknown>,
): void {
this.logFeatureMissionEvent(featureId, "warning", code, description, metadata);
}
private logFeatureErrorEvent(
featureId: string,
code: string,
description: string,
metadata: Record<string, unknown>,
): void {
this.logFeatureMissionEvent(featureId, "error", code, description, metadata);
}
private logFeatureMissionEvent(
featureId: string,
eventType: "warning" | "error",
code: string,
description: string,
metadata: Record<string, unknown>,
): void {
const feature = this.missionStore.getFeature(featureId);
if (!feature) return;
const slice = this.missionStore.getSlice(feature.sliceId);
if (!slice) return;
const milestone = this.missionStore.getMilestone(slice.milestoneId);
if (!milestone) return;
try {
this.missionStore.logMissionEvent?.(milestone.missionId, eventType, description, {
code,
featureId,
sliceId: slice.id,
milestoneId: milestone.id,
...metadata,
});
} catch (err) {
loopLog.warn(`Failed to log mission ${eventType} event for feature ${featureId}:`, err);
}
}
}