FN-5891: respect validator model settings in mission validation
Use the configured validator lane when creating mission validation sessions. - resolve mission validation session models from test mode, assigned agent runtime, and validator/default task settings in precedence order - surface validation session creation failures as mission validation errors instead of triggering fix-feature retries - add regression coverage and documentation for mission validation model resolution and error handling Files changed: AGENTS.md | 1 + docs/missions.md | 4 +- docs/settings-reference.md | 6 +- .../src/__tests__/mission-execution-loop.test.ts | 252 ++++++++++++++++++++- packages/engine/src/mission-execution-loop.ts | 63 +++++- 5 files changed, 316 insertions(+), 10 deletions(-) Fusion-Task-Id: FN-5891 Fusion-Task-Lineage: 6254daee-fb6b-4ebb-9fb4-3636237404b1
This commit is contained in:
gsxdsm
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { TEST_MODE_RESOLVED } from "@fusion/core";
|
||||
import type {
|
||||
Mission,
|
||||
Milestone,
|
||||
@@ -44,6 +45,19 @@ vi.mock("../logger.js", () => ({
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("../agent-session-helpers.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../agent-session-helpers.js")>();
|
||||
return {
|
||||
...actual,
|
||||
createResolvedAgentSession: vi.fn(async () => ({
|
||||
session: mockSessionHolder.session as any,
|
||||
sessionFile: undefined,
|
||||
runtimeId: "test-runtime",
|
||||
wasConfigured: true,
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
// Helper to reset mock session state
|
||||
function resetMockSession() {
|
||||
mockSessionHolder.session.state.messages = [];
|
||||
@@ -51,6 +65,7 @@ function resetMockSession() {
|
||||
}
|
||||
|
||||
// Import AFTER vi.mock so the mock is applied
|
||||
import { createResolvedAgentSession } from "../agent-session-helpers.js";
|
||||
import { MissionExecutionLoop, loopLog } from "../mission-execution-loop.js";
|
||||
|
||||
// ── Mock Factories ──────────────────────────────────────────────────────────
|
||||
@@ -327,7 +342,19 @@ function createMockMissionStore() {
|
||||
}
|
||||
|
||||
function createMockTaskStore() {
|
||||
const tasks = new Map<string, { id: string; title?: string; description?: string; log?: Array<{ action?: string }>; column?: string; missionId?: string; sliceId?: string; status?: string }>();
|
||||
const tasks = new Map<string, {
|
||||
id: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
log?: Array<{ action?: string }>;
|
||||
column?: string;
|
||||
missionId?: string;
|
||||
sliceId?: string;
|
||||
status?: string;
|
||||
assignedAgentId?: string;
|
||||
validatorModelProvider?: string;
|
||||
validatorModelId?: string;
|
||||
}>();
|
||||
|
||||
const store = {
|
||||
getTask: vi.fn(async (id: string) => tasks.get(id)),
|
||||
@@ -346,7 +373,19 @@ function createMockTaskStore() {
|
||||
on: vi.fn(),
|
||||
off: vi.fn(),
|
||||
|
||||
_setTask: (t: { id: string; title?: string; description?: string; log?: Array<{ action?: string }>; column?: string; missionId?: string; sliceId?: string; status?: string }) => tasks.set(t.id, t),
|
||||
_setTask: (t: {
|
||||
id: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
log?: Array<{ action?: string }>;
|
||||
column?: string;
|
||||
missionId?: string;
|
||||
sliceId?: string;
|
||||
status?: string;
|
||||
assignedAgentId?: string;
|
||||
validatorModelProvider?: string;
|
||||
validatorModelId?: string;
|
||||
}) => tasks.set(t.id, t),
|
||||
_clear: () => tasks.clear(),
|
||||
};
|
||||
|
||||
@@ -391,11 +430,23 @@ describe("MissionExecutionLoop", () => {
|
||||
let loop: MissionExecutionLoop;
|
||||
let missionStore: ReturnType<typeof createMockMissionStore>;
|
||||
let taskStore: ReturnType<typeof createMockTaskStore>;
|
||||
let agentStore: { getAgent: ReturnType<typeof vi.fn> };
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
missionStore = createMockMissionStore();
|
||||
taskStore = createMockTaskStore();
|
||||
agentStore = {
|
||||
getAgent: vi.fn(),
|
||||
};
|
||||
|
||||
vi.mocked(createResolvedAgentSession).mockReset();
|
||||
vi.mocked(createResolvedAgentSession).mockResolvedValue({
|
||||
session: mockSessionHolder.session as any,
|
||||
sessionFile: undefined,
|
||||
runtimeId: "test-runtime",
|
||||
wasConfigured: true,
|
||||
});
|
||||
|
||||
const mission = createMockMission();
|
||||
missionStore._setMission(mission);
|
||||
@@ -715,6 +766,154 @@ describe("MissionExecutionLoop", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("threads configured validator/default model settings into mission validation sessions", async () => {
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-MODEL-SETTINGS", status: "in-progress" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue(makeAssertions(1));
|
||||
taskStore._setTask({
|
||||
id: "FN-MODEL-SETTINGS",
|
||||
title: "Validation model settings",
|
||||
description: "Implementation",
|
||||
log: [],
|
||||
validatorModelProvider: "task-validator",
|
||||
validatorModelId: "task-validator-model",
|
||||
});
|
||||
vi.mocked(taskStore.getSettings).mockResolvedValue({
|
||||
missionStaleThresholdMs: 600_000,
|
||||
missionMaxTaskRetries: 3,
|
||||
validatorProvider: "project-validator",
|
||||
validatorModelId: "project-validator-model",
|
||||
defaultProviderOverride: "project-default-override",
|
||||
defaultModelIdOverride: "project-default-override-model",
|
||||
defaultProvider: "project-default",
|
||||
defaultModelId: "project-default-model",
|
||||
fallbackProvider: "fallback-provider",
|
||||
fallbackModelId: "fallback-model",
|
||||
});
|
||||
mockSessionHolder.session.state.messages = [
|
||||
{ role: "assistant", content: JSON.stringify({ status: "pass", assertions: [{ assertionId: "CA-1", passed: true }], summary: "all good" }) },
|
||||
];
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-MODEL-SETTINGS");
|
||||
|
||||
expect(createResolvedAgentSession).toHaveBeenCalledWith(expect.objectContaining({
|
||||
sessionPurpose: "validation",
|
||||
defaultProvider: "task-validator",
|
||||
defaultModelId: "task-validator-model",
|
||||
fallbackProvider: "fallback-provider",
|
||||
fallbackModelId: "fallback-model",
|
||||
settings: expect.objectContaining({
|
||||
validatorProvider: "project-validator",
|
||||
validatorModelId: "project-validator-model",
|
||||
defaultProvider: "project-default",
|
||||
defaultModelId: "project-default-model",
|
||||
}),
|
||||
}));
|
||||
});
|
||||
|
||||
it("uses assigned agent runtime model ahead of task/settings for mission validation", async () => {
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-MODEL-AGENT", status: "in-progress" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue(makeAssertions(1));
|
||||
taskStore._setTask({
|
||||
id: "FN-MODEL-AGENT",
|
||||
title: "Validation model agent",
|
||||
description: "Implementation",
|
||||
log: [],
|
||||
assignedAgentId: "agent-1",
|
||||
validatorModelProvider: "task-validator",
|
||||
validatorModelId: "task-validator-model",
|
||||
});
|
||||
agentStore.getAgent.mockResolvedValue({
|
||||
id: "agent-1",
|
||||
runtimeConfig: { model: "agent-provider/agent-model" },
|
||||
});
|
||||
vi.mocked(taskStore.getSettings).mockResolvedValue({
|
||||
missionStaleThresholdMs: 600_000,
|
||||
missionMaxTaskRetries: 3,
|
||||
validatorProvider: "project-validator",
|
||||
validatorModelId: "project-validator-model",
|
||||
});
|
||||
mockSessionHolder.session.state.messages = [
|
||||
{ role: "assistant", content: JSON.stringify({ status: "pass", assertions: [{ assertionId: "CA-1", passed: true }], summary: "all good" }) },
|
||||
];
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
agentStore: agentStore as any,
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-MODEL-AGENT");
|
||||
|
||||
expect(createResolvedAgentSession).toHaveBeenCalledWith(expect.objectContaining({
|
||||
sessionPurpose: "validation",
|
||||
defaultProvider: "agent-provider",
|
||||
defaultModelId: "agent-model",
|
||||
}));
|
||||
});
|
||||
|
||||
it("forces mock/scripted validator lane when test mode is active", async () => {
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-MODEL-TESTMODE", status: "in-progress" });
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue(makeAssertions(1));
|
||||
taskStore._setTask({
|
||||
id: "FN-MODEL-TESTMODE",
|
||||
title: "Validation model test mode",
|
||||
description: "Implementation",
|
||||
log: [],
|
||||
assignedAgentId: "agent-1",
|
||||
validatorModelProvider: "task-validator",
|
||||
validatorModelId: "task-validator-model",
|
||||
});
|
||||
agentStore.getAgent.mockResolvedValue({
|
||||
id: "agent-1",
|
||||
runtimeConfig: { model: "agent-provider/agent-model" },
|
||||
});
|
||||
vi.mocked(taskStore.getSettings).mockResolvedValue({
|
||||
missionStaleThresholdMs: 600_000,
|
||||
missionMaxTaskRetries: 3,
|
||||
testMode: true,
|
||||
validatorProvider: "project-validator",
|
||||
validatorModelId: "project-validator-model",
|
||||
fallbackProvider: "fallback-provider",
|
||||
fallbackModelId: "fallback-model",
|
||||
});
|
||||
mockSessionHolder.session.state.messages = [
|
||||
{ role: "assistant", content: JSON.stringify({ status: "pass", assertions: [{ assertionId: "CA-1", passed: true }], summary: "all good" }) },
|
||||
];
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
agentStore: agentStore as any,
|
||||
});
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-MODEL-TESTMODE");
|
||||
|
||||
expect(createResolvedAgentSession).toHaveBeenCalledWith(expect.objectContaining({
|
||||
sessionPurpose: "validation",
|
||||
defaultProvider: TEST_MODE_RESOLVED.provider,
|
||||
defaultModelId: TEST_MODE_RESOLVED.modelId,
|
||||
fallbackProvider: "fallback-provider",
|
||||
fallbackModelId: "fallback-model",
|
||||
}));
|
||||
});
|
||||
|
||||
it("runs linked assertions and marks completion only when validation passes", async () => {
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-ASSERT-PASS", status: "in-progress" });
|
||||
missionStore._setFeature(feature);
|
||||
@@ -1329,6 +1528,55 @@ describe("MissionExecutionLoop", () => {
|
||||
// ── handleValidationError ───────────────────────────────────────────────
|
||||
|
||||
describe("handleValidationError", () => {
|
||||
it("surfaces validation session creation failures as mission errors", async () => {
|
||||
const assertions = makeAssertions(1);
|
||||
const feature = createMockFeature({
|
||||
loopState: "implementing",
|
||||
taskId: "FN-SESSION-ERROR",
|
||||
id: "F-001",
|
||||
});
|
||||
missionStore._setFeature(feature);
|
||||
missionStore.getFeatureByTaskId = vi.fn().mockReturnValue(feature);
|
||||
missionStore.listAssertionsForFeature = vi.fn().mockReturnValue(assertions);
|
||||
taskStore._setTask({ id: "FN-SESSION-ERROR", title: "Test", description: "Implementation", log: [] });
|
||||
vi.mocked(createResolvedAgentSession).mockRejectedValueOnce(new Error("401 insufficient credits"));
|
||||
|
||||
loop = new MissionExecutionLoop({
|
||||
taskStore: taskStore as any,
|
||||
missionStore: missionStore as any,
|
||||
rootDir: "/tmp",
|
||||
});
|
||||
const emitSpy = vi.spyOn(loop, "emit");
|
||||
loop.start();
|
||||
|
||||
await loop.processTaskOutcome("FN-SESSION-ERROR");
|
||||
|
||||
expect(missionStore.completeValidatorRun).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
"error",
|
||||
"Validation failed due to error: 401 insufficient credits",
|
||||
);
|
||||
expect(missionStore.createGeneratedFixFeature).not.toHaveBeenCalled();
|
||||
expect(emitSpy).toHaveBeenCalledWith(
|
||||
"validation:error",
|
||||
expect.objectContaining({
|
||||
featureId: "F-001",
|
||||
error: "Validation failed due to error: 401 insufficient credits",
|
||||
}),
|
||||
);
|
||||
expect(missionStore.logMissionEvent).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
"error",
|
||||
expect.stringContaining("Validation error"),
|
||||
expect.objectContaining({
|
||||
code: "validation_error",
|
||||
featureId: "F-001",
|
||||
error: "Validation failed due to error: 401 insufficient credits",
|
||||
}),
|
||||
);
|
||||
expectNoValidationBoardTaskMutation(taskStore);
|
||||
});
|
||||
|
||||
it("emits validation:error without mutating any board task", async () => {
|
||||
const assertions = makeAssertions(1);
|
||||
const feature = createMockFeature({
|
||||
|
||||
@@ -19,9 +19,19 @@ import type {
|
||||
MissionFeature,
|
||||
MissionValidatorRun,
|
||||
AgentStore,
|
||||
Settings,
|
||||
} from "@fusion/core";
|
||||
import {
|
||||
TEST_MODE_RESOLVED,
|
||||
isTestModeActive,
|
||||
resolveTaskValidatorModel,
|
||||
} from "@fusion/core";
|
||||
import { createFnAgent, promptWithFallback, type AgentResult } from "./pi.js";
|
||||
import { createResolvedAgentSession, extractRuntimeHint } from "./agent-session-helpers.js";
|
||||
import {
|
||||
createResolvedAgentSession,
|
||||
extractRuntimeHint,
|
||||
extractRuntimeModel,
|
||||
} from "./agent-session-helpers.js";
|
||||
import { createLogger } from "./logger.js";
|
||||
import { createFallbackModelObserver } from "./fallback-model-observer.js";
|
||||
import { createRunAuditor, generateSyntheticRunId } from "./run-audit.js";
|
||||
@@ -348,9 +358,16 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
// Get task context for validation
|
||||
const task = feature.taskId ? await this.taskStore.getTask(feature.taskId) : null;
|
||||
const taskContext = task ? this.buildTaskContext(task) : "";
|
||||
const validationRuntimeHint = task?.assignedAgentId && this.agentStore
|
||||
? extractRuntimeHint((await this.agentStore.getAgent(task.assignedAgentId).catch(() => null))?.runtimeConfig)
|
||||
: undefined;
|
||||
const assignedAgent = task?.assignedAgentId && this.agentStore
|
||||
? await this.agentStore.getAgent(task.assignedAgentId).catch(() => null)
|
||||
: null;
|
||||
const validationRuntimeHint = extractRuntimeHint(assignedAgent?.runtimeConfig);
|
||||
const settings = await this.taskStore.getSettings().catch(() => undefined);
|
||||
const validationSessionModel = this.resolveValidationSessionModel(
|
||||
task,
|
||||
settings,
|
||||
assignedAgent?.runtimeConfig,
|
||||
);
|
||||
|
||||
let session: AgentResult | null = null;
|
||||
|
||||
@@ -370,8 +387,13 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
cwd: this.rootDir,
|
||||
systemPrompt: this.buildValidationSystemPrompt(feature, assertions, taskContext),
|
||||
tools: "readonly",
|
||||
defaultProvider: validationSessionModel.provider,
|
||||
defaultModelId: validationSessionModel.modelId,
|
||||
fallbackProvider: settings?.fallbackProvider,
|
||||
fallbackModelId: settings?.fallbackModelId,
|
||||
defaultThinkingLevel: "medium",
|
||||
runAuditor,
|
||||
settings,
|
||||
onText: (_delta) => {
|
||||
// Could stream this to a log entry if needed
|
||||
},
|
||||
@@ -410,7 +432,7 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
|
||||
// Return an error result - the loop will handle it
|
||||
return {
|
||||
status: "fail",
|
||||
status: "error",
|
||||
assertions: assertions.map((a) => ({
|
||||
assertionId: a.id,
|
||||
passed: false,
|
||||
@@ -431,6 +453,37 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
private resolveValidationSessionModel(
|
||||
task: Awaited<ReturnType<TaskStore["getTask"]>> | null,
|
||||
settings: Partial<Settings> | undefined,
|
||||
assignedAgentRuntimeConfig?: Record<string, unknown>,
|
||||
): { provider: string | undefined; modelId: string | undefined } {
|
||||
if (isTestModeActive(settings)) {
|
||||
return {
|
||||
provider: TEST_MODE_RESOLVED.provider,
|
||||
modelId: TEST_MODE_RESOLVED.modelId,
|
||||
};
|
||||
}
|
||||
|
||||
const assignedRuntimeModel = extractRuntimeModel(assignedAgentRuntimeConfig);
|
||||
if (assignedRuntimeModel.provider && assignedRuntimeModel.modelId) {
|
||||
return assignedRuntimeModel;
|
||||
}
|
||||
|
||||
const resolvedTaskModel = resolveTaskValidatorModel(
|
||||
{
|
||||
validatorModelProvider: task?.validatorModelProvider,
|
||||
validatorModelId: task?.validatorModelId,
|
||||
},
|
||||
settings,
|
||||
);
|
||||
|
||||
return {
|
||||
provider: resolvedTaskModel.provider,
|
||||
modelId: resolvedTaskModel.modelId,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the actual validation session with the AI agent.
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user