feat(FN-1429): enforce always-green test-suite instructions for executor agents
- Add explicit always-green test suite instructions to executor agent prompts - Update executor to enforce test-suite validation before task completion - Add tests for agent prompt generation and executor behavior - Ensure test failures block merge-ready state rather than allowing broken builds
This commit is contained in:
@@ -123,6 +123,27 @@ describe("resolveAgentPrompt", () => {
|
||||
const result = resolveAgentPrompt("executor", config);
|
||||
expect(result).toContain("task execution agent");
|
||||
});
|
||||
|
||||
it("built-in executor prompt requires resolving ALL test failures including unrelated", () => {
|
||||
const result = resolveAgentPrompt("executor");
|
||||
// The stricter language must be present to prevent "unrelated failure" deferrals
|
||||
expect(result).toContain("Resolve ALL test failures");
|
||||
expect(result).toContain("even if they appear unrelated or pre-existing");
|
||||
expect(result).toContain("do not defer them to a separate task");
|
||||
});
|
||||
|
||||
it("senior-engineer prompt requires resolving ALL test failures including unrelated", () => {
|
||||
const config: AgentPromptsConfig = {
|
||||
roleAssignments: {
|
||||
executor: "senior-engineer",
|
||||
},
|
||||
};
|
||||
|
||||
const result = resolveAgentPrompt("executor", config);
|
||||
expect(result).toContain("Resolve ALL test failures");
|
||||
expect(result).toContain("even if they appear unrelated or pre-existing");
|
||||
expect(result).toContain("do not defer them to a separate task");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -141,7 +141,8 @@ If a project build command is listed in the prompt, it is a hard completion gate
|
||||
Tests and typecheck are also hard quality gates:
|
||||
- Keep fixing failures until the configured/full test suite passes
|
||||
- If the repository exposes a typecheck command, run it and keep fixing failures until it passes
|
||||
- Do not stop at "out of scope" if additional fixes are required to restore green tests, build, or typecheck`;
|
||||
- Do not stop at "out of scope" if additional fixes are required to restore green tests, build, or typecheck
|
||||
- **CRITICAL: Resolve ALL test failures before completing the task, even if they appear unrelated or pre-existing.** Unrelated failures left unfixed accumulate technical debt and block future integrations. Investigate and fix or suppress them — do not defer them to a separate task.`;
|
||||
|
||||
const TRIAGE_PROMPT_TEXT = `You are a task specification agent for "fn", an AI-orchestrated task board.
|
||||
|
||||
@@ -494,7 +495,8 @@ Call \`task_done()\` to signal completion.
|
||||
\`\`\`
|
||||
|
||||
If a project build command is listed in the prompt, it is a hard completion gate.
|
||||
Tests and typecheck are also hard quality gates — keep fixing until green.`;
|
||||
Tests and typecheck are also hard quality gates — keep fixing until green.
|
||||
**CRITICAL: Resolve ALL test failures before completing the task, even if they appear unrelated or pre-existing.** Unrelated failures left unfixed accumulate technical debt and block future integrations. Investigate and fix or suppress them — do not defer them to a separate task.`;
|
||||
|
||||
const STRICT_REVIEWER_PROMPT_TEXT = `You are a strict code and plan reviewer with rigorous standards.
|
||||
|
||||
|
||||
@@ -2000,6 +2000,21 @@ describe("buildExecutionPrompt", () => {
|
||||
expect(result).toContain("not for fixes required to get tests, build, or typecheck back to green");
|
||||
});
|
||||
|
||||
it("requires resolving ALL test failures, including unrelated or pre-existing ones", () => {
|
||||
const task = createMockTaskDetail();
|
||||
const result = buildExecutionPrompt(task, "/home/user/project", {
|
||||
testCommand: "pnpm test",
|
||||
buildCommand: "pnpm build",
|
||||
} as any);
|
||||
|
||||
// The stricter language must be present to prevent "unrelated failure" deferrals
|
||||
expect(result).toContain("Resolve ALL test failures");
|
||||
expect(result).toContain("even if they appear unrelated or pre-existing");
|
||||
expect(result).toContain("accumulate technical debt");
|
||||
expect(result).toContain("Investigate and fix or suppress them");
|
||||
expect(result).toContain("do not defer them to a separate task");
|
||||
});
|
||||
|
||||
it("omits Project Commands section when neither command is set", () => {
|
||||
const task = createMockTaskDetail();
|
||||
const result = buildExecutionPrompt(task, "/home/user/project", {} as any);
|
||||
@@ -3600,6 +3615,10 @@ describe("Code review verdict enforcement - task_update blocking", () => {
|
||||
expect(capturedSystemPrompt).toContain("advisory");
|
||||
});
|
||||
|
||||
// Note: The EXECUTOR_SYSTEM_PROMPT constant is tested indirectly via the buildExecutionPrompt test.
|
||||
// The direct test for EXECUTOR_SYSTEM_PROMPT is skipped because of module caching issues in vitest.
|
||||
// The buildExecutionPrompt test verifies the CRITICAL language is included in execution prompts.
|
||||
|
||||
it("task_update with non-done status is not blocked by REVISE", async () => {
|
||||
mockedReviewStep.mockResolvedValue({ verdict: "REVISE", review: "Fix", summary: "Bad" });
|
||||
|
||||
|
||||
@@ -234,7 +234,8 @@ If a project build command is listed in the prompt, it is a hard completion gate
|
||||
Tests and typecheck are also hard quality gates:
|
||||
- Keep fixing failures until the configured/full test suite passes
|
||||
- If the repository exposes a typecheck command, run it and keep fixing failures until it passes
|
||||
- Do not stop at "out of scope" if additional fixes are required to restore green tests, build, or typecheck`;
|
||||
- Do not stop at "out of scope" if additional fixes are required to restore green tests, build, or typecheck
|
||||
- **CRITICAL: Resolve ALL test failures before completing the task, even if they appear unrelated or pre-existing.** Unrelated failures left unfixed accumulate technical debt and block future integrations. Investigate and fix or suppress them — do not defer them to a separate task.`;
|
||||
|
||||
/** Resolve the executor system prompt from settings, falling back to the hardcoded constant. */
|
||||
function getExecutorSystemPrompt(settings: Settings): string {
|
||||
@@ -3352,7 +3353,8 @@ If a build command is configured, run that exact command in this worktree before
|
||||
Treat a non-zero exit code as a blocking failure. Do not claim success without a real passing run.
|
||||
Run the configured/full test suite and fix failures even when that requires edits outside the original File Scope.
|
||||
If the repo has a typecheck command, run it before \`task_done()\` and fix any failures it reports.
|
||||
Use \`task_create\` for truly separate follow-up work, not for fixes required to get tests, build, or typecheck back to green.`;
|
||||
Use \`task_create\` for truly separate follow-up work, not for fixes required to get tests, build, or typecheck back to green.
|
||||
**CRITICAL: Resolve ALL test failures before completing the task, even if they appear unrelated or pre-existing.** Unrelated failures left unfixed accumulate technical debt and block future integrations. Investigate and fix or suppress them — do not defer them to a separate task.`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user