From 975d8a0ed2f03f34943282c0ca92cee51ecc65e9 Mon Sep 17 00:00:00 2001 From: Fusion Agent Date: Mon, 24 Aug 2026 21:20:23 +0000 Subject: [PATCH] test(FN-WF): stop non-review gates consuming a scenario's scripted review verdicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two places in the pipeline-smoke mock treated any readonly, non-Plan-Review turn as a Code Review, so on a review-column workflow the Documentation & Delivery gate ate the verdict scripted for Code Review. S07 scripts `codeReviewModes: ["empty-revise"]` to exercise an unactionable Code Review rejection; the card instead died with `documentation-delivery: failed: REVISE` before Code Review ever ran. - `emitReview` classified everything that was not Plan Review as "code". - The executor script forces `emitReview(context, "code")` whenever a readonly session coincides with scripted `codeReviewModes` — and a documentation gate is readonly too, so it took that branch as well. Both now identify the executing step from the workflow-step system prompt ("You are a workflow step agent executing: ") and exclude the known non-review gates. Detection is by EXCLUSION rather than an allow-list on purpose: the real reviewer prompt does not carry the literal "Code Review", so allow-listing silently approves every genuine review instead — measured, it turned S07 green on `builtin:coding-ideas` for the wrong reason. S07 on builtin:coding-ideas-v2 now gets past that misattribution and reaches the review seal instead, which is a different and still-open problem, so the scenario stays on its original workflows. Lane is green and faster than the previous matrix: 6 files, 82 tests, 19/19 scenarios, 97.2s and 100.3s against the 150s budget. --- .../pipeline-smoke/_pipeline-mock-scripts.ts | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/engine/src/__tests__/pipeline-smoke/_pipeline-mock-scripts.ts b/packages/engine/src/__tests__/pipeline-smoke/_pipeline-mock-scripts.ts index 9649b04d8f..4f5f667eb3 100644 --- a/packages/engine/src/__tests__/pipeline-smoke/_pipeline-mock-scripts.ts +++ b/packages/engine/src/__tests__/pipeline-smoke/_pipeline-mock-scripts.ts @@ -143,11 +143,28 @@ export function installPipelineMockScripts(input: { input.observeMockRuntime(); const systemPrompt = context.options.systemPrompt ?? ""; const reviewKind = forcedKind ?? ( - /^Execute the workflow step "Plan Review"/i.test(context.prompt) + /workflow step agent executing:\s*(?:Documentation & Delivery|Verification)\b/i.test(systemPrompt) + ? "non-review" + : /^Execute the workflow step "Plan Review"/i.test(context.prompt) || /workflow step agent executing:\s*Plan Review\b/i.test(systemPrompt) ? "plan" : "code" ); + /* + FNXC:PipelineSmoke 2026-08-24-17:30: + Only a REAL review gate may consume a scenario's scripted verdicts. This classifier answered + "code" for anything that was not Plan Review, so on a review-column workflow the + Documentation & Delivery gate ate the verdict aimed at Code Review: S07 scripts + `codeReviewModes: ["empty-revise"]` to exercise an unactionable Code Review rejection, and the + card died with `documentation-delivery: failed: REVISE` before Code Review ever ran. + Detected on the workflow-step system prompt, which names the executing step, and by EXCLUSION + rather than an allow-list: the real reviewer prompt does not carry the literal "Code Review", so + allow-listing silently approves every genuine review instead. + */ + if (reviewKind === "non-review") { + context.options.onText?.(JSON.stringify({ verdict: "APPROVE", notes: "Mock gate completed a non-review workflow step.", findings: [] })); + return; + } const modes = reviewKind === "plan" ? (behavior.planReviewModes ?? behavior.reviewModes ?? ["approve"]) : (behavior.codeReviewModes ?? behavior.reviewModes ?? ["approve"]); @@ -172,7 +189,18 @@ export function installPipelineMockScripts(input: { return; } const hasTaskUpdateTool = context.tools.some((tool) => tool.name === "fn_task_update"); - if (!hasTaskUpdateTool && behavior.codeReviewModes) { + /* + FNXC:PipelineSmoke 2026-08-24-17:30: + ...but a NON-REVIEW gate also arrives readonly. Documentation & Delivery has no task-update + tool either, so this heuristic handed it the verdict scripted for Code Review: S07 scripts + `codeReviewModes: ["empty-revise"]` to exercise an unactionable Code Review rejection, and on a + review-column workflow the card instead died with `documentation-delivery: failed: REVISE` + before Code Review ever ran. Identify the executing step from the workflow-step system prompt + and let only a real review consume the scripted verdicts. + */ + const executingNonReviewGate = /workflow step agent executing:\s*(?:Documentation & Delivery|Verification)\b/i + .test(context.options.systemPrompt ?? ""); + if (!hasTaskUpdateTool && behavior.codeReviewModes && !executingNonReviewGate) { /* FNXC:PipelineSmoke 2026-08-23-20:23: Final Code Review can arrive on the executor runtime with a generic dispatch prompt. Its