diff --git a/.changeset/fn-8692-validator-assertion-ids.md b/.changeset/fn-8692-validator-assertion-ids.md new file mode 100644 index 0000000000..cfc4d26164 --- /dev/null +++ b/.changeset/fn-8692-validator-assertion-ids.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fix mission validators rejecting passing feature assertion results. +category: fix +dev: Validator prompts now provide assertion IDs and recover exact-count legacy responses safely. diff --git a/docs/missions.md b/docs/missions.md index 6dd598bd73..844c3bc43e 100644 --- a/docs/missions.md +++ b/docs/missions.md @@ -494,7 +494,7 @@ interface MilestoneValidationRollup { #### Completion Gate Contract -Canonical authored feature criteria live on `MissionFeature.acceptanceCriteria`, and each feature validator derives its verdict only from its **linked feature-scoped assertions**. Model summary prose, milestone prose, and behavioral results that are not mapped to a linked behavioral assertion cannot override that verdict. +Canonical authored feature criteria live on `MissionFeature.acceptanceCriteria`, and each feature validator derives its verdict only from its **linked feature-scoped assertions**. Validator prompts list each authoritative assertion ID in brackets; responses must return exactly one result keyed by each listed ID. To recover older model output safely, only an exact-count response with zero recognized IDs is matched positionally and recorded in diagnostics. Partial matches, duplicate IDs, and count mismatches remain fail-closed. Model summary prose, milestone prose, and behavioral results that are not mapped to a linked behavioral assertion cannot override that verdict. Milestone prose is synchronized to one canonical milestone-scoped assertion with `origin: "derived_milestone_acceptance"`. PostgreSQL restricts uniqueness to that derived origin per project/milestone; authored, imported, and migrated legacy milestone assertions stay independent, are never inferred from title/text, and require no feature links. The rollup evaluates all milestone-scoped assertions after feature coverage and feature assertion passes are ready; unmet parent criteria therefore block milestone completion without failing an already-passing feature. See [Mission Completion Gate Contract](./missions-completion-contract.md). diff --git a/packages/engine/src/__tests__/mission-execution-loop.test.ts b/packages/engine/src/__tests__/mission-execution-loop.test.ts index b7953d709e..cfc8bd5604 100644 --- a/packages/engine/src/__tests__/mission-execution-loop.test.ts +++ b/packages/engine/src/__tests__/mission-execution-loop.test.ts @@ -1146,13 +1146,20 @@ describe("MissionExecutionLoop", () => { rootDir: "/tmp", }); - const prompt = (loop as any).buildValidationPrompt(feature, assertions, milestone); - const systemPrompt = (loop as any).buildValidationSystemPrompt(feature, assertions, "Task context", milestone); + const prompt = (loop as any).buildValidationPrompt(feature, assertions, "feature"); + const milestonePrompt = (loop as any).buildValidationPrompt(feature, assertions, "milestone"); + const systemPrompt = (loop as any).buildValidationSystemPrompt(feature, assertions, "Task context", "feature"); expect(prompt).not.toContain("Milestone pass bar text"); expect(prompt).toContain("only the following linked feature contract assertions"); + expect(milestonePrompt).toContain("only the following milestone-scoped contract assertions"); + for (const assertion of assertions) { + expect(prompt).toContain(`[${assertion.id}]`); + expect(milestonePrompt).toContain(`[${assertion.id}]`); + } expect(systemPrompt).not.toContain("Milestone pass bar text"); expect(systemPrompt).toContain("linked feature contract assertions"); + expect(systemPrompt).toContain("The bracketed assertion ID shown for that assertion in the user message"); }); it("does NOT create a board task for single-feature validation", async () => { @@ -1591,6 +1598,137 @@ describe("MissionExecutionLoop", () => { }); }); + // ── Validation assertion identity contract ───────────────────────────────── + + describe("validation assertion identity contract", () => { + function createValidatorLoop() { + return new MissionExecutionLoop({ + taskStore: taskStore as any, + missionStore: missionStore as any, + rootDir: "/tmp", + }); + } + + it("keeps correctly keyed and legacy passed-only results canonical", () => { + const validatorLoop = createValidatorLoop(); + const assertions = makeAssertions(2); + const results = (validatorLoop as any).extractAssertionResults({ + assertions: [ + { assertionId: "CA-1", verdict: "pass", passed: true }, + { assertionId: "CA-2", passed: true }, + ], + }, assertions); + + expect(results).toMatchObject([ + { assertionId: "CA-1", verdict: "pass", passed: true }, + { assertionId: "CA-2", verdict: "pass", passed: true }, + ]); + expect((validatorLoop as any).deriveFeatureValidationStatus({ status: "fail", assertions: results }, false).status).toBe("pass"); + }); + + it("recovers the reported single-assertion pass when the validator did not echo its ID", () => { + const validatorLoop = createValidatorLoop(); + const assertions = makeAssertions(1); + const results = (validatorLoop as any).extractAssertionResults({ + status: "pass", + assertions: [{ assertionId: "CA-NOT-ECHOED", verdict: "pass", passed: true, message: "ok" }], + }, assertions); + + expect(results).toMatchObject([{ + assertionId: "CA-1", + verdict: "pass", + passed: true, + message: expect.stringContaining("matched positionally"), + }]); + expect((validatorLoop as any).deriveFeatureValidationStatus({ status: "fail", assertions: results }, false).status).toBe("pass"); + }); + + it("maps an exact-count zero-ID multi-assertion response positionally", () => { + const validatorLoop = createValidatorLoop(); + const assertions = makeAssertions(2); + const results = (validatorLoop as any).extractAssertionResults({ + assertions: [ + { assertionId: "unrecognized-first", verdict: "pass", passed: true }, + { passed: true }, + ], + }, assertions); + + expect(results).toMatchObject([ + { assertionId: "CA-1", verdict: "pass", passed: true, message: expect.stringContaining("matched positionally") }, + { assertionId: "CA-2", verdict: "pass", passed: true, message: expect.stringContaining("matched positionally") }, + ]); + expect((validatorLoop as any).deriveFeatureValidationStatus({ status: "fail", assertions: results }, false).status).toBe("pass"); + }); + + it("fails closed for an unrecognized mismatched count or partially recognized IDs", () => { + const validatorLoop = createValidatorLoop(); + const assertions = makeAssertions(2); + const wrongCount = (validatorLoop as any).extractAssertionResults({ + assertions: [{ assertionId: "CA-NOT-ECHOED", passed: true }], + }, assertions); + const partialMatch = (validatorLoop as any).extractAssertionResults({ + assertions: [ + { assertionId: "CA-1", passed: true }, + { assertionId: "CA-NOT-ECHOED", passed: true }, + ], + }, assertions); + + expect(wrongCount).toMatchObject([ + { assertionId: "CA-1", passed: false, message: "Validator result IDs did not match any linked assertion ID." }, + { assertionId: "CA-2", passed: false, message: "Validator result IDs did not match any linked assertion ID." }, + ]); + expect(partialMatch).toMatchObject([ + { assertionId: "CA-1", passed: true }, + { assertionId: "CA-2", passed: false, message: "Validator omitted linked assertion result." }, + ]); + }); + + it("fails closed when positional fallback receives duplicate unknown or empty IDs", () => { + const validatorLoop = createValidatorLoop(); + const assertions = makeAssertions(2); + const duplicateUnknown = (validatorLoop as any).extractAssertionResults({ + assertions: [ + { assertionId: "CA-NOT-ECHOED", passed: true }, + { assertionId: "CA-NOT-ECHOED", passed: true }, + ], + }, assertions); + const duplicateEmpty = (validatorLoop as any).extractAssertionResults({ + assertions: [ + { assertionId: "", passed: true }, + { assertionId: "", passed: true }, + ], + }, assertions); + + for (const results of [duplicateUnknown, duplicateEmpty]) { + expect(results).toMatchObject([ + { assertionId: "CA-1", passed: false, message: "Validator result IDs did not match any linked assertion ID." }, + { assertionId: "CA-2", passed: false, message: "Validator result IDs did not match any linked assertion ID." }, + ]); + } + }); + + it("preserves duplicate failure, empty/non-array failure, blocked aggregation, and zero assertions", () => { + const validatorLoop = createValidatorLoop(); + const assertions = makeAssertions(1); + const duplicate = (validatorLoop as any).extractAssertionResults({ + assertions: [{ assertionId: "CA-1", passed: true }, { assertionId: "CA-1", passed: true }], + }, assertions); + const empty = (validatorLoop as any).extractAssertionResults({ assertions: [] }, assertions); + const nonArray = (validatorLoop as any).extractAssertionResults({ assertions: "not-an-array" }, assertions); + const blocked = (validatorLoop as any).extractAssertionResults({ + assertions: [{ assertionId: "CA-1", verdict: "blocked", passed: false }], + }, assertions); + const zeroAssertions = (validatorLoop as any).extractAssertionResults({ assertions: [] }, []); + + expect(duplicate[0]).toMatchObject({ passed: false, message: "Duplicate validator result for linked assertion." }); + expect(empty[0]).toMatchObject({ passed: false, message: "Validator result IDs did not match any linked assertion ID." }); + expect(nonArray[0]).toMatchObject({ passed: false, message: "Validator result IDs did not match any linked assertion ID." }); + expect((validatorLoop as any).deriveFeatureValidationStatus({ status: "fail", assertions: blocked }, false).status).toBe("blocked"); + expect(zeroAssertions).toEqual([]); + expect((validatorLoop as any).deriveFeatureValidationStatus({ status: "pass", assertions: zeroAssertions }, false).status).toBe("fail"); + }); + }); + // ── parseValidationResult JSON extraction ───────────────────────────────── describe("parseValidationResult", () => { diff --git a/packages/engine/src/mission-execution-loop.ts b/packages/engine/src/mission-execution-loop.ts index 0e3d0851e2..4e2e5263fb 100644 --- a/packages/engine/src/mission-execution-loop.ts +++ b/packages/engine/src/mission-execution-loop.ts @@ -1276,8 +1276,11 @@ export class MissionExecutionLoop extends EventEmitter { assertions: MissionContractAssertion[], ): ValidationResult["assertions"] { const byId = new Map(); + const returnedResults: ValidationResult["assertions"] = []; + const returnedIds = new Set(); const authoritativeIds = new Set(assertions.map((assertion) => assertion.id)); const duplicateIds = new Set(); + let hasDuplicateReturnedId = false; // FNXC:MissionValidation 2026-07-23-14:00: // FN-8542 makes a contradictory aggregate structurally impossible. Only one @@ -1311,25 +1314,42 @@ export class MissionExecutionLoop extends EventEmitter { return kind || text ? [{ ...(kind ? { kind } : {}), ...(text ? { text } : {}) }] : []; }) : undefined; - if (!assertionId || !authoritativeIds.has(assertionId)) continue; - if (byId.has(assertionId)) { - duplicateIds.add(assertionId); - continue; - } - byId.set(assertionId, { - assertionId, + const normalizedResult: ValidationResult["assertions"][number] = { + assertionId: assertionId ?? "", verdict, passed, message: typeof assertionItem.message === "string" ? assertionItem.message : undefined, expected: typeof assertionItem.expected === "string" ? assertionItem.expected : undefined, actual: typeof assertionItem.actual === "string" ? assertionItem.actual : undefined, ...(evidence ? { evidence } : {}), - }); + }; + returnedResults.push(normalizedResult); + if (returnedIds.has(normalizedResult.assertionId)) { + hasDuplicateReturnedId = true; + } + returnedIds.add(normalizedResult.assertionId); + + if (!assertionId || !authoritativeIds.has(assertionId)) continue; + if (byId.has(assertionId)) { + duplicateIds.add(assertionId); + continue; + } + byId.set(assertionId, normalizedResult); } } } - return assertions.map((assertion) => { + /* + FNXC:MissionValidation 2026-08-01-16:15: + Recover only an exact-count response with no recognized or duplicate IDs. + A broader fallback could assign a genuinely failing, partial, or duplicate + response to the wrong assertion and let a feature pass without authoritative evidence. + */ + const usePositionalFallback = byId.size === 0 + && !hasDuplicateReturnedId + && returnedResults.length === assertions.length; + + return assertions.map((assertion, index) => { if (duplicateIds.has(assertion.id)) { return { assertionId: assertion.id, @@ -1338,11 +1358,25 @@ export class MissionExecutionLoop extends EventEmitter { message: "Duplicate validator result for linked assertion.", }; } - return byId.get(assertion.id) ?? { + const recognizedResult = byId.get(assertion.id); + if (recognizedResult) return recognizedResult; + if (usePositionalFallback) { + const positionalResult = returnedResults[index]; + return { + ...positionalResult, + assertionId: assertion.id, + message: positionalResult.message + ? `Validator result matched positionally; assertion IDs were not echoed. ${positionalResult.message}` + : "Validator result matched positionally; assertion IDs were not echoed.", + }; + } + return { assertionId: assertion.id, verdict: "fail" as const, passed: false, - message: "Validator omitted linked assertion result.", + message: byId.size === 0 + ? "Validator result IDs did not match any linked assertion ID." + : "Validator omitted linked assertion result.", }; }); } @@ -1394,8 +1428,14 @@ export class MissionExecutionLoop extends EventEmitter { assertions: MissionContractAssertion[], scope: "feature" | "milestone" = "feature", ): string { + /* + FNXC:MissionValidation 2026-08-01-15:38: + The prompt must carry the authoritative IDs that the parser keys on. Omitting + them made every validator response deterministically fail closed because the + model could not echo an ID it was never given. + */ const assertionTexts = assertions - .map((a, i) => `${i + 1}. **${a.title}**: ${a.assertion}`) + .map((a, i) => `${i + 1}. [${a.id}] **${a.title}**: ${a.assertion}`) .join("\n"); const subject = scope === "milestone" ? "milestone rollup" : `feature "${feature.title}"`; @@ -1404,6 +1444,8 @@ export class MissionExecutionLoop extends EventEmitter { ${assertionTexts} For each assertion: +- Return exactly one result for every listed assertion +- Set each result's assertionId to that assertion's bracketed ID exactly - Determine if the implementation satisfies the assertion (pass/fail/blocked) - If failed, explain what was expected vs what was actually observed - If blocked, explain what external factor prevented validation @@ -1413,7 +1455,7 @@ Respond with a JSON object in this format: "status": "pass|fail|blocked", "assertions": [ { - "assertionId": "CA-...", + "assertionId": "One bracketed assertion ID listed above", "verdict": "pass|fail|blocked", "passed": true|false, "message": "Explanation for this verdict", @@ -1466,7 +1508,7 @@ Response format: Return ONLY a JSON object (no additional text) with this struct "status": "pass|fail|blocked", "assertions": [ { - "assertionId": "The assertion ID", + "assertionId": "The bracketed assertion ID shown for that assertion in the user message", "passed": true|false, "message": "Explanation of your evaluation", "expected": "What the assertion required",