FN-8541: expose actionable validator diagnostics
Surface normalized, secret-safe validator evidence and assertion verdicts across mission events and remediation work. - Normalize, redact, bound, and render per-assertion validation diagnostics in shared core APIs. - Attach verdict-driven diagnostics to failure events and generated fixes while suppressing duplicate triage noise. - Display validator evidence in Mission activity and document the operator workflow. - Add core and engine coverage for evidence, mixed blocked verdicts, and triage behavior. Files changed: .changeset/fn-8541-validator-diagnostics.md | 7 ++ docs/missions.md | 17 +++ .../mission-store.validation-diagnostics.test.ts | 65 ++++++++++ packages/core/src/async-mission-store.ts | 8 +- packages/core/src/index.gate.ts | 10 ++ packages/core/src/index.ts | 10 ++ packages/core/src/mission-store.ts | 10 +- packages/core/src/mission-types.ts | 136 +++++++++++++++++++++ .../dashboard/app/components/MissionManager.css | 21 ++++ .../dashboard/app/components/MissionManager.tsx | 56 +++++++++ packages/dashboard/app/components/mission-types.ts | 3 + .../src/__tests__/mission-execution-loop.test.ts | 44 +++++++ .../mission-validator-behavioral-posture.test.ts | 37 +++++- packages/engine/src/mission-execution-loop.ts | 117 +++++++++++++----- 14 files changed, 501 insertions(+), 40 deletions(-) Fusion-Task-Id: FN-8541 Fusion-Task-Lineage: b226fc34-f35c-4ce5-bcb5-e418529caaa7 Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
gsxdsm
7
.changeset/fn-8541-validator-diagnostics.md
Normal file
7
.changeset/fn-8541-validator-diagnostics.md
Normal file
@@ -0,0 +1,7 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
summary: Show failed mission assertions and safe validator evidence in remediation work.
|
||||
category: fix
|
||||
dev: Validation diagnostics are normalized consistently across SQLite and PostgreSQL mission stores.
|
||||
@@ -657,6 +657,23 @@ interface MissionAssertionFailureRecord {
|
||||
|
||||
**Full state snapshots:** `MissionFeatureLoopSnapshot` captures complete loop state including all validator runs and lineage chains for post-mortem analysis.
|
||||
|
||||
### Validation failure diagnostics
|
||||
|
||||
A `validation_failed` Mission activity event includes `metadata.validationDiagnostics`, the typed source of truth for failure reporting. It contains the validator `runId`, `sourceFeatureId`, overall outcome, next action, and ordered per-assertion verdicts with expected, observed, message, and evidence references. The visible event text is derived from this object—not an AI summary—so a failed event always names failed assertion IDs and labels any separately blocked assertion IDs as blocked (never as failed).
|
||||
|
||||
Evidence is secret-redacted before persistence. Each assertion retains at most 16 evidence entries and every message, expected, observed, and evidence text field is capped at 4,096 UTF-8 bytes. Bounded fields carry `truncated: true`, excess evidence is reported as `omittedEvidenceCount`, project paths become project-relative, and external or disposable absolute paths become `[external path omitted]`.
|
||||
|
||||
Generated fix features and their triaged tasks include the same **Validation cause** section with source feature, validator run, failed assertion IDs, bounded observations, and evidence. SQLite `MissionStore` and PostgreSQL `AsyncMissionStore` use the shared renderer, so a retry does not produce backend-specific causes or duplicate sections. A fix that is already linked to a canonical task is an idempotent race; otherwise Mission activity tells the operator to inspect and retry triage rather than exposing internal exception/loop-state prose.
|
||||
|
||||
The loop state is internal scheduling context, not an operator diagnosis. Its public meanings and actions are:
|
||||
|
||||
| Public state | Meaning | Operator action |
|
||||
|---|---|---|
|
||||
| validating | A validator run is evaluating the landed implementation. | Inspect the run only if it remains active beyond the stale-run window. |
|
||||
| needs_fix | A validator found a remediable assertion failure. | Review the event’s Validation diagnostics and triage the generated Fix feature/task. |
|
||||
| blocked | Validation could not obtain sufficient proof, or retry budget is exhausted. | Resolve the stated external constraint or root cause, then retry/triage the feature. |
|
||||
| implementing | A task is carrying out the feature or its generated remediation. | Follow the linked task; duplicate validator triggers with a canonical task are ignored. |
|
||||
|
||||
### Operator Troubleshooting
|
||||
|
||||
| Symptom | Diagnosis | Resolution |
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
normalizeValidationDiagnostics,
|
||||
renderValidationCause,
|
||||
renderValidationFailureDescription,
|
||||
VALIDATION_DIAGNOSTICS_MAX_EVIDENCE_PER_ASSERTION,
|
||||
VALIDATION_DIAGNOSTICS_MAX_TEXT_BYTES,
|
||||
} from "../mission-types.js";
|
||||
|
||||
describe("validation diagnostics normalization", () => {
|
||||
it("preserves order, limits evidence, and records omissions", () => {
|
||||
const result = normalizeValidationDiagnostics({
|
||||
runId: "VR-1", sourceFeatureId: "F-1", outcome: "fail",
|
||||
assertions: [{ assertionId: "CA-1", passed: false, evidence: Array.from({ length: 17 }, (_, index) => ({ text: `evidence-${index}` })) }],
|
||||
});
|
||||
expect(result.assertions[0].evidence).toHaveLength(VALIDATION_DIAGNOSTICS_MAX_EVIDENCE_PER_ASSERTION);
|
||||
expect(result.assertions[0].evidence[0].text).toBe("evidence-0");
|
||||
expect(result.assertions[0].omittedEvidenceCount).toBe(1);
|
||||
});
|
||||
|
||||
it("redacts before safely truncating multibyte evidence and fields", () => {
|
||||
const result = normalizeValidationDiagnostics({
|
||||
runId: "VR-1", sourceFeatureId: "F-1", outcome: "fail",
|
||||
projectRoot: "/repo",
|
||||
assertions: [{ assertionId: "CA-1", passed: false, message: `token=secret-value ${"😀".repeat(2000)}`, expected: "/repo/src/example.ts", actual: "/private/tmp/secret.txt", evidence: [{ text: "Authorization: Bearer sk-live-ABCDEFG1234567890abcdef" }] }],
|
||||
});
|
||||
const assertion = result.assertions[0];
|
||||
expect(assertion.message).toContain("[REDACTED]");
|
||||
expect(Buffer.byteLength(assertion.message!, "utf8")).toBeLessThanOrEqual(VALIDATION_DIAGNOSTICS_MAX_TEXT_BYTES);
|
||||
expect(assertion.expected).toBe("src/example.ts");
|
||||
expect(assertion.actual).toBe("[external path omitted]");
|
||||
expect(assertion.evidence[0].text).toContain("[REDACTED]");
|
||||
expect(assertion.message).toContain("… [truncated]");
|
||||
});
|
||||
|
||||
it("omits Windows absolute paths outside the project root", () => {
|
||||
const result = normalizeValidationDiagnostics({
|
||||
runId: "VR-1", sourceFeatureId: "F-1", outcome: "fail",
|
||||
projectRoot: "C:\\repo",
|
||||
assertions: [{ assertionId: "CA-1", passed: false, evidence: [{ text: "C:\\external\\secret.txt" }, { text: "C:\\repo\\src\\proof.test.ts" }] }],
|
||||
});
|
||||
expect(result.assertions[0].evidence.map((entry) => entry.text)).toEqual(["[external path omitted]", "src/proof.test.ts"]);
|
||||
});
|
||||
|
||||
it("renders outcome-consistent event and remediation prose", () => {
|
||||
const diagnostics = normalizeValidationDiagnostics({
|
||||
runId: "VR-1", sourceFeatureId: "F-1", outcome: "fail",
|
||||
assertions: [
|
||||
{ assertionId: "CA-pass", verdict: "pass", passed: true },
|
||||
{ assertionId: "CA-fail", verdict: "fail", passed: false, expected: "green", actual: "red" },
|
||||
{ assertionId: "CA-blocked", verdict: "blocked", passed: false, message: "Service unavailable" },
|
||||
],
|
||||
});
|
||||
const eventDescription = renderValidationFailureDescription(diagnostics);
|
||||
expect(eventDescription).toContain("CA-fail");
|
||||
expect(eventDescription).toContain("CA-blocked");
|
||||
expect(eventDescription).toContain("1 assertion failed");
|
||||
expect(eventDescription).toContain("1 assertion is blocked");
|
||||
expect(eventDescription).not.toContain("CA-pass");
|
||||
const remediationCause = renderValidationCause(diagnostics);
|
||||
expect(remediationCause).toContain("Validator run: VR-1");
|
||||
expect(remediationCause).toContain("Blocked assertions: CA-blocked");
|
||||
expect(remediationCause).toContain("### CA-blocked (blocked)");
|
||||
});
|
||||
});
|
||||
@@ -9,7 +9,7 @@ import { EventEmitter } from "node:events";
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import * as schema from "./postgres/schema/index.js";
|
||||
import type { AsyncDataLayer } from "./postgres/data-layer.js";
|
||||
import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType } from "./mission-types.js";
|
||||
import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType, renderValidationCause } from "./mission-types.js";
|
||||
import type {
|
||||
Mission,
|
||||
Milestone,
|
||||
@@ -39,6 +39,7 @@ import type {
|
||||
ContractAssertionCreateInput,
|
||||
ContractAssertionUpdateInput,
|
||||
FeatureLoopState,
|
||||
ValidationDiagnostics,
|
||||
} from "./mission-types.js";
|
||||
import type { Goal } from "./goal-types.js";
|
||||
import {
|
||||
@@ -1201,12 +1202,15 @@ export class AsyncMissionStore extends EventEmitter<MissionStoreEvents> {
|
||||
failedAssertionIds: string[],
|
||||
failureReason?: string,
|
||||
title?: string,
|
||||
diagnostics?: ValidationDiagnostics,
|
||||
): Promise<MissionFeature> {
|
||||
const run = await getValidatorRun(this.db, runId);
|
||||
if (!run) throw new Error(`Validator run ${runId} not found`);
|
||||
if (run.featureId !== sourceFeatureId) throw new Error(`Validator run ${runId} belongs to feature ${run.featureId}, expected ${sourceFeatureId}`);
|
||||
const now = new Date().toISOString();
|
||||
const reasonText = failureReason?.trim();
|
||||
// FNXC:MissionValidationDiagnostics 2026-07-23-12:00: PostgreSQL remediation uses the identical shared cause renderer as SQLite to prevent backend-specific operator diagnostics.
|
||||
const causeText = diagnostics ? renderValidationCause(diagnostics) : undefined;
|
||||
/*
|
||||
FNXC:MissionFixIdempotency 2026-07-14-18:45:
|
||||
Generated remediation is one source/run operation. Lock the source feature, re-check lineage/open fixes under that lock, and increment the retry counter in the same transaction so concurrent validator workers cannot create duplicates or consume two attempts.
|
||||
@@ -1244,7 +1248,7 @@ export class AsyncMissionStore extends EventEmitter<MissionStoreEvents> {
|
||||
id: this.generateId("F"),
|
||||
sliceId: source.sliceId,
|
||||
title: title ?? `Fix: ${source.title}`,
|
||||
description: reasonText ? `${source.description ? `${source.description}\n\n` : ""}## Verification failure detail\n${reasonText}` : source.description,
|
||||
description: [source.description, causeText ?? (reasonText ? `## Verification failure detail\n${reasonText}` : undefined)].filter(Boolean).join("\n\n") || undefined,
|
||||
acceptanceCriteria: source.acceptanceCriteria,
|
||||
status: "defined",
|
||||
createdAt: now,
|
||||
|
||||
@@ -1526,6 +1526,11 @@ export {
|
||||
SLICE_PLAN_STATES,
|
||||
FEATURE_LOOP_STATES,
|
||||
VALIDATOR_RUN_STATUSES,
|
||||
VALIDATION_DIAGNOSTICS_MAX_EVIDENCE_PER_ASSERTION,
|
||||
VALIDATION_DIAGNOSTICS_MAX_TEXT_BYTES,
|
||||
normalizeValidationDiagnostics,
|
||||
renderValidationFailureDescription,
|
||||
renderValidationCause,
|
||||
MISSION_ASSERTION_STATUSES,
|
||||
MISSION_ASSERTION_TYPES,
|
||||
DEFAULT_MISSION_ASSERTION_TYPE,
|
||||
@@ -1542,6 +1547,11 @@ export type {
|
||||
SlicePlanState,
|
||||
FeatureLoopState,
|
||||
ValidatorRunStatus,
|
||||
ValidationAssertionVerdict,
|
||||
ValidationEvidenceReference,
|
||||
ValidationAssertionDiagnostic,
|
||||
ValidationDiagnostics,
|
||||
ValidationDiagnosticsInput,
|
||||
MissionEventType,
|
||||
AutopilotStatus,
|
||||
Mission,
|
||||
|
||||
@@ -1665,6 +1665,11 @@ export {
|
||||
SLICE_PLAN_STATES,
|
||||
FEATURE_LOOP_STATES,
|
||||
VALIDATOR_RUN_STATUSES,
|
||||
VALIDATION_DIAGNOSTICS_MAX_EVIDENCE_PER_ASSERTION,
|
||||
VALIDATION_DIAGNOSTICS_MAX_TEXT_BYTES,
|
||||
normalizeValidationDiagnostics,
|
||||
renderValidationFailureDescription,
|
||||
renderValidationCause,
|
||||
MISSION_ASSERTION_STATUSES,
|
||||
MISSION_ASSERTION_TYPES,
|
||||
DEFAULT_MISSION_ASSERTION_TYPE,
|
||||
@@ -1681,6 +1686,11 @@ export type {
|
||||
SlicePlanState,
|
||||
FeatureLoopState,
|
||||
ValidatorRunStatus,
|
||||
ValidationAssertionVerdict,
|
||||
ValidationEvidenceReference,
|
||||
ValidationAssertionDiagnostic,
|
||||
ValidationDiagnostics,
|
||||
ValidationDiagnosticsInput,
|
||||
MissionEventType,
|
||||
AutopilotStatus,
|
||||
Mission,
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import type { Database } from "./db.js";
|
||||
import { fromJson, toJson, toJsonNullable } from "./db.js";
|
||||
import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType } from "./mission-types.js";
|
||||
import { FEATURE_LOOP_TRANSITIONS, normalizeMissionAssertionType, renderValidationCause } from "./mission-types.js";
|
||||
import type { Goal, GoalStatus } from "./goal-types.js";
|
||||
import type {
|
||||
Mission,
|
||||
@@ -51,6 +51,7 @@ import type {
|
||||
MilestoneValidationState,
|
||||
ValidatorRunStatus,
|
||||
FeatureLoopState,
|
||||
ValidationDiagnostics,
|
||||
} from "./mission-types.js";
|
||||
import { reconcileDeterministicDuplicate, runDeterministicDuplicateGuard } from "./duplicate-guard.js";
|
||||
import { resolveEntryPointBranchAssignment } from "./branch-assignment.js";
|
||||
@@ -2978,6 +2979,7 @@ export class MissionStore extends EventEmitter<MissionStoreEvents> {
|
||||
failedAssertionIds: string[],
|
||||
failureReason?: string,
|
||||
title?: string,
|
||||
diagnostics?: ValidationDiagnostics,
|
||||
): MissionFeature {
|
||||
const sourceFeature = this.getFeature(sourceFeatureId);
|
||||
if (!sourceFeature) {
|
||||
@@ -3039,9 +3041,9 @@ export class MissionStore extends EventEmitter<MissionStoreEvents> {
|
||||
|
||||
// R6 — surface the observed-vs-expected reason to the remediation agent.
|
||||
const reasonText = failureReason?.trim();
|
||||
const fixDescription = reasonText
|
||||
? `${sourceFeature.description ? `${sourceFeature.description}\n\n` : ""}## Verification failure detail\n${reasonText}`
|
||||
: sourceFeature.description;
|
||||
// FNXC:MissionValidationDiagnostics 2026-07-23-12:00: Generated remediation carries the same normalized cause as its event so operators and executors never need to reconstruct a validator failure.
|
||||
const causeText = diagnostics ? renderValidationCause(diagnostics) : undefined;
|
||||
const fixDescription = [sourceFeature.description, causeText ?? (reasonText ? `## Verification failure detail\n${reasonText}` : undefined)].filter(Boolean).join("\n\n") || undefined;
|
||||
|
||||
const fixFeature: MissionFeature = {
|
||||
id: fixFeatureId,
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
*/
|
||||
|
||||
import type { Goal } from "./goal-types.js";
|
||||
import { redactSecrets } from "./redact-secrets.js";
|
||||
|
||||
// ── Status Enums ─────────────────────────────────────────────────────
|
||||
|
||||
@@ -54,6 +55,141 @@ export const FEATURE_LOOP_TRANSITIONS: Readonly<Record<FeatureLoopState, readonl
|
||||
export const VALIDATOR_RUN_STATUSES = ["running", "passed", "failed", "blocked", "error"] as const;
|
||||
export type ValidatorRunStatus = (typeof VALIDATOR_RUN_STATUSES)[number];
|
||||
|
||||
/**
|
||||
* FNXC:MissionValidationDiagnostics 2026-07-23-12:00:
|
||||
* Validator failures cross engine, both stores, and dashboard activity. This
|
||||
* normalized contract is the only persisted diagnostic source so prose cannot
|
||||
* drift from the verdict and unbounded/secret-bearing evidence cannot escape.
|
||||
*/
|
||||
export const VALIDATION_DIAGNOSTICS_MAX_EVIDENCE_PER_ASSERTION = 16;
|
||||
export const VALIDATION_DIAGNOSTICS_MAX_TEXT_BYTES = 4096;
|
||||
|
||||
export type ValidationAssertionVerdict = "pass" | "fail" | "blocked";
|
||||
|
||||
export interface ValidationEvidenceReference {
|
||||
kind?: string;
|
||||
text?: string;
|
||||
/** True when text was bounded before persistence. */
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
export interface ValidationAssertionDiagnostic {
|
||||
assertionId: string;
|
||||
verdict: ValidationAssertionVerdict;
|
||||
message?: string;
|
||||
expected?: string;
|
||||
actual?: string;
|
||||
evidence: ValidationEvidenceReference[];
|
||||
omittedEvidenceCount?: number;
|
||||
}
|
||||
|
||||
export interface ValidationDiagnostics {
|
||||
runId: string;
|
||||
sourceFeatureId: string;
|
||||
outcome: "pass" | "fail" | "blocked" | "error" | "inconclusive";
|
||||
assertions: ValidationAssertionDiagnostic[];
|
||||
nextAction: string;
|
||||
}
|
||||
|
||||
export interface ValidationDiagnosticsInput {
|
||||
runId: string;
|
||||
sourceFeatureId: string;
|
||||
outcome: ValidationDiagnostics["outcome"];
|
||||
assertions: Array<{
|
||||
assertionId: string;
|
||||
verdict?: ValidationAssertionVerdict;
|
||||
passed?: boolean;
|
||||
message?: unknown;
|
||||
expected?: unknown;
|
||||
actual?: unknown;
|
||||
evidence?: Array<{ kind?: unknown; text?: unknown }>;
|
||||
}>;
|
||||
projectRoot?: string;
|
||||
}
|
||||
|
||||
function boundValidationText(value: unknown, projectRoot?: string): { value?: string; truncated?: boolean } {
|
||||
if (typeof value !== "string") return {};
|
||||
let text = redactSecrets(value);
|
||||
// Paths from the project are useful evidence; disposable/external paths are not.
|
||||
text = text.replace(/(?:[A-Za-z]:\\|\/)[^\s'"`]+/g, (path) => {
|
||||
const normalizedRoot = projectRoot?.replace(/\\/g, "/").replace(/\/+$/, "");
|
||||
const normalizedPath = path.replace(/\\/g, "/");
|
||||
return normalizedRoot && (normalizedPath === normalizedRoot || normalizedPath.startsWith(`${normalizedRoot}/`))
|
||||
? normalizedPath.slice(normalizedRoot.length).replace(/^\//, "") || "."
|
||||
: "[external path omitted]";
|
||||
});
|
||||
const bytes = Buffer.byteLength(text, "utf8");
|
||||
if (bytes <= VALIDATION_DIAGNOSTICS_MAX_TEXT_BYTES) return { value: text };
|
||||
const marker = "… [truncated]";
|
||||
const limit = VALIDATION_DIAGNOSTICS_MAX_TEXT_BYTES - Buffer.byteLength(marker, "utf8");
|
||||
let end = 0;
|
||||
let used = 0;
|
||||
for (const character of text) {
|
||||
const size = Buffer.byteLength(character, "utf8");
|
||||
if (used + size > limit) break;
|
||||
used += size;
|
||||
end += character.length;
|
||||
}
|
||||
return { value: `${text.slice(0, end)}${marker}`, truncated: true };
|
||||
}
|
||||
|
||||
/** Normalize and redact validation evidence before any mission artifact persists it. */
|
||||
export function normalizeValidationDiagnostics(input: ValidationDiagnosticsInput): ValidationDiagnostics {
|
||||
return {
|
||||
runId: input.runId,
|
||||
sourceFeatureId: input.sourceFeatureId,
|
||||
outcome: input.outcome,
|
||||
nextAction: input.outcome === "fail" ? "Review the failed assertions and triage the generated fix work." : "Review the validator run and retry or triage the feature when ready.",
|
||||
assertions: input.assertions.map((assertion) => {
|
||||
const evidence = (assertion.evidence ?? []).slice(0, VALIDATION_DIAGNOSTICS_MAX_EVIDENCE_PER_ASSERTION).map((item) => {
|
||||
const bounded = boundValidationText(item.text, input.projectRoot);
|
||||
return { ...(typeof item.kind === "string" ? { kind: item.kind } : {}), ...(bounded.value !== undefined ? { text: bounded.value } : {}), ...(bounded.truncated ? { truncated: true } : {}) };
|
||||
});
|
||||
const message = boundValidationText(assertion.message, input.projectRoot);
|
||||
const expected = boundValidationText(assertion.expected, input.projectRoot);
|
||||
const actual = boundValidationText(assertion.actual, input.projectRoot);
|
||||
return {
|
||||
assertionId: assertion.assertionId,
|
||||
verdict: assertion.verdict ?? (assertion.passed ? "pass" : "fail"),
|
||||
...(message.value !== undefined ? { message: message.value } : {}),
|
||||
...(expected.value !== undefined ? { expected: expected.value } : {}),
|
||||
...(actual.value !== undefined ? { actual: actual.value } : {}),
|
||||
evidence,
|
||||
...((assertion.evidence?.length ?? 0) > evidence.length ? { omittedEvidenceCount: (assertion.evidence?.length ?? 0) - evidence.length } : {}),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Render failure prose from normalized data only; never from non-authoritative judge summaries. */
|
||||
export function renderValidationFailureDescription(diagnostics: ValidationDiagnostics): string {
|
||||
const failed = diagnostics.assertions.filter((assertion) => assertion.verdict === "fail");
|
||||
const blocked = diagnostics.assertions.filter((assertion) => assertion.verdict === "blocked");
|
||||
const failedText = `${failed.length} assertion${failed.length === 1 ? "" : "s"} failed (${failed.map((assertion) => assertion.assertionId).join(", ") || "no assertion identity"})`;
|
||||
const blockedText = blocked.length > 0
|
||||
? `; ${blocked.length} assertion${blocked.length === 1 ? " is" : "s are"} blocked (${blocked.map((assertion) => assertion.assertionId).join(", ")})`
|
||||
: "";
|
||||
return `Validation failed for feature ${diagnostics.sourceFeatureId}: ${failedText}${blockedText}. ${diagnostics.nextAction}`;
|
||||
}
|
||||
|
||||
/** Stable remediation context used by both store implementations and their task descriptions. */
|
||||
export function renderValidationCause(diagnostics: ValidationDiagnostics): string {
|
||||
const nonPassing = diagnostics.assertions.filter((assertion) => assertion.verdict !== "pass");
|
||||
const failed = nonPassing.filter((assertion) => assertion.verdict === "fail");
|
||||
const blocked = nonPassing.filter((assertion) => assertion.verdict === "blocked");
|
||||
const lines = [
|
||||
"## Validation cause",
|
||||
`Source feature: ${diagnostics.sourceFeatureId}`,
|
||||
`Validator run: ${diagnostics.runId}`,
|
||||
`Failed assertions: ${failed.map((assertion) => assertion.assertionId).join(", ") || "none recorded"}`,
|
||||
...(blocked.length > 0 ? [`Blocked assertions: ${blocked.map((assertion) => assertion.assertionId).join(", ")}`] : []),
|
||||
];
|
||||
for (const assertion of nonPassing) {
|
||||
lines.push(`### ${assertion.assertionId} (${assertion.verdict})`, ...(assertion.expected ? [`Expected: ${assertion.expected}`] : []), ...(assertion.actual ? [`Observed: ${assertion.actual}`] : []), ...(assertion.message ? [`Details: ${assertion.message}`] : []), ...assertion.evidence.map((evidence) => `Evidence: ${evidence.text ?? evidence.kind ?? "recorded"}${evidence.truncated ? " (truncated)" : ""}`), ...(assertion.omittedEvidenceCount ? [`Additional evidence omitted: ${assertion.omittedEvidenceCount}`] : []));
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/** Interview state for AI-assisted specification */
|
||||
export const INTERVIEW_STATES = ["not_started", "in_progress", "completed", "needs_update"] as const;
|
||||
export type InterviewState = (typeof INTERVIEW_STATES)[number];
|
||||
|
||||
@@ -1461,6 +1461,27 @@ Narrow/mobile Missions puts Plan New Mission at the bottom of the list, using th
|
||||
color: var(--text-dim);
|
||||
}
|
||||
|
||||
.mission-event__diagnostics,
|
||||
.mission-event__diagnostic {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-xs);
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.mission-event__diagnostics {
|
||||
padding: var(--space-sm);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-sm);
|
||||
background: var(--surface);
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.mission-event__diagnostic + .mission-event__diagnostic {
|
||||
padding-top: var(--space-sm);
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.mission-event__metadata {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
@@ -55,6 +55,7 @@ import type {
|
||||
MilestoneValidationTelemetry,
|
||||
MissionFeatureLoopSnapshot,
|
||||
MissionValidatorRun,
|
||||
ValidationDiagnostics,
|
||||
} from "./mission-types";
|
||||
import {
|
||||
fetchMissions,
|
||||
@@ -545,6 +546,46 @@ function matchesEventFilter(
|
||||
}
|
||||
}
|
||||
|
||||
function getValidationDiagnostics(metadata: Record<string, unknown> | null): ValidationDiagnostics | undefined {
|
||||
const candidate = metadata?.validationDiagnostics;
|
||||
if (!candidate || typeof candidate !== "object") return undefined;
|
||||
const diagnostics = candidate as Partial<ValidationDiagnostics>;
|
||||
if (typeof diagnostics.runId !== "string" || typeof diagnostics.sourceFeatureId !== "string" || !Array.isArray(diagnostics.assertions)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// FNXC:MissionValidationDiagnostics 2026-07-23-13:15: Mission events are
|
||||
// durable and can predate this contract. Normalize partial JSON at the UI
|
||||
// boundary so malformed legacy metadata cannot crash activity on any viewport.
|
||||
return {
|
||||
runId: diagnostics.runId,
|
||||
sourceFeatureId: diagnostics.sourceFeatureId,
|
||||
outcome: diagnostics.outcome === "pass" || diagnostics.outcome === "fail" || diagnostics.outcome === "blocked" || diagnostics.outcome === "error" || diagnostics.outcome === "inconclusive" ? diagnostics.outcome : "error",
|
||||
nextAction: typeof diagnostics.nextAction === "string" ? diagnostics.nextAction : "Review the validator run before retrying or triaging the feature.",
|
||||
assertions: diagnostics.assertions.map((value, index) => {
|
||||
const assertion = value && typeof value === "object" ? value as Partial<ValidationDiagnostics["assertions"][number]> : {};
|
||||
const evidence = Array.isArray(assertion.evidence) ? assertion.evidence.flatMap((item) => {
|
||||
if (!item || typeof item !== "object") return [];
|
||||
const reference = item as { kind?: unknown; text?: unknown; truncated?: unknown };
|
||||
return [{
|
||||
...(typeof reference.kind === "string" ? { kind: reference.kind } : {}),
|
||||
...(typeof reference.text === "string" ? { text: reference.text } : {}),
|
||||
...(reference.truncated === true ? { truncated: true } : {}),
|
||||
}];
|
||||
}) : [];
|
||||
return {
|
||||
assertionId: typeof assertion.assertionId === "string" ? assertion.assertionId : `Assertion ${index + 1}`,
|
||||
verdict: assertion.verdict === "pass" || assertion.verdict === "fail" || assertion.verdict === "blocked" ? assertion.verdict : "blocked",
|
||||
...(typeof assertion.message === "string" ? { message: assertion.message } : {}),
|
||||
...(typeof assertion.expected === "string" ? { expected: assertion.expected } : {}),
|
||||
...(typeof assertion.actual === "string" ? { actual: assertion.actual } : {}),
|
||||
evidence,
|
||||
...(typeof assertion.omittedEvidenceCount === "number" && assertion.omittedEvidenceCount > 0 ? { omittedEvidenceCount: assertion.omittedEvidenceCount } : {}),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function getEventTypeClassName(eventType: MissionEventType): string {
|
||||
if (eventType === "error" || eventType === "warning") {
|
||||
return "mission-event__type--error";
|
||||
@@ -4260,6 +4301,21 @@ export function MissionManager({ isOpen, isInline = false, onClose, addToast, pr
|
||||
<span className="mission-event__timestamp">
|
||||
{new Date(event.timestamp).toLocaleString()}
|
||||
</span>
|
||||
{(() => {
|
||||
const diagnostics = getValidationDiagnostics(event.metadata);
|
||||
if (!diagnostics) return null;
|
||||
return <section className="mission-event__diagnostics" aria-label={t("missions.validationDiagnostics", "Validation diagnostics")}>
|
||||
<strong>{t("missions.validatorRun", "Validator run")}: {diagnostics.runId}</strong>
|
||||
<span>{t("missions.nextAction", "Next action")}: {diagnostics.nextAction}</span>
|
||||
{diagnostics.assertions.map((assertion) => <div className="mission-event__diagnostic" key={assertion.assertionId}>
|
||||
<strong>{assertion.assertionId}: {assertion.verdict}</strong>
|
||||
{assertion.expected && <span>{t("missions.expected", "Expected")}: {assertion.expected}</span>}
|
||||
{assertion.actual && <span>{t("missions.observed", "Observed")}: {assertion.actual}</span>}
|
||||
{assertion.evidence.map((evidence, index) => <span key={index}>{t("missions.evidence", "Evidence")}: {evidence.text ?? evidence.kind ?? t("missions.recorded", "recorded")}{evidence.truncated ? ` (${t("missions.truncated", "truncated")})` : ""}</span>)}
|
||||
{assertion.omittedEvidenceCount ? <span>{t("missions.evidenceOmitted", "Additional evidence omitted")}: {assertion.omittedEvidenceCount}</span> : null}
|
||||
</div>)}
|
||||
</section>;
|
||||
})()}
|
||||
{hasMetadata && (
|
||||
<div className="mission-event__metadata">
|
||||
<button
|
||||
|
||||
@@ -5,8 +5,11 @@ import type {
|
||||
MissionEvent as CoreMissionEvent,
|
||||
MissionEventType as CoreMissionEventType,
|
||||
MissionHealth as CoreMissionHealth,
|
||||
ValidationDiagnostics as CoreValidationDiagnostics,
|
||||
} from "@fusion/core";
|
||||
|
||||
export type ValidationDiagnostics = CoreValidationDiagnostics;
|
||||
|
||||
export type MissionStatus = "planning" | "active" | "blocked" | "complete" | "archived";
|
||||
export type MilestoneStatus = "planning" | "active" | "blocked" | "complete";
|
||||
export type SliceStatus = "pending" | "active" | "complete";
|
||||
|
||||
@@ -1694,6 +1694,50 @@ describe("MissionExecutionLoop", () => {
|
||||
expectNoValidationBoardTaskMutation(taskStore);
|
||||
});
|
||||
|
||||
it("preserves judge-provided evidence references for normalized diagnostics", async () => {
|
||||
const assertions = makeAssertions(1);
|
||||
mockSessionHolder.session.state.messages = [{
|
||||
role: "assistant",
|
||||
content: JSON.stringify({
|
||||
status: "fail",
|
||||
assertions: [{
|
||||
assertionId: "CA-1",
|
||||
passed: false,
|
||||
evidence: [{ kind: "test-output", text: "pnpm test: CA-1 failed" }],
|
||||
}],
|
||||
summary: "misleading summary",
|
||||
}),
|
||||
}];
|
||||
loop = new MissionExecutionLoop({ taskStore: taskStore as any, missionStore: missionStore as any, rootDir: "/tmp" });
|
||||
|
||||
await expect((loop as any).parseValidationResult(mockSessionHolder.session, assertions)).resolves.toMatchObject({
|
||||
assertions: [{ evidence: [{ kind: "test-output", text: "pnpm test: CA-1 failed" }] }],
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves blocked assertion verdicts in mixed failed results", async () => {
|
||||
const assertions = makeAssertions(2);
|
||||
mockSessionHolder.session.state.messages = [{
|
||||
role: "assistant",
|
||||
content: JSON.stringify({
|
||||
status: "fail",
|
||||
assertions: [
|
||||
{ assertionId: "CA-1", verdict: "fail", passed: false, message: "Observed failure" },
|
||||
{ assertionId: "CA-2", verdict: "blocked", passed: false, message: "Service unavailable" },
|
||||
],
|
||||
summary: "misleading fully satisfied summary",
|
||||
}),
|
||||
}];
|
||||
loop = new MissionExecutionLoop({ taskStore: taskStore as any, missionStore: missionStore as any, rootDir: "/tmp" });
|
||||
|
||||
await expect((loop as any).parseValidationResult(mockSessionHolder.session, assertions)).resolves.toMatchObject({
|
||||
assertions: [
|
||||
{ assertionId: "CA-1", verdict: "fail", passed: false },
|
||||
{ assertionId: "CA-2", verdict: "blocked", passed: false },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("should handle malformed JSON gracefully", async () => {
|
||||
const assertions = makeAssertions(1);
|
||||
// Malformed JSON with trailing comma
|
||||
|
||||
@@ -458,6 +458,12 @@ describe("Validator behavioral posture (U2 + U3)", () => {
|
||||
expect(call[2]).toEqual(["CA-1"]);
|
||||
expect(typeof call[3]).toBe("string");
|
||||
expect(call[3]).toContain("defect still reproduces");
|
||||
expect(call[5]).toMatchObject({
|
||||
assertions: [{
|
||||
assertionId: "CA-1",
|
||||
evidence: [{ kind: "behavioral-verification", text: expect.stringContaining("button still does nothing") }],
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
it("U6/R16: a verification FAILURE emits a persisted mission event with outcome=fail", async () => {
|
||||
@@ -529,10 +535,35 @@ describe("Validator behavioral posture (U2 + U3)", () => {
|
||||
await loop.processTaskOutcome("FN-TRIAGE");
|
||||
|
||||
const triageEvent = (missionStore.logMissionEvent as any).mock.calls.find(
|
||||
(c: any[]) => c[3]?.code === "fix_feature_triage_failed",
|
||||
(c: any[]) => c[3]?.code === "fix_feature_triage_needs_attention",
|
||||
);
|
||||
expect(triageEvent).toBeDefined();
|
||||
expect(triageEvent[1]).toBe("error");
|
||||
expect(triageEvent[3]?.error).toContain("triage boom");
|
||||
expect(triageEvent[1]).toBe("warning");
|
||||
expect(triageEvent[2]).not.toContain("triage boom");
|
||||
expect(triageEvent[3]).toMatchObject({ state: "needs-triage" });
|
||||
});
|
||||
|
||||
it("silently reuses an already-triaged fix with its canonical linked task", async () => {
|
||||
proveLandedInspection();
|
||||
const verify = vi.fn(async (req): Promise<VerificationOutcome> => ({ verdict: "fail", assertionId: req.assertionId, reason: "defect still reproduces" }));
|
||||
const feature = createMockFeature({ loopState: "implementing", taskId: "FN-DUPLICATE", status: "in-progress" });
|
||||
const existingFix = createMockFeature({
|
||||
id: "FIX-F-001", taskId: "TASK-FIX-F-001", status: "in-progress", loopState: "implementing",
|
||||
generatedFromFeatureId: "F-001", generatedFromRunId: "VR-existing",
|
||||
});
|
||||
missionStore._setFeature(feature);
|
||||
missionStore._setFeature(existingFix);
|
||||
missionStore._setAssertions("F-001", [assertionRow({ id: "CA-1", type: "behavioral" })]);
|
||||
taskStore._setTask(landedTask("FN-DUPLICATE", "duplicate trigger"));
|
||||
taskStore._setTask({ id: "TASK-FIX-F-001", title: "existing remediation" });
|
||||
missionStore.createGeneratedFixFeature = vi.fn(() => existingFix) as any;
|
||||
judgePass(["CA-1"]);
|
||||
|
||||
loop = new MissionExecutionLoop({ taskStore: taskStore as any, missionStore: missionStore as any, rootDir: "/tmp", verificationCapability: { verifyBehavioralAssertion: verify } });
|
||||
loop.start();
|
||||
await loop.processTaskOutcome("FN-DUPLICATE");
|
||||
|
||||
expect(missionStore.triageFeature).not.toHaveBeenCalled();
|
||||
expect((missionStore.logMissionEvent as any).mock.calls.some((call: any[]) => call[3]?.code === "fix_feature_triage_needs_attention")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -23,8 +23,9 @@ import type {
|
||||
Settings,
|
||||
Milestone,
|
||||
Mission,
|
||||
ValidationDiagnostics,
|
||||
} from "@fusion/core";
|
||||
import { normalizeMissionAssertionType } from "@fusion/core";
|
||||
import { normalizeMissionAssertionType, normalizeValidationDiagnostics, renderValidationFailureDescription } from "@fusion/core";
|
||||
import { GitCheckoutMaterializer, type CheckoutMaterializer, type VerificationOutcome } from "./mission-verification.js";
|
||||
import { createFnAgent, promptWithFallback, type AgentResult } from "./pi.js";
|
||||
import { mergeEffectiveSettings } from "./effective-settings.js";
|
||||
@@ -93,10 +94,13 @@ export interface ValidationResult {
|
||||
/** Per-assertion results */
|
||||
assertions: Array<{
|
||||
assertionId: string;
|
||||
/** Per-assertion outcome; `passed` remains for legacy judge responses. */
|
||||
verdict: "pass" | "fail" | "blocked";
|
||||
passed: boolean;
|
||||
message?: string;
|
||||
expected?: string;
|
||||
actual?: string;
|
||||
evidence?: Array<{ kind?: string; text?: string }>;
|
||||
}>;
|
||||
/** Summary message for overall result */
|
||||
summary: string;
|
||||
@@ -794,6 +798,7 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
status: "error",
|
||||
assertions: assertions.map((a) => ({
|
||||
assertionId: a.id,
|
||||
verdict: "fail",
|
||||
passed: false,
|
||||
message: `Validation error: ${message}`,
|
||||
})),
|
||||
@@ -871,7 +876,7 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
let inconclusiveReason: string | undefined;
|
||||
|
||||
const newAssertionResults = await Promise.all(
|
||||
judgeResult.assertions.map(async (judged) => {
|
||||
judgeResult.assertions.map(async (judged): Promise<ValidationResult["assertions"][number]> => {
|
||||
const type = typeById.get(judged.assertionId) ?? "static";
|
||||
if (type !== "behavioral") {
|
||||
// Static: keep judge verdict verbatim.
|
||||
@@ -882,6 +887,7 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
if (!this.verificationCapability) {
|
||||
return {
|
||||
...judged,
|
||||
verdict: "fail",
|
||||
passed: false,
|
||||
message: "Behavioral assertion defaults to fail: no verification evidence (advisory judge verdict is not authoritative).",
|
||||
expected: judged.expected ?? "Behavior confirmed by a verification run",
|
||||
@@ -904,27 +910,36 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
outcome = { verdict: "inconclusive", assertionId: judged.assertionId, reason: `verification error: ${message}` };
|
||||
}
|
||||
|
||||
// FNXC:MissionValidationDiagnostics 2026-07-23-12:30: Behavioral verification is an authoritative execution path, so its reason/detail must join judge evidence before the shared normalizer bounds and redacts it.
|
||||
const behavioralEvidence = [{
|
||||
kind: "behavioral-verification",
|
||||
text: outcome.detail ? `${outcome.reason}\n${outcome.detail}` : outcome.reason,
|
||||
}];
|
||||
if (outcome.verdict === "pass") {
|
||||
return { ...judged, passed: true, message: outcome.reason };
|
||||
return { ...judged, verdict: "pass", passed: true, message: outcome.reason, evidence: [...(judged.evidence ?? []), ...behavioralEvidence] };
|
||||
}
|
||||
if (outcome.verdict === "inconclusive") {
|
||||
sawInconclusive = true;
|
||||
inconclusiveReason = inconclusiveReason ?? outcome.reason;
|
||||
return {
|
||||
...judged,
|
||||
verdict: "blocked",
|
||||
passed: false,
|
||||
message: `Behavioral verification inconclusive: ${outcome.reason}`,
|
||||
expected: judged.expected ?? "Behavior confirmed by a verification run",
|
||||
actual: outcome.detail ?? "Verification could not conclude",
|
||||
evidence: [...(judged.evidence ?? []), ...behavioralEvidence],
|
||||
};
|
||||
}
|
||||
// fail
|
||||
return {
|
||||
...judged,
|
||||
verdict: "fail",
|
||||
passed: false,
|
||||
message: outcome.reason,
|
||||
expected: judged.expected ?? "Behavior confirmed by a verification run",
|
||||
actual: outcome.detail ?? judged.actual ?? "Behavior not confirmed",
|
||||
evidence: [...(judged.evidence ?? []), ...behavioralEvidence],
|
||||
};
|
||||
}),
|
||||
);
|
||||
@@ -1175,14 +1190,8 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
private extractAssertionResults(
|
||||
parsed: Record<string, unknown>,
|
||||
assertions: MissionContractAssertion[],
|
||||
): Array<{ assertionId: string; passed: boolean; message?: string; expected?: string; actual?: string }> {
|
||||
const results: Array<{
|
||||
assertionId: string;
|
||||
passed: boolean;
|
||||
message?: string;
|
||||
expected?: string;
|
||||
actual?: string;
|
||||
}> = [];
|
||||
): ValidationResult["assertions"] {
|
||||
const results: ValidationResult["assertions"] = [];
|
||||
|
||||
// If assertions array is provided in the response, use it
|
||||
if (Array.isArray(parsed.assertions)) {
|
||||
@@ -1196,14 +1205,31 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
? assertionItem.id
|
||||
: undefined;
|
||||
|
||||
const passed = typeof assertionItem.passed === "boolean" ? assertionItem.passed : false;
|
||||
// FNXC:MissionValidationDiagnostics 2026-07-23-13:15: A validation
|
||||
// run may fail while an individual assertion is blocked. Preserve that
|
||||
// identity instead of collapsing every non-pass into a failed assertion.
|
||||
const verdict = assertionItem.verdict === "pass" || assertionItem.verdict === "fail" || assertionItem.verdict === "blocked"
|
||||
? assertionItem.verdict
|
||||
: assertionItem.passed === true ? "pass" : "fail";
|
||||
const passed = verdict === "pass";
|
||||
|
||||
const evidence = Array.isArray(assertionItem.evidence)
|
||||
? assertionItem.evidence.flatMap((entry) => {
|
||||
if (typeof entry !== "object" || entry === null) return [];
|
||||
const candidate = entry as Record<string, unknown>;
|
||||
const kind = typeof candidate.kind === "string" ? candidate.kind : undefined;
|
||||
const text = typeof candidate.text === "string" ? candidate.text : undefined;
|
||||
return kind || text ? [{ ...(kind ? { kind } : {}), ...(text ? { text } : {}) }] : [];
|
||||
})
|
||||
: undefined;
|
||||
results.push({
|
||||
assertionId: assertionId || "unknown",
|
||||
verdict,
|
||||
passed,
|
||||
message: typeof assertionItem.message === "string" ? assertionItem.message : undefined,
|
||||
expected: typeof assertionItem.expected === "string" ? assertionItem.expected : undefined,
|
||||
actual: typeof assertionItem.actual === "string" ? assertionItem.actual : undefined,
|
||||
...(evidence ? { evidence } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1220,6 +1246,7 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
if (seen.has(assertion.id)) continue;
|
||||
results.push({
|
||||
assertionId: assertion.id,
|
||||
verdict: overallPassed ? "pass" : "fail",
|
||||
passed: overallPassed,
|
||||
message: overallPassed ? "Passed" : "Failed",
|
||||
});
|
||||
@@ -1240,6 +1267,7 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
status: "error",
|
||||
assertions: assertions.map((a) => ({
|
||||
assertionId: a.id,
|
||||
verdict: "fail",
|
||||
passed: false,
|
||||
message: errorMessage,
|
||||
})),
|
||||
@@ -1278,10 +1306,12 @@ Respond with a JSON object in this format:
|
||||
"assertions": [
|
||||
{
|
||||
"assertionId": "CA-...",
|
||||
"verdict": "pass|fail|blocked",
|
||||
"passed": true|false,
|
||||
"message": "Explanation if failed",
|
||||
"message": "Explanation for this verdict",
|
||||
"expected": "What was expected",
|
||||
"actual": "What was observed"
|
||||
"actual": "What was observed",
|
||||
"evidence": [{ "kind": "file|command|test-output|other", "text": "Concise file, command, or test-output reference used for this verdict" }]
|
||||
}
|
||||
],
|
||||
"summary": "Overall summary of validation",
|
||||
@@ -1323,6 +1353,7 @@ Evaluation guidance:
|
||||
- "blocked" means you cannot evaluate due to missing/insufficient evidence or external constraints.
|
||||
- Partial satisfaction must be marked as failed with clear expected vs actual details.
|
||||
- Milestone acceptance criteria are validator-executed requirements, not informational context.
|
||||
- For every assertion, include the concrete evidence you considered. Evidence must identify the relevant file, command, or concise test output; do not include secrets or full unbounded command output.
|
||||
|
||||
Response format: Return ONLY a JSON object (no additional text) with this structure:
|
||||
{
|
||||
@@ -1333,7 +1364,8 @@ Response format: Return ONLY a JSON object (no additional text) with this struct
|
||||
"passed": true|false,
|
||||
"message": "Explanation of your evaluation",
|
||||
"expected": "What the assertion required",
|
||||
"actual": "What you observed in the implementation"
|
||||
"actual": "What you observed in the implementation",
|
||||
"evidence": [{ "kind": "file|command|test-output|other", "text": "Concise file, command, or test-output reference used for this verdict" }]
|
||||
}
|
||||
],
|
||||
"summary": "A concise summary of your overall evaluation",
|
||||
@@ -1469,16 +1501,27 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
|
||||
|
||||
loopLog.log(`Feature ${featureId} failed validation with ${failures.length} failures`);
|
||||
|
||||
// R6 — build an observed-vs-expected reason so the remediation agent sees
|
||||
// what behavior was wrong, not just which assertion ids failed.
|
||||
const failureReason = this.buildFailureReason(failures, result.summary);
|
||||
|
||||
// R16 — durable observability: a verification/validation failure is a
|
||||
// persisted mission event, not just a log line.
|
||||
await this.logFeatureMissionEvent(featureId, "error", "validation_failed", `Validation failed for feature ${featureId}: ${result.summary}`, {
|
||||
runId: runId ?? null,
|
||||
// FNXC:MissionValidationDiagnostics 2026-07-23-12:00: The normalized verdict—not an LLM summary—drives every persisted failure surface.
|
||||
const diagnostics: ValidationDiagnostics = normalizeValidationDiagnostics({
|
||||
runId: runId ?? "unknown",
|
||||
sourceFeatureId: featureId,
|
||||
outcome: "fail",
|
||||
projectRoot: this.rootDir,
|
||||
assertions: result.assertions.map((assertion) => ({
|
||||
assertionId: assertion.assertionId,
|
||||
verdict: assertion.verdict,
|
||||
passed: assertion.passed,
|
||||
message: assertion.message,
|
||||
expected: assertion.expected,
|
||||
actual: assertion.actual,
|
||||
evidence: assertion.evidence,
|
||||
})),
|
||||
});
|
||||
const failureReason = this.buildFailureReason(failures, "");
|
||||
await this.logFeatureMissionEvent(featureId, "error", "validation_failed", renderValidationFailureDescription(diagnostics), {
|
||||
validationDiagnostics: diagnostics,
|
||||
runId: diagnostics.runId,
|
||||
failedAssertionIds: failures.map((f) => f.assertionId),
|
||||
reason: failureReason,
|
||||
outcome: "fail",
|
||||
});
|
||||
|
||||
@@ -1489,11 +1532,23 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
|
||||
runId || "unknown",
|
||||
failures.map((f) => f.assertionId),
|
||||
failureReason,
|
||||
undefined,
|
||||
diagnostics,
|
||||
);
|
||||
loopLog.log(`Created fix feature ${fixFeature.id} for ${featureId}`);
|
||||
|
||||
// Auto-triage the fix feature so the retry loop can continue
|
||||
try {
|
||||
// Auto-triage only a newly untriaged fix. createGeneratedFixFeature is
|
||||
// deliberately idempotent and can return an existing in-progress fix;
|
||||
// its durable task link is the canonical proof that triage already won.
|
||||
// FNXC:MissionValidationDiagnostics 2026-07-23-12:35: Duplicate validator triggers must silently reuse a fix feature with a linked board task instead of surfacing a false triage failure.
|
||||
const linkedFixTask = fixFeature.taskId ? await this.taskStore.getTask(fixFeature.taskId).catch(() => undefined) : undefined;
|
||||
// FNXC:MissionValidationDiagnostics 2026-07-23-13:15: A stale task ID
|
||||
// is not proof that remediation is live. Only an open, non-deleted task
|
||||
// makes duplicate triage safe to suppress; otherwise persist an action.
|
||||
const hasLiveFixTask = Boolean(linkedFixTask && !linkedFixTask.deletedAt && linkedFixTask.column !== "done" && linkedFixTask.column !== "archived" && linkedFixTask.status !== "failed");
|
||||
if (hasLiveFixTask) {
|
||||
loopLog.log(`Fix feature ${fixFeature.id} already has canonical task ${fixFeature.taskId}; skipping duplicate triage`);
|
||||
} else try {
|
||||
await this.missionStore.triageFeature(fixFeature.id);
|
||||
loopLog.log(`Auto-triaged fix feature ${fixFeature.id}`);
|
||||
} catch (triageErr) {
|
||||
@@ -1503,10 +1558,10 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
|
||||
// logged. The branch-group-collision learning: silent triage stalls
|
||||
// are invisible mission deadlocks. The Fix Feature was created and can
|
||||
// be triaged manually, so we continue, but the failure is persisted.
|
||||
await this.logFeatureMissionEvent(featureId, "error", "fix_feature_triage_failed", `Auto-triage of fix feature ${fixFeature.id} failed: ${triageMessage}`, {
|
||||
await this.logFeatureMissionEvent(featureId, "warning", "fix_feature_triage_needs_attention", `Fix feature ${fixFeature.id} was created but needs operator triage. Inspect the feature and retry triage.`, {
|
||||
runId: runId ?? null,
|
||||
fixFeatureId: fixFeature.id,
|
||||
error: triageMessage,
|
||||
state: "needs-triage",
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1529,9 +1584,9 @@ ${taskContext ? `\n\nImplementation context:\n${taskContext}` : ""}`;
|
||||
} else {
|
||||
loopLog.error(`Error creating fix feature for ${featureId}:`, message);
|
||||
// R16 — a swallowed Fix-Feature creation error is durably recorded.
|
||||
await this.logFeatureMissionEvent(featureId, "error", "fix_feature_creation_failed", `Failed to create fix feature for ${featureId}: ${message}`, {
|
||||
await this.logFeatureMissionEvent(featureId, "error", "fix_feature_creation_needs_attention", `Validation remediation could not be created for feature ${featureId}. Inspect the validator run and retry validation or triage.`, {
|
||||
runId: runId ?? null,
|
||||
error: message,
|
||||
state: "remediation-creation-failed",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user