feat(FN-3392): implement follow-up suggestion contract and policy system
Merges the evaluator follow-up suggestion system (FN-3392, Steps 1–5), which adds a normalized contract, provenance tracking, and lifecycle documentation for AI-generated follow-up tasks, alongside chat UI improvements including unread indicators in header and mobile nav, corrected message routing, Fusion-Task-Id: FN-3392
This commit is contained in:
Fusion
118
packages/engine/src/__tests__/eval-followups.test.ts
Normal file
118
packages/engine/src/__tests__/eval-followups.test.ts
Normal file
@@ -0,0 +1,118 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { normalizeEvalFollowUpText } from "@fusion/core";
|
||||
import { materializeEvalFollowUps, normalizeEvalFollowUps, resolveEvalFollowUpPolicyMode } from "../eval-followups.js";
|
||||
|
||||
function makeStore(params: { openTasks?: Array<{ id: string; column: string; title?: string; description: string }>; priorDedupeKeys?: string[] }) {
|
||||
const openTasks = params.openTasks ?? [];
|
||||
const priorDedupeKeys = params.priorDedupeKeys ?? [];
|
||||
return {
|
||||
listTasks: async () => openTasks,
|
||||
createTask: vi.fn(async () => ({ id: "FN-created" })),
|
||||
getEvalStore: () => ({
|
||||
listTaskResults: () => [{ followUps: priorDedupeKeys.map((dedupeKey) => ({ dedupeKey })) }],
|
||||
}),
|
||||
} as any;
|
||||
}
|
||||
|
||||
describe("normalizeEvalFollowUps", () => {
|
||||
it("suppresses empty or generic suggestions", async () => {
|
||||
const followUps = await normalizeEvalFollowUps({
|
||||
parentTaskId: "FN-1",
|
||||
runId: "ER-1",
|
||||
overallBand: "weak",
|
||||
drafts: [{ title: "Follow-up", description: "too short", reason: "", evidenceRefs: [] }],
|
||||
store: makeStore({}),
|
||||
policyMode: "persist_only",
|
||||
});
|
||||
|
||||
expect(followUps[0]?.state).toBe("suppressed");
|
||||
expect(followUps[0]?.suppressedReason).toBe("empty_or_generic");
|
||||
});
|
||||
|
||||
it("suppresses duplicates of open tasks", async () => {
|
||||
const followUps = await normalizeEvalFollowUps({
|
||||
parentTaskId: "FN-1",
|
||||
runId: "ER-1",
|
||||
overallBand: "weak",
|
||||
drafts: [{ title: "Investigate flaky verification command", description: "Investigate flaky verification command causing reruns.", reason: "Failed verification", evidenceRefs: ["workflow-1"] }],
|
||||
store: makeStore({
|
||||
openTasks: [{ id: "FN-open", column: "todo", title: "Investigate flaky verification command", description: "x" }],
|
||||
}),
|
||||
policyMode: "persist_only",
|
||||
});
|
||||
|
||||
expect(followUps[0]?.state).toBe("suppressed");
|
||||
expect(followUps[0]?.suppressedReason).toBe("duplicate_open_task");
|
||||
expect(followUps[0]?.matchedTaskId).toBe("FN-open");
|
||||
});
|
||||
|
||||
it("suppresses duplicates from prior eval results", async () => {
|
||||
const priorKey = normalizeEvalFollowUpText("FN-1:Add regression test for merge flow:Add regression test for merge flow regressions.");
|
||||
const followUps = await normalizeEvalFollowUps({
|
||||
parentTaskId: "FN-1",
|
||||
runId: "ER-1",
|
||||
overallBand: "weak",
|
||||
drafts: [{ title: "Add regression test for merge flow", description: "Add regression test for merge flow regressions.", reason: "Missing tests", evidenceRefs: ["workflow-2"] }],
|
||||
store: makeStore({ priorDedupeKeys: [priorKey] }),
|
||||
policyMode: "persist_only",
|
||||
});
|
||||
|
||||
expect(followUps[0]?.state).toBe("suppressed");
|
||||
expect(followUps[0]?.suppressedReason).toBe("duplicate_prior_suggestion");
|
||||
});
|
||||
|
||||
it("marks qualified follow-ups for creation in create-all mode", async () => {
|
||||
const followUps = await normalizeEvalFollowUps({
|
||||
parentTaskId: "FN-1",
|
||||
runId: "ER-1",
|
||||
overallBand: "weak",
|
||||
drafts: [{ title: "Add flaky test diagnostics", description: "Add flaky test diagnostics for failing suite evidence.", reason: "Multiple failing runs", evidenceRefs: ["workflow-3"] }],
|
||||
store: makeStore({}),
|
||||
policyMode: "create_all_non_duplicates",
|
||||
});
|
||||
|
||||
expect(followUps[0]?.state).toBe("suggested");
|
||||
expect(followUps[0]?.recommendation.shouldCreate).toBe(true);
|
||||
expect(followUps[0]?.policyMode).toBe("create_all_non_duplicates");
|
||||
});
|
||||
|
||||
it("resolves project follow-up policy to backend policy mode", () => {
|
||||
expect(resolveEvalFollowUpPolicyMode("off")).toBe("persist_only");
|
||||
expect(resolveEvalFollowUpPolicyMode("suggest")).toBe("persist_only");
|
||||
expect(resolveEvalFollowUpPolicyMode("create")).toBe("auto_create_qualified");
|
||||
});
|
||||
|
||||
it("creates task and stamps provenance when suggestion is qualified", async () => {
|
||||
const store = makeStore({});
|
||||
const [created] = await materializeEvalFollowUps({
|
||||
parentTaskId: "FN-parent",
|
||||
runId: "ER-5",
|
||||
policyMode: "create_all_non_duplicates",
|
||||
overallScore: 42,
|
||||
store,
|
||||
followUps: [{
|
||||
suggestionId: "efs-1",
|
||||
dedupeKey: "k",
|
||||
title: "Investigate issue",
|
||||
description: "Investigate issue found by eval.",
|
||||
priority: "high",
|
||||
severity: "weak",
|
||||
rationale: "Signals showed repeated failures.",
|
||||
evidenceRefs: [{ evidenceId: "workflow-1", source: "other" }],
|
||||
recommendation: { shouldCreate: true, reason: "qualified", policyQualified: true },
|
||||
state: "suggested",
|
||||
policyMode: "create_all_non_duplicates",
|
||||
}],
|
||||
});
|
||||
|
||||
expect(store.createTask).toHaveBeenCalledTimes(1);
|
||||
expect(store.createTask.mock.calls[0][0].source.sourceParentTaskId).toBe("FN-parent");
|
||||
expect(store.createTask.mock.calls[0][0].source.sourceMetadata).toMatchObject({
|
||||
runId: "ER-5",
|
||||
suggestionId: "efs-1",
|
||||
policyMode: "create_all_non_duplicates",
|
||||
});
|
||||
expect(created?.state).toBe("created");
|
||||
expect(created?.createdTaskId).toBe("FN-created");
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { TASK_EVALUATION_EVIDENCE_SOURCE_ORDER, computeOverallScore, createDatabase, EvalStore, runScheduledEvalBatch, type TaskDetail, type TaskEvaluationEvidenceBundle } from "@fusion/core";
|
||||
import { HybridEvaluatorService, buildEvaluationPrompt, parseAiResponse, resolveEvaluatorModel } from "../evaluator.js";
|
||||
|
||||
@@ -38,7 +38,12 @@ function makeAiResponse(overrides: Partial<Record<string, unknown>> = {}): strin
|
||||
},
|
||||
},
|
||||
overallRationale: "Solid result with complete verification.",
|
||||
followUpDrafts: [],
|
||||
followUpDrafts: [{
|
||||
title: "Investigate flaky verification command",
|
||||
description: "Investigate flaky verification command failures seen in workflow output.",
|
||||
reason: "Verification command failed repeatedly",
|
||||
evidenceRefs: ["workflow-1"],
|
||||
}],
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
@@ -143,10 +148,15 @@ describe("evaluator", () => {
|
||||
});
|
||||
|
||||
it("returns merged evaluation payload shape for persistence", async () => {
|
||||
const createTask = vi.fn(async () => ({ id: "FN-900" }));
|
||||
const service = new HybridEvaluatorService({
|
||||
cwd: process.cwd(),
|
||||
runPrompt: async () => makeAiResponse(),
|
||||
store: {} as any,
|
||||
store: {
|
||||
listTasks: async () => [],
|
||||
createTask,
|
||||
getEvalStore: () => ({ listTaskResults: () => [] }),
|
||||
} as any,
|
||||
collectEvidence: async ({ task, runId }) => ({
|
||||
taskId: task.id,
|
||||
runId,
|
||||
@@ -161,7 +171,7 @@ describe("evaluator", () => {
|
||||
runAudit: [],
|
||||
}),
|
||||
});
|
||||
const result = await service.evaluateTask(makeTask(), { runId: "ER-1", startedAt: "2026-05-01T00:00:00.000Z" }, {});
|
||||
const result = await service.evaluateTask(makeTask(), { runId: "ER-1", startedAt: "2026-05-01T00:00:00.000Z" }, { taskEvaluationFollowUpPolicy: "create" });
|
||||
expect(result.status).toBe("scored");
|
||||
expect(result.overallScore).toBeGreaterThanOrEqual(0);
|
||||
expect(result.overallScore).toBeLessThanOrEqual(100);
|
||||
@@ -176,6 +186,42 @@ describe("evaluator", () => {
|
||||
expect(result.evidenceBundle?.sourceOrder).toEqual(TASK_EVALUATION_EVIDENCE_SOURCE_ORDER);
|
||||
expect(result.evidenceBundle?.agentLogs[0]?.excerpt).toBe("summary only");
|
||||
expect((result.evidenceBundle?.agentLogs[0] as any)?.detail).toBeUndefined();
|
||||
expect(result.followUps?.[0]?.suggestionId).toMatch(/^efs-/);
|
||||
expect(result.followUps?.[0]?.policyMode).toBe("auto_create_qualified");
|
||||
expect(result.followUps?.[0]?.state).toBe("created");
|
||||
expect(result.followUps?.[0]?.createdTaskId).toBe("FN-900");
|
||||
expect(createTask).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("persists-only in suggest mode without creating tasks", async () => {
|
||||
const createTask = vi.fn(async () => ({ id: "FN-created" }));
|
||||
const service = new HybridEvaluatorService({
|
||||
cwd: process.cwd(),
|
||||
runPrompt: async () => makeAiResponse(),
|
||||
store: {
|
||||
listTasks: async () => [],
|
||||
createTask,
|
||||
getEvalStore: () => ({ listTaskResults: () => [] }),
|
||||
} as any,
|
||||
collectEvidence: async ({ task, runId }) => ({
|
||||
taskId: task.id,
|
||||
runId,
|
||||
sourceOrder: TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
|
||||
taskMetadata: [],
|
||||
commits: [],
|
||||
workflow: [],
|
||||
reviews: [],
|
||||
documents: [],
|
||||
taskActivity: [],
|
||||
agentLogs: [],
|
||||
runAudit: [],
|
||||
}),
|
||||
});
|
||||
|
||||
const result = await service.evaluateTask(makeTask(), { runId: "ER-suggest", startedAt: "2026-05-01T00:00:00.000Z" }, { taskEvaluationFollowUpPolicy: "suggest" });
|
||||
expect(result.followUps?.[0]?.policyMode).toBe("persist_only");
|
||||
expect(result.followUps?.[0]?.state).toBe("suggested");
|
||||
expect(createTask).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("integrates scheduled batch with evaluator and persists one result per run/task", async () => {
|
||||
|
||||
228
packages/engine/src/eval-followups.ts
Normal file
228
packages/engine/src/eval-followups.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
import {
|
||||
buildEvalFollowUpSuggestionId,
|
||||
normalizeEvalFollowUpText,
|
||||
type EvalFollowUpPolicyMode,
|
||||
type EvalFollowUpSuggestion,
|
||||
type EvalScoreBand,
|
||||
type FollowUpDraft,
|
||||
type TaskStore,
|
||||
} from "@fusion/core";
|
||||
|
||||
const OPEN_COLUMNS = new Set(["triage", "todo", "in-progress", "in-review"]);
|
||||
const GENERIC_TITLE_PATTERNS = [/^follow\s*-?up$/i, /^todo$/i, /^fix\s+issue$/i, /^improve\s+task$/i, /^investigate$/i];
|
||||
|
||||
export interface NormalizeEvalFollowUpsInput {
|
||||
parentTaskId: string;
|
||||
runId: string;
|
||||
overallBand: EvalScoreBand;
|
||||
drafts: FollowUpDraft[];
|
||||
store: TaskStore;
|
||||
policyMode: EvalFollowUpPolicyMode;
|
||||
}
|
||||
|
||||
export interface MaterializeEvalFollowUpsInput {
|
||||
parentTaskId: string;
|
||||
runId: string;
|
||||
policyMode: EvalFollowUpPolicyMode;
|
||||
overallScore: number;
|
||||
followUps: EvalFollowUpSuggestion[];
|
||||
store: TaskStore;
|
||||
}
|
||||
|
||||
function inferPriority(overallBand: EvalScoreBand): EvalFollowUpSuggestion["priority"] {
|
||||
if (overallBand === "failing") return "urgent";
|
||||
if (overallBand === "weak") return "high";
|
||||
if (overallBand === "acceptable") return "normal";
|
||||
return "low";
|
||||
}
|
||||
|
||||
function isGenericDraft(draft: FollowUpDraft): boolean {
|
||||
const title = draft.title?.trim() ?? "";
|
||||
const description = draft.description?.trim() ?? "";
|
||||
if (!title || !description) return true;
|
||||
if (description.length < 20) return true;
|
||||
return GENERIC_TITLE_PATTERNS.some((pattern) => pattern.test(title));
|
||||
}
|
||||
|
||||
function isSignalInsufficient(draft: FollowUpDraft): boolean {
|
||||
return !draft.reason?.trim() || !Array.isArray(draft.evidenceRefs) || draft.evidenceRefs.length === 0;
|
||||
}
|
||||
|
||||
function toBaseSuggestion(params: {
|
||||
parentTaskId: string;
|
||||
runId: string;
|
||||
draft: FollowUpDraft;
|
||||
overallBand: EvalScoreBand;
|
||||
policyMode: EvalFollowUpPolicyMode;
|
||||
}): EvalFollowUpSuggestion {
|
||||
const { parentTaskId, runId, draft, overallBand, policyMode } = params;
|
||||
const dedupeSeed = `${parentTaskId}:${draft.title}:${draft.description}`;
|
||||
const dedupeKey = normalizeEvalFollowUpText(dedupeSeed);
|
||||
return {
|
||||
suggestionId: buildEvalFollowUpSuggestionId(`${runId}:${dedupeSeed}`),
|
||||
dedupeKey,
|
||||
title: draft.title.trim(),
|
||||
description: draft.description.trim(),
|
||||
priority: inferPriority(overallBand),
|
||||
severity: overallBand,
|
||||
rationale: draft.reason?.trim() || "No rationale provided.",
|
||||
evidenceRefs: (draft.evidenceRefs ?? []).map((evidenceId) => ({ evidenceId, source: "other" })),
|
||||
recommendation: {
|
||||
shouldCreate: false,
|
||||
reason: "Pending follow-up policy evaluation",
|
||||
policyQualified: false,
|
||||
},
|
||||
state: "suggested",
|
||||
policyMode,
|
||||
metadata: {
|
||||
parentTaskId,
|
||||
runId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveEvalFollowUpPolicyMode(policy?: "off" | "suggest" | "create"): EvalFollowUpPolicyMode {
|
||||
if (policy === "create") return "auto_create_qualified";
|
||||
if (policy === "suggest") return "persist_only";
|
||||
return "persist_only";
|
||||
}
|
||||
|
||||
export async function normalizeEvalFollowUps(input: NormalizeEvalFollowUpsInput): Promise<EvalFollowUpSuggestion[]> {
|
||||
const { parentTaskId, runId, drafts, overallBand, store, policyMode } = input;
|
||||
const openTasks = (await store.listTasks({ slim: true, includeArchived: false })).filter((task) =>
|
||||
OPEN_COLUMNS.has(task.column)
|
||||
);
|
||||
const priorResults = store.getEvalStore().listTaskResults({ taskId: parentTaskId });
|
||||
const priorKeys = new Set(
|
||||
priorResults
|
||||
.flatMap((result) => result.followUps)
|
||||
.map((suggestion) => suggestion.dedupeKey)
|
||||
.filter((key): key is string => Boolean(key)),
|
||||
);
|
||||
|
||||
return drafts.map((draft) => {
|
||||
const suggestion = toBaseSuggestion({ parentTaskId, runId, draft, overallBand, policyMode });
|
||||
|
||||
if (isGenericDraft(draft)) {
|
||||
return {
|
||||
...suggestion,
|
||||
state: "suppressed",
|
||||
suppressedReason: "empty_or_generic",
|
||||
recommendation: {
|
||||
shouldCreate: false,
|
||||
reason: "Suppressed due to empty/generic title or weak description",
|
||||
policyQualified: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (isSignalInsufficient(draft)) {
|
||||
return {
|
||||
...suggestion,
|
||||
state: "suppressed",
|
||||
suppressedReason: "insufficient_signal",
|
||||
recommendation: {
|
||||
shouldCreate: false,
|
||||
reason: "Suppressed due to missing rationale/evidence",
|
||||
policyQualified: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const matchingOpenTask = openTasks.find((task) => {
|
||||
const title = normalizeEvalFollowUpText(task.title ?? task.description);
|
||||
return title.includes(suggestion.dedupeKey) || suggestion.dedupeKey.includes(title);
|
||||
});
|
||||
if (matchingOpenTask) {
|
||||
return {
|
||||
...suggestion,
|
||||
state: "suppressed",
|
||||
suppressedReason: "duplicate_open_task",
|
||||
matchedTaskId: matchingOpenTask.id,
|
||||
recommendation: {
|
||||
shouldCreate: false,
|
||||
reason: `Suppressed as duplicate of open task ${matchingOpenTask.id}`,
|
||||
policyQualified: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (priorKeys.has(suggestion.dedupeKey)) {
|
||||
return {
|
||||
...suggestion,
|
||||
state: "suppressed",
|
||||
suppressedReason: "duplicate_prior_suggestion",
|
||||
matchedSuggestionId: suggestion.dedupeKey,
|
||||
recommendation: {
|
||||
shouldCreate: false,
|
||||
reason: "Suppressed as duplicate from prior eval result",
|
||||
policyQualified: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const shouldCreate = policyMode === "create_all_non_duplicates"
|
||||
|| (policyMode === "auto_create_qualified" && (suggestion.priority === "high" || suggestion.priority === "urgent"));
|
||||
|
||||
return {
|
||||
...suggestion,
|
||||
recommendation: {
|
||||
shouldCreate,
|
||||
policyQualified: shouldCreate,
|
||||
reason: shouldCreate
|
||||
? "Qualified for creation by follow-up policy"
|
||||
: "Persisted for manual review",
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function materializeEvalFollowUps(input: MaterializeEvalFollowUpsInput): Promise<EvalFollowUpSuggestion[]> {
|
||||
const { parentTaskId, runId, policyMode, overallScore, followUps, store } = input;
|
||||
const created: EvalFollowUpSuggestion[] = [];
|
||||
|
||||
for (const followUp of followUps) {
|
||||
if (!followUp.recommendation.shouldCreate || followUp.state !== "suggested") {
|
||||
created.push(followUp);
|
||||
continue;
|
||||
}
|
||||
|
||||
const createdTask = await store.createTask({
|
||||
title: followUp.title,
|
||||
description: [
|
||||
`Follow-up generated from evaluation run ${runId} for ${parentTaskId}.`,
|
||||
"",
|
||||
`Problem summary: ${followUp.description}`,
|
||||
"Expected outcome: Investigate and resolve the issue identified by evaluation findings.",
|
||||
`Eval severity/score: ${followUp.severity} (${overallScore})`,
|
||||
`Rationale: ${followUp.rationale}`,
|
||||
`Evidence refs: ${followUp.evidenceRefs.map((ref) => ref.evidenceId).join(", ") || "none"}`,
|
||||
].join("\n"),
|
||||
column: "triage",
|
||||
priority: followUp.priority,
|
||||
source: {
|
||||
sourceType: "automation",
|
||||
sourceParentTaskId: parentTaskId,
|
||||
sourceMetadata: {
|
||||
type: "eval_follow_up",
|
||||
runId,
|
||||
suggestionId: followUp.suggestionId,
|
||||
policyMode,
|
||||
dedupeKey: followUp.dedupeKey,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
created.push({
|
||||
...followUp,
|
||||
state: "created",
|
||||
createdTaskId: createdTask.id,
|
||||
recommendation: {
|
||||
...followUp.recommendation,
|
||||
reason: `Created as ${createdTask.id} by follow-up policy`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return created;
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
collectDeterministicSignals,
|
||||
computeOverallScore,
|
||||
normalizeCategoryScore,
|
||||
resolveScoreBand,
|
||||
resolveValidatorSettingsModel,
|
||||
EVAL_SCORE_CATEGORIES,
|
||||
type DeterministicSignals,
|
||||
@@ -15,6 +16,7 @@ import {
|
||||
type TaskStore,
|
||||
} from "@fusion/core";
|
||||
import { collectTaskEvaluationEvidence } from "./evaluator-evidence.js";
|
||||
import { materializeEvalFollowUps, normalizeEvalFollowUps, resolveEvalFollowUpPolicyMode } from "./eval-followups.js";
|
||||
import { createFnAgent, promptWithFallback } from "./pi.js";
|
||||
import { createLogger } from "./logger.js";
|
||||
|
||||
@@ -100,6 +102,24 @@ export class HybridEvaluatorService {
|
||||
});
|
||||
|
||||
const overallScore = computeOverallScore(categoryScores);
|
||||
const followUpPolicyMode = resolveEvalFollowUpPolicyMode(settings.taskEvaluationFollowUpPolicy);
|
||||
const followUps = this.deps.store
|
||||
? await materializeEvalFollowUps({
|
||||
parentTaskId: task.id,
|
||||
runId: run.runId,
|
||||
policyMode: followUpPolicyMode,
|
||||
overallScore,
|
||||
store: this.deps.store,
|
||||
followUps: await normalizeEvalFollowUps({
|
||||
parentTaskId: task.id,
|
||||
runId: run.runId,
|
||||
overallBand: resolveScoreBand(overallScore),
|
||||
drafts: ai.followUpDrafts,
|
||||
store: this.deps.store,
|
||||
policyMode: followUpPolicyMode,
|
||||
}),
|
||||
})
|
||||
: [];
|
||||
|
||||
return {
|
||||
status: "scored",
|
||||
@@ -110,11 +130,7 @@ export class HybridEvaluatorService {
|
||||
evidence: categoryScores.flatMap((categoryScore) => categoryScore.evidence),
|
||||
evidenceBundle,
|
||||
deterministicSignals: deterministicSignalsToEvalSignals(deterministicSignals),
|
||||
followUps: ai.followUpDrafts.map((draft) => ({
|
||||
title: draft.title,
|
||||
description: draft.description,
|
||||
metadata: { reason: draft.reason, evidenceRefs: draft.evidenceRefs },
|
||||
})),
|
||||
followUps,
|
||||
metadata: {
|
||||
runId: run.runId,
|
||||
evaluatorModel: model,
|
||||
|
||||
Reference in New Issue
Block a user