feat(FN-3392): implement follow-up suggestion contract and policy system
Merges the evaluator follow-up suggestion system (FN-3392, Steps 1–5), which adds a normalized contract, provenance tracking, and lifecycle documentation for AI-generated follow-up tasks, alongside chat UI improvements including unread indicators in header and mobile nav, corrected message routing, Fusion-Task-Id: FN-3392
This commit is contained in:
Fusion
228
packages/engine/src/eval-followups.ts
Normal file
228
packages/engine/src/eval-followups.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
import {
|
||||
buildEvalFollowUpSuggestionId,
|
||||
normalizeEvalFollowUpText,
|
||||
type EvalFollowUpPolicyMode,
|
||||
type EvalFollowUpSuggestion,
|
||||
type EvalScoreBand,
|
||||
type FollowUpDraft,
|
||||
type TaskStore,
|
||||
} from "@fusion/core";
|
||||
|
||||
const OPEN_COLUMNS = new Set(["triage", "todo", "in-progress", "in-review"]);
|
||||
const GENERIC_TITLE_PATTERNS = [/^follow\s*-?up$/i, /^todo$/i, /^fix\s+issue$/i, /^improve\s+task$/i, /^investigate$/i];
|
||||
|
||||
export interface NormalizeEvalFollowUpsInput {
|
||||
parentTaskId: string;
|
||||
runId: string;
|
||||
overallBand: EvalScoreBand;
|
||||
drafts: FollowUpDraft[];
|
||||
store: TaskStore;
|
||||
policyMode: EvalFollowUpPolicyMode;
|
||||
}
|
||||
|
||||
export interface MaterializeEvalFollowUpsInput {
|
||||
parentTaskId: string;
|
||||
runId: string;
|
||||
policyMode: EvalFollowUpPolicyMode;
|
||||
overallScore: number;
|
||||
followUps: EvalFollowUpSuggestion[];
|
||||
store: TaskStore;
|
||||
}
|
||||
|
||||
function inferPriority(overallBand: EvalScoreBand): EvalFollowUpSuggestion["priority"] {
|
||||
if (overallBand === "failing") return "urgent";
|
||||
if (overallBand === "weak") return "high";
|
||||
if (overallBand === "acceptable") return "normal";
|
||||
return "low";
|
||||
}
|
||||
|
||||
function isGenericDraft(draft: FollowUpDraft): boolean {
|
||||
const title = draft.title?.trim() ?? "";
|
||||
const description = draft.description?.trim() ?? "";
|
||||
if (!title || !description) return true;
|
||||
if (description.length < 20) return true;
|
||||
return GENERIC_TITLE_PATTERNS.some((pattern) => pattern.test(title));
|
||||
}
|
||||
|
||||
function isSignalInsufficient(draft: FollowUpDraft): boolean {
|
||||
return !draft.reason?.trim() || !Array.isArray(draft.evidenceRefs) || draft.evidenceRefs.length === 0;
|
||||
}
|
||||
|
||||
function toBaseSuggestion(params: {
|
||||
parentTaskId: string;
|
||||
runId: string;
|
||||
draft: FollowUpDraft;
|
||||
overallBand: EvalScoreBand;
|
||||
policyMode: EvalFollowUpPolicyMode;
|
||||
}): EvalFollowUpSuggestion {
|
||||
const { parentTaskId, runId, draft, overallBand, policyMode } = params;
|
||||
const dedupeSeed = `${parentTaskId}:${draft.title}:${draft.description}`;
|
||||
const dedupeKey = normalizeEvalFollowUpText(dedupeSeed);
|
||||
return {
|
||||
suggestionId: buildEvalFollowUpSuggestionId(`${runId}:${dedupeSeed}`),
|
||||
dedupeKey,
|
||||
title: draft.title.trim(),
|
||||
description: draft.description.trim(),
|
||||
priority: inferPriority(overallBand),
|
||||
severity: overallBand,
|
||||
rationale: draft.reason?.trim() || "No rationale provided.",
|
||||
evidenceRefs: (draft.evidenceRefs ?? []).map((evidenceId) => ({ evidenceId, source: "other" })),
|
||||
recommendation: {
|
||||
shouldCreate: false,
|
||||
reason: "Pending follow-up policy evaluation",
|
||||
policyQualified: false,
|
||||
},
|
||||
state: "suggested",
|
||||
policyMode,
|
||||
metadata: {
|
||||
parentTaskId,
|
||||
runId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveEvalFollowUpPolicyMode(policy?: "off" | "suggest" | "create"): EvalFollowUpPolicyMode {
|
||||
if (policy === "create") return "auto_create_qualified";
|
||||
if (policy === "suggest") return "persist_only";
|
||||
return "persist_only";
|
||||
}
|
||||
|
||||
export async function normalizeEvalFollowUps(input: NormalizeEvalFollowUpsInput): Promise<EvalFollowUpSuggestion[]> {
|
||||
const { parentTaskId, runId, drafts, overallBand, store, policyMode } = input;
|
||||
const openTasks = (await store.listTasks({ slim: true, includeArchived: false })).filter((task) =>
|
||||
OPEN_COLUMNS.has(task.column)
|
||||
);
|
||||
const priorResults = store.getEvalStore().listTaskResults({ taskId: parentTaskId });
|
||||
const priorKeys = new Set(
|
||||
priorResults
|
||||
.flatMap((result) => result.followUps)
|
||||
.map((suggestion) => suggestion.dedupeKey)
|
||||
.filter((key): key is string => Boolean(key)),
|
||||
);
|
||||
|
||||
return drafts.map((draft) => {
|
||||
const suggestion = toBaseSuggestion({ parentTaskId, runId, draft, overallBand, policyMode });
|
||||
|
||||
if (isGenericDraft(draft)) {
|
||||
return {
|
||||
...suggestion,
|
||||
state: "suppressed",
|
||||
suppressedReason: "empty_or_generic",
|
||||
recommendation: {
|
||||
shouldCreate: false,
|
||||
reason: "Suppressed due to empty/generic title or weak description",
|
||||
policyQualified: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (isSignalInsufficient(draft)) {
|
||||
return {
|
||||
...suggestion,
|
||||
state: "suppressed",
|
||||
suppressedReason: "insufficient_signal",
|
||||
recommendation: {
|
||||
shouldCreate: false,
|
||||
reason: "Suppressed due to missing rationale/evidence",
|
||||
policyQualified: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const matchingOpenTask = openTasks.find((task) => {
|
||||
const title = normalizeEvalFollowUpText(task.title ?? task.description);
|
||||
return title.includes(suggestion.dedupeKey) || suggestion.dedupeKey.includes(title);
|
||||
});
|
||||
if (matchingOpenTask) {
|
||||
return {
|
||||
...suggestion,
|
||||
state: "suppressed",
|
||||
suppressedReason: "duplicate_open_task",
|
||||
matchedTaskId: matchingOpenTask.id,
|
||||
recommendation: {
|
||||
shouldCreate: false,
|
||||
reason: `Suppressed as duplicate of open task ${matchingOpenTask.id}`,
|
||||
policyQualified: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (priorKeys.has(suggestion.dedupeKey)) {
|
||||
return {
|
||||
...suggestion,
|
||||
state: "suppressed",
|
||||
suppressedReason: "duplicate_prior_suggestion",
|
||||
matchedSuggestionId: suggestion.dedupeKey,
|
||||
recommendation: {
|
||||
shouldCreate: false,
|
||||
reason: "Suppressed as duplicate from prior eval result",
|
||||
policyQualified: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const shouldCreate = policyMode === "create_all_non_duplicates"
|
||||
|| (policyMode === "auto_create_qualified" && (suggestion.priority === "high" || suggestion.priority === "urgent"));
|
||||
|
||||
return {
|
||||
...suggestion,
|
||||
recommendation: {
|
||||
shouldCreate,
|
||||
policyQualified: shouldCreate,
|
||||
reason: shouldCreate
|
||||
? "Qualified for creation by follow-up policy"
|
||||
: "Persisted for manual review",
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function materializeEvalFollowUps(input: MaterializeEvalFollowUpsInput): Promise<EvalFollowUpSuggestion[]> {
|
||||
const { parentTaskId, runId, policyMode, overallScore, followUps, store } = input;
|
||||
const created: EvalFollowUpSuggestion[] = [];
|
||||
|
||||
for (const followUp of followUps) {
|
||||
if (!followUp.recommendation.shouldCreate || followUp.state !== "suggested") {
|
||||
created.push(followUp);
|
||||
continue;
|
||||
}
|
||||
|
||||
const createdTask = await store.createTask({
|
||||
title: followUp.title,
|
||||
description: [
|
||||
`Follow-up generated from evaluation run ${runId} for ${parentTaskId}.`,
|
||||
"",
|
||||
`Problem summary: ${followUp.description}`,
|
||||
"Expected outcome: Investigate and resolve the issue identified by evaluation findings.",
|
||||
`Eval severity/score: ${followUp.severity} (${overallScore})`,
|
||||
`Rationale: ${followUp.rationale}`,
|
||||
`Evidence refs: ${followUp.evidenceRefs.map((ref) => ref.evidenceId).join(", ") || "none"}`,
|
||||
].join("\n"),
|
||||
column: "triage",
|
||||
priority: followUp.priority,
|
||||
source: {
|
||||
sourceType: "automation",
|
||||
sourceParentTaskId: parentTaskId,
|
||||
sourceMetadata: {
|
||||
type: "eval_follow_up",
|
||||
runId,
|
||||
suggestionId: followUp.suggestionId,
|
||||
policyMode,
|
||||
dedupeKey: followUp.dedupeKey,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
created.push({
|
||||
...followUp,
|
||||
state: "created",
|
||||
createdTaskId: createdTask.id,
|
||||
recommendation: {
|
||||
...followUp.recommendation,
|
||||
reason: `Created as ${createdTask.id} by follow-up policy`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return created;
|
||||
}
|
||||
Reference in New Issue
Block a user