feat(FN-3392): implement follow-up suggestion contract and policy system

Merges the evaluator follow-up suggestion system (FN-3392, Steps 1–5), which adds a normalized contract, provenance tracking, and lifecycle documentation for AI-generated follow-up tasks, alongside chat UI improvements including unread indicators in header and mobile nav, corrected message routing,

Fusion-Task-Id: FN-3392
This commit is contained in:
Fusion
2026-05-06 19:34:27 -07:00
committed by gsxdsm
parent c95517af31
commit 52c673a4cb
11 changed files with 594 additions and 15 deletions

View File

@@ -1,7 +1,12 @@
import { beforeEach, describe, expect, it } from "vitest";
import { createDatabase, type Database } from "../db.js";
import { EvalLifecycleError, EvalStore } from "../eval-store.js";
import { EVIDENCE_EXCERPT_TRUNCATION_MARKER, EVIDENCE_LIMITS, TASK_EVALUATION_EVIDENCE_SOURCE_ORDER } from "../eval-types.js";
import {
EVIDENCE_EXCERPT_TRUNCATION_MARKER,
EVIDENCE_LIMITS,
TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
buildEvalFollowUpSuggestionId,
} from "../eval-types.js";
let db: Database;
let store: EvalStore;
@@ -72,6 +77,19 @@ describe("EvalStore", () => {
}],
evidence: [{ type: "task_log", ref: "log:1" }],
deterministicSignals: [{ signalId: "s1", kind: "test", name: "tests-pass", passed: true }],
followUps: [{
suggestionId: buildEvalFollowUpSuggestionId("FN-123 missing tests"),
dedupeKey: "fn-123:missing-tests",
title: "Add regression tests for merged behavior",
description: "Investigate uncovered behavior and add targeted regression tests.",
priority: "high",
severity: "weak",
rationale: "Outcome quality signals showed verification gaps.",
evidenceRefs: [{ evidenceId: "workflow-1", source: "workflow", note: "verification failure" }],
recommendation: { shouldCreate: true, reason: "Actionable and high confidence", policyQualified: true },
state: "suggested",
policyMode: "persist_only",
}],
});
db.prepare("DELETE FROM tasks WHERE id = ?").run("FN-123");
@@ -84,6 +102,8 @@ describe("EvalStore", () => {
expect(fetched?.categoryScores[0]?.deterministicScore).toBe(78);
expect(fetched?.categoryScores[1]?.weight).toBe(0.45);
expect(fetched?.categoryScores[2]?.band).toBe("acceptable");
expect(fetched?.followUps[0]?.suggestionId).toMatch(/^efs-/);
expect(fetched?.followUps[0]?.recommendation.policyQualified).toBe(true);
});
it("persists run window boundaries and evaluated task rollups", () => {
@@ -241,6 +261,35 @@ describe("EvalStore", () => {
})).toThrow(/sourceOrder must match/);
});
it("persists suppression metadata for dedupe/noise control", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
const result = store.createTaskResult(run.id, {
taskId: "FN-suppressed",
taskSnapshot: { taskId: "FN-suppressed" },
status: "scored",
followUps: [{
suggestionId: "efs-suppress-1",
dedupeKey: "dedupe:1",
title: "Investigate flaky verification command",
description: "Identify root cause and stabilize verification.",
priority: "normal",
severity: "acceptable",
rationale: "Same recommendation already exists in open triage task.",
evidenceRefs: [{ evidenceId: "task-activity-2", source: "taskActivity" }],
recommendation: { shouldCreate: false, reason: "Duplicate of existing task", policyQualified: false },
state: "suppressed",
policyMode: "auto_create_qualified",
suppressedReason: "duplicate_open_task",
matchedTaskId: "FN-existing",
}],
});
const fetched = store.getTaskResult(result.id);
expect(fetched?.followUps[0]?.state).toBe("suppressed");
expect(fetched?.followUps[0]?.suppressedReason).toBe("duplicate_open_task");
expect(fetched?.followUps[0]?.matchedTaskId).toBe("FN-existing");
});
it("round-trips optional empty evidence source groups", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
const result = store.createTaskResult(run.id, {

View File

@@ -241,14 +241,78 @@ export interface EvalCategoryScore {
evidence: EvalEvidenceReference[];
}
export const EVAL_FOLLOW_UP_POLICY_MODES = [
"persist_only",
"auto_create_qualified",
"create_all_non_duplicates",
] as const;
export type EvalFollowUpPolicyMode = typeof EVAL_FOLLOW_UP_POLICY_MODES[number];
export const EVAL_FOLLOW_UP_SUGGESTION_STATES = ["suggested", "suppressed", "created"] as const;
export type EvalFollowUpSuggestionState = typeof EVAL_FOLLOW_UP_SUGGESTION_STATES[number];
export const EVAL_FOLLOW_UP_SUPPRESSION_REASONS = [
"duplicate_open_task",
"duplicate_prior_suggestion",
"insufficient_signal",
"empty_or_generic",
"policy_filtered",
] as const;
export type EvalFollowUpSuppressionReason = typeof EVAL_FOLLOW_UP_SUPPRESSION_REASONS[number];
export interface EvalFollowUpEvidenceReference {
evidenceId: string;
source: TaskEvaluationEvidenceSource | "category" | "signal" | "other";
note?: string;
}
export interface EvalFollowUpCreationRecommendation {
shouldCreate: boolean;
reason: string;
policyQualified: boolean;
}
export interface EvalFollowUpSuggestion {
suggestionId: string;
dedupeKey: string;
title: string;
description: string;
priority?: "low" | "normal" | "high" | "urgent";
priority: "low" | "normal" | "high" | "urgent";
severity: EvalScoreBand;
rationale: string;
evidenceRefs: EvalFollowUpEvidenceReference[];
recommendation: EvalFollowUpCreationRecommendation;
state: EvalFollowUpSuggestionState;
policyMode: EvalFollowUpPolicyMode;
suppressedReason?: EvalFollowUpSuppressionReason;
matchedTaskId?: string;
matchedSuggestionId?: string;
createdTaskId?: string;
tags?: string[];
metadata?: Record<string, unknown>;
}
export function normalizeEvalFollowUpText(value: string): string {
return value
.toLowerCase()
.replace(/\s+/g, " ")
.replace(/[^a-z0-9 ]/g, "")
.trim();
}
export function buildEvalFollowUpSuggestionId(seed: string): string {
const normalized = normalizeEvalFollowUpText(seed);
let hash = 2166136261;
for (let i = 0; i < normalized.length; i += 1) {
hash ^= normalized.charCodeAt(i);
hash = Math.imul(hash, 16777619);
}
return `efs-${(hash >>> 0).toString(16).padStart(8, "0")}`;
}
export interface EvalTaskResult {
id: string;
runId: string;

View File

@@ -743,6 +743,11 @@ export type {
RunAuditEvidence,
TaskEvaluationEvidenceBundle,
EvalSignal,
EvalFollowUpPolicyMode,
EvalFollowUpSuggestionState,
EvalFollowUpSuppressionReason,
EvalFollowUpEvidenceReference,
EvalFollowUpCreationRecommendation,
EvalFollowUpSuggestion,
EvalProvenance,
EvalStoreEvents,
@@ -758,10 +763,15 @@ export {
EVAL_SCORE_BANDS,
EVAL_SCORE_SCALE_MIN,
EVAL_SCORE_SCALE_MAX,
EVAL_FOLLOW_UP_POLICY_MODES,
EVAL_FOLLOW_UP_SUGGESTION_STATES,
EVAL_FOLLOW_UP_SUPPRESSION_REASONS,
TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
EVIDENCE_LIMITS,
MAX_EVIDENCE_EXCERPT_LENGTH,
EVIDENCE_EXCERPT_TRUNCATION_MARKER,
normalizeEvalFollowUpText,
buildEvalFollowUpSuggestionId,
} from "./eval-types.js";
export {
EVAL_CATEGORY_WEIGHTS,