feat(FN-3392): implement follow-up suggestion contract and policy system
Merges the evaluator follow-up suggestion system (FN-3392, Steps 1–5), which adds a normalized contract, provenance tracking, and lifecycle documentation for AI-generated follow-up tasks, alongside chat UI improvements including unread indicators in header and mobile nav, corrected message routing, Fusion-Task-Id: FN-3392
This commit is contained in:
@@ -1,7 +1,12 @@
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { createDatabase, type Database } from "../db.js";
|
||||
import { EvalLifecycleError, EvalStore } from "../eval-store.js";
|
||||
import { EVIDENCE_EXCERPT_TRUNCATION_MARKER, EVIDENCE_LIMITS, TASK_EVALUATION_EVIDENCE_SOURCE_ORDER } from "../eval-types.js";
|
||||
import {
|
||||
EVIDENCE_EXCERPT_TRUNCATION_MARKER,
|
||||
EVIDENCE_LIMITS,
|
||||
TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
|
||||
buildEvalFollowUpSuggestionId,
|
||||
} from "../eval-types.js";
|
||||
|
||||
let db: Database;
|
||||
let store: EvalStore;
|
||||
@@ -72,6 +77,19 @@ describe("EvalStore", () => {
|
||||
}],
|
||||
evidence: [{ type: "task_log", ref: "log:1" }],
|
||||
deterministicSignals: [{ signalId: "s1", kind: "test", name: "tests-pass", passed: true }],
|
||||
followUps: [{
|
||||
suggestionId: buildEvalFollowUpSuggestionId("FN-123 missing tests"),
|
||||
dedupeKey: "fn-123:missing-tests",
|
||||
title: "Add regression tests for merged behavior",
|
||||
description: "Investigate uncovered behavior and add targeted regression tests.",
|
||||
priority: "high",
|
||||
severity: "weak",
|
||||
rationale: "Outcome quality signals showed verification gaps.",
|
||||
evidenceRefs: [{ evidenceId: "workflow-1", source: "workflow", note: "verification failure" }],
|
||||
recommendation: { shouldCreate: true, reason: "Actionable and high confidence", policyQualified: true },
|
||||
state: "suggested",
|
||||
policyMode: "persist_only",
|
||||
}],
|
||||
});
|
||||
|
||||
db.prepare("DELETE FROM tasks WHERE id = ?").run("FN-123");
|
||||
@@ -84,6 +102,8 @@ describe("EvalStore", () => {
|
||||
expect(fetched?.categoryScores[0]?.deterministicScore).toBe(78);
|
||||
expect(fetched?.categoryScores[1]?.weight).toBe(0.45);
|
||||
expect(fetched?.categoryScores[2]?.band).toBe("acceptable");
|
||||
expect(fetched?.followUps[0]?.suggestionId).toMatch(/^efs-/);
|
||||
expect(fetched?.followUps[0]?.recommendation.policyQualified).toBe(true);
|
||||
});
|
||||
|
||||
it("persists run window boundaries and evaluated task rollups", () => {
|
||||
@@ -241,6 +261,35 @@ describe("EvalStore", () => {
|
||||
})).toThrow(/sourceOrder must match/);
|
||||
});
|
||||
|
||||
it("persists suppression metadata for dedupe/noise control", () => {
|
||||
const run = store.createRun({ projectId: "p1", scope: "window" });
|
||||
const result = store.createTaskResult(run.id, {
|
||||
taskId: "FN-suppressed",
|
||||
taskSnapshot: { taskId: "FN-suppressed" },
|
||||
status: "scored",
|
||||
followUps: [{
|
||||
suggestionId: "efs-suppress-1",
|
||||
dedupeKey: "dedupe:1",
|
||||
title: "Investigate flaky verification command",
|
||||
description: "Identify root cause and stabilize verification.",
|
||||
priority: "normal",
|
||||
severity: "acceptable",
|
||||
rationale: "Same recommendation already exists in open triage task.",
|
||||
evidenceRefs: [{ evidenceId: "task-activity-2", source: "taskActivity" }],
|
||||
recommendation: { shouldCreate: false, reason: "Duplicate of existing task", policyQualified: false },
|
||||
state: "suppressed",
|
||||
policyMode: "auto_create_qualified",
|
||||
suppressedReason: "duplicate_open_task",
|
||||
matchedTaskId: "FN-existing",
|
||||
}],
|
||||
});
|
||||
|
||||
const fetched = store.getTaskResult(result.id);
|
||||
expect(fetched?.followUps[0]?.state).toBe("suppressed");
|
||||
expect(fetched?.followUps[0]?.suppressedReason).toBe("duplicate_open_task");
|
||||
expect(fetched?.followUps[0]?.matchedTaskId).toBe("FN-existing");
|
||||
});
|
||||
|
||||
it("round-trips optional empty evidence source groups", () => {
|
||||
const run = store.createRun({ projectId: "p1", scope: "window" });
|
||||
const result = store.createTaskResult(run.id, {
|
||||
|
||||
@@ -241,14 +241,78 @@ export interface EvalCategoryScore {
|
||||
evidence: EvalEvidenceReference[];
|
||||
}
|
||||
|
||||
export const EVAL_FOLLOW_UP_POLICY_MODES = [
|
||||
"persist_only",
|
||||
"auto_create_qualified",
|
||||
"create_all_non_duplicates",
|
||||
] as const;
|
||||
|
||||
export type EvalFollowUpPolicyMode = typeof EVAL_FOLLOW_UP_POLICY_MODES[number];
|
||||
|
||||
export const EVAL_FOLLOW_UP_SUGGESTION_STATES = ["suggested", "suppressed", "created"] as const;
|
||||
|
||||
export type EvalFollowUpSuggestionState = typeof EVAL_FOLLOW_UP_SUGGESTION_STATES[number];
|
||||
|
||||
export const EVAL_FOLLOW_UP_SUPPRESSION_REASONS = [
|
||||
"duplicate_open_task",
|
||||
"duplicate_prior_suggestion",
|
||||
"insufficient_signal",
|
||||
"empty_or_generic",
|
||||
"policy_filtered",
|
||||
] as const;
|
||||
|
||||
export type EvalFollowUpSuppressionReason = typeof EVAL_FOLLOW_UP_SUPPRESSION_REASONS[number];
|
||||
|
||||
export interface EvalFollowUpEvidenceReference {
|
||||
evidenceId: string;
|
||||
source: TaskEvaluationEvidenceSource | "category" | "signal" | "other";
|
||||
note?: string;
|
||||
}
|
||||
|
||||
export interface EvalFollowUpCreationRecommendation {
|
||||
shouldCreate: boolean;
|
||||
reason: string;
|
||||
policyQualified: boolean;
|
||||
}
|
||||
|
||||
export interface EvalFollowUpSuggestion {
|
||||
suggestionId: string;
|
||||
dedupeKey: string;
|
||||
title: string;
|
||||
description: string;
|
||||
priority?: "low" | "normal" | "high" | "urgent";
|
||||
priority: "low" | "normal" | "high" | "urgent";
|
||||
severity: EvalScoreBand;
|
||||
rationale: string;
|
||||
evidenceRefs: EvalFollowUpEvidenceReference[];
|
||||
recommendation: EvalFollowUpCreationRecommendation;
|
||||
state: EvalFollowUpSuggestionState;
|
||||
policyMode: EvalFollowUpPolicyMode;
|
||||
suppressedReason?: EvalFollowUpSuppressionReason;
|
||||
matchedTaskId?: string;
|
||||
matchedSuggestionId?: string;
|
||||
createdTaskId?: string;
|
||||
tags?: string[];
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function normalizeEvalFollowUpText(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, " ")
|
||||
.replace(/[^a-z0-9 ]/g, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function buildEvalFollowUpSuggestionId(seed: string): string {
|
||||
const normalized = normalizeEvalFollowUpText(seed);
|
||||
let hash = 2166136261;
|
||||
for (let i = 0; i < normalized.length; i += 1) {
|
||||
hash ^= normalized.charCodeAt(i);
|
||||
hash = Math.imul(hash, 16777619);
|
||||
}
|
||||
return `efs-${(hash >>> 0).toString(16).padStart(8, "0")}`;
|
||||
}
|
||||
|
||||
export interface EvalTaskResult {
|
||||
id: string;
|
||||
runId: string;
|
||||
|
||||
@@ -743,6 +743,11 @@ export type {
|
||||
RunAuditEvidence,
|
||||
TaskEvaluationEvidenceBundle,
|
||||
EvalSignal,
|
||||
EvalFollowUpPolicyMode,
|
||||
EvalFollowUpSuggestionState,
|
||||
EvalFollowUpSuppressionReason,
|
||||
EvalFollowUpEvidenceReference,
|
||||
EvalFollowUpCreationRecommendation,
|
||||
EvalFollowUpSuggestion,
|
||||
EvalProvenance,
|
||||
EvalStoreEvents,
|
||||
@@ -758,10 +763,15 @@ export {
|
||||
EVAL_SCORE_BANDS,
|
||||
EVAL_SCORE_SCALE_MIN,
|
||||
EVAL_SCORE_SCALE_MAX,
|
||||
EVAL_FOLLOW_UP_POLICY_MODES,
|
||||
EVAL_FOLLOW_UP_SUGGESTION_STATES,
|
||||
EVAL_FOLLOW_UP_SUPPRESSION_REASONS,
|
||||
TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
|
||||
EVIDENCE_LIMITS,
|
||||
MAX_EVIDENCE_EXCERPT_LENGTH,
|
||||
EVIDENCE_EXCERPT_TRUNCATION_MARKER,
|
||||
normalizeEvalFollowUpText,
|
||||
buildEvalFollowUpSuggestionId,
|
||||
} from "./eval-types.js";
|
||||
export {
|
||||
EVAL_CATEGORY_WEIGHTS,
|
||||
|
||||
Reference in New Issue
Block a user