fusion/packages/engine/src/__tests__/eval-followups.test.ts

import { describe, expect, it, vi } from "vitest";
import { normalizeEvalFollowUpText } from "@fusion/core";
import { materializeEvalFollowUps, normalizeEvalFollowUps, resolveEvalFollowUpPolicyMode } from "../eval-followups.js";

function makeStore(params: { openTasks?: Array<{ id: string; column: string; title?: string; description: string }>; priorDedupeKeys?: string[] }) {
  const openTasks = params.openTasks ?? [];
  const priorDedupeKeys = params.priorDedupeKeys ?? [];
  return {
    listTasks: async () => openTasks,
    createTask: vi.fn(async () => ({ id: "FN-created" })),
    getEvalStore: () => ({
      listTaskResults: () => [{ followUps: priorDedupeKeys.map((dedupeKey) => ({ dedupeKey })) }],
    }),
  } as any;
}

describe("normalizeEvalFollowUps", () => {
  it("suppresses empty or generic suggestions", async () => {
    const followUps = await normalizeEvalFollowUps({
      parentTaskId: "FN-1",
      runId: "ER-1",
      overallBand: "weak",
      drafts: [{ title: "Follow-up", description: "too short", reason: "", evidenceRefs: [] }],
      store: makeStore({}),
      policyMode: "persist_only",
    });

    expect(followUps[0]?.state).toBe("suppressed");
    expect(followUps[0]?.suppressedReason).toBe("empty_or_generic");
  });

  it("suppresses duplicates of open tasks", async () => {
    const followUps = await normalizeEvalFollowUps({
      parentTaskId: "FN-1",
      runId: "ER-1",
      overallBand: "weak",
      drafts: [{ title: "Investigate flaky verification command", description: "Investigate flaky verification command causing reruns.", reason: "Failed verification", evidenceRefs: ["workflow-1"] }],
      store: makeStore({
        openTasks: [{ id: "FN-open", column: "todo", title: "Investigate flaky verification command", description: "x" }],
      }),
      policyMode: "persist_only",
    });

    expect(followUps[0]?.state).toBe("suppressed");
    expect(followUps[0]?.suppressedReason).toBe("duplicate_open_task");
    expect(followUps[0]?.matchedTaskId).toBe("FN-open");
  });

  it("suppresses duplicates from prior eval results", async () => {
    const priorKey = normalizeEvalFollowUpText("FN-1:Add regression test for merge flow:Add regression test for merge flow regressions.");
    const followUps = await normalizeEvalFollowUps({
      parentTaskId: "FN-1",
      runId: "ER-1",
      overallBand: "weak",
      drafts: [{ title: "Add regression test for merge flow", description: "Add regression test for merge flow regressions.", reason: "Missing tests", evidenceRefs: ["workflow-2"] }],
      store: makeStore({ priorDedupeKeys: [priorKey] }),
      policyMode: "persist_only",
    });

    expect(followUps[0]?.state).toBe("suppressed");
    expect(followUps[0]?.suppressedReason).toBe("duplicate_prior_suggestion");
  });

  it("marks qualified follow-ups for creation in create-all mode", async () => {
    const followUps = await normalizeEvalFollowUps({
      parentTaskId: "FN-1",
      runId: "ER-1",
      overallBand: "weak",
      drafts: [{ title: "Add flaky test diagnostics", description: "Add flaky test diagnostics for failing suite evidence.", reason: "Multiple failing runs", evidenceRefs: ["workflow-3"] }],
      store: makeStore({}),
      policyMode: "create_all_non_duplicates",
    });

    expect(followUps[0]?.state).toBe("suggested");
    expect(followUps[0]?.recommendation.shouldCreate).toBe(true);
    expect(followUps[0]?.policyMode).toBe("create_all_non_duplicates");
  });

  it("resolves project follow-up policy to backend policy mode", () => {
    expect(resolveEvalFollowUpPolicyMode("off")).toBe("persist_only");
    expect(resolveEvalFollowUpPolicyMode("suggest")).toBe("persist_only");
    expect(resolveEvalFollowUpPolicyMode("create")).toBe("auto_create_qualified");
  });

  it("creates task and stamps provenance when suggestion is qualified", async () => {
    const store = makeStore({});
    const [created] = await materializeEvalFollowUps({
      parentTaskId: "FN-parent",
      runId: "ER-5",
      policyMode: "create_all_non_duplicates",
      overallScore: 42,
      store,
      followUps: [{
        suggestionId: "efs-1",
        dedupeKey: "k",
        title: "Investigate issue",
        description: "Investigate issue found by eval.",
        priority: "high",
        severity: "weak",
        rationale: "Signals showed repeated failures.",
        evidenceRefs: [{ evidenceId: "workflow-1", source: "other" }],
        recommendation: { shouldCreate: true, reason: "qualified", policyQualified: true },
        state: "suggested",
        policyMode: "create_all_non_duplicates",
      }],
    });

    expect(store.createTask).toHaveBeenCalledTimes(1);
    expect(store.createTask.mock.calls[0][0].source.sourceParentTaskId).toBe("FN-parent");
    expect(store.createTask.mock.calls[0][0].source.sourceMetadata).toMatchObject({
      runId: "ER-5",
      suggestionId: "efs-1",
      policyMode: "create_all_non_duplicates",
    });
    expect(created?.state).toBe("created");
    expect(created?.createdTaskId).toBe("FN-created");
  });
});