feat(FN-3393): add eval settings with scheduled eval project settings UI

Adds a scheduled eval settings feature (FN-3393) with a new `evalSettings` contract in core, type-safe numeric validation, a project settings UI section, and an API route for persisting the payload. The bulk of the changes are in the settings modal and supporting test coverage. Two smaller, unrelate

Fusion-Task-Id: FN-3393

packages/core/src/__tests__/eval-settings.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, it } from "vitest";
+import { resolveEvalSettings } from "../eval-settings.js";
+
+describe("resolveEvalSettings", () => {
+  it("returns deterministic defaults when eval settings are unset", () => {
+    expect(resolveEvalSettings({})).toEqual({
+      enabled: false,
+      intervalMs: 86_400_000,
+      evaluatorProvider: undefined,
+      evaluatorModelId: undefined,
+      followUpPolicy: "suggest-only",
+      retentionDays: 30,
+    });
+  });
+
+  it("falls back to validator lane model when evaluator model is unset", () => {
+    expect(
+      resolveEvalSettings({
+        validatorProvider: "anthropic",
+        validatorModelId: "claude-sonnet-4-5",
+      }),
+    ).toEqual({
+      enabled: false,
+      intervalMs: 86_400_000,
+      evaluatorProvider: "anthropic",
+      evaluatorModelId: "claude-sonnet-4-5",
+      followUpPolicy: "suggest-only",
+      retentionDays: 30,
+    });
+  });
+
+  it("prefers explicit evalSettings model overrides over validator lane", () => {
+    expect(
+      resolveEvalSettings({
+        validatorProvider: "anthropic",
+        validatorModelId: "claude-sonnet-4-5",
+        evalSettings: {
+          evaluatorProvider: "openai",
+          evaluatorModelId: "gpt-5",
+        },
+      }),
+    ).toEqual({
+      enabled: false,
+      intervalMs: 86_400_000,
+      evaluatorProvider: "openai",
+      evaluatorModelId: "gpt-5",
+      followUpPolicy: "suggest-only",
+      retentionDays: 30,
+    });
+  });
+
+  it("ignores incomplete evaluator pair and keeps partial override + validator fallback", () => {
+    expect(
+      resolveEvalSettings({
+        validatorProvider: "anthropic",
+        validatorModelId: "claude-sonnet-4-5",
+        evalSettings: {
+          evaluatorProvider: "openai",
+          intervalMs: 120_000,
+          enabled: true,
+          followUpPolicy: "auto-create",
+          retentionDays: 14,
+        },
+      }),
+    ).toEqual({
+      enabled: true,
+      intervalMs: 120_000,
+      evaluatorProvider: "openai",
+      evaluatorModelId: "claude-sonnet-4-5",
+      followUpPolicy: "auto-create",
+      retentionDays: 14,
+    });
+  });
+});

packages/core/src/__tests__/settings-parity.test.ts

@@ -121,24 +121,20 @@ describe("research global key parity regression (FN-3313)", () => {
 
 // ── Model Lane Key Parity Regression Tests (FN-1729) ────────────────────────
 
-describe("task evaluation key parity regression (FN-3514)", () => {
-  const taskEvaluationDefaults = {
-    taskEvaluationEnabled: false,
-    taskEvaluationSchedule: "0 5 * * *",
-    taskEvaluationProvider: undefined,
-    taskEvaluationModelId: undefined,
-    taskEvaluationFollowUpPolicy: "off",
-    taskEvaluationRetention: undefined,
-  } as const;
+describe("eval settings parity regression (FN-3393)", () => {
+  it("keeps evalSettings project-scoped with expected defaults", () => {
+    expect(isProjectSettingsKey("evalSettings")).toBe(true);
+    expect(isGlobalSettingsKey("evalSettings")).toBe(false);
 
-  it.each(Object.entries(taskEvaluationDefaults))(
-    "%s is project-scoped with expected default",
-    (key, expectedDefault) => {
-      expect(isProjectSettingsKey(key)).toBe(true);
-      expect(isGlobalSettingsKey(key)).toBe(false);
-      expect((DEFAULT_PROJECT_SETTINGS as Record<string, unknown>)[key]).toBe(expectedDefault);
-    },
-  );
+    expect(DEFAULT_PROJECT_SETTINGS.evalSettings).toEqual({
+      enabled: false,
+      intervalMs: 86_400_000,
+      evaluatorProvider: undefined,
+      evaluatorModelId: undefined,
+      followUpPolicy: "suggest-only",
+      retentionDays: 30,
+    });
+  });
 });
 
 describe("model lane key parity regression (FN-1729)", () => {

packages/core/src/eval-settings.ts

@@ -0,0 +1,23 @@
+import { resolveValidatorSettingsModel } from "./model-resolution.js";
+import type { ResolvedEvalSettings, Settings } from "./types.js";
+
+const DEFAULT_EVAL_SETTINGS: Omit<ResolvedEvalSettings, "evaluatorProvider" | "evaluatorModelId"> = {
+  enabled: false,
+  intervalMs: 86_400_000,
+  followUpPolicy: "suggest-only",
+  retentionDays: 30,
+};
+
+export function resolveEvalSettings(settings: Partial<Settings> | undefined): ResolvedEvalSettings {
+  const scopedSettings = settings?.evalSettings;
+  const validatorModel = resolveValidatorSettingsModel(settings);
+
+  return {
+    enabled: scopedSettings?.enabled ?? DEFAULT_EVAL_SETTINGS.enabled,
+    intervalMs: scopedSettings?.intervalMs ?? DEFAULT_EVAL_SETTINGS.intervalMs,
+    evaluatorProvider: scopedSettings?.evaluatorProvider ?? validatorModel.provider,
+    evaluatorModelId: scopedSettings?.evaluatorModelId ?? validatorModel.modelId,
+    followUpPolicy: scopedSettings?.followUpPolicy ?? DEFAULT_EVAL_SETTINGS.followUpPolicy,
+    retentionDays: scopedSettings?.retentionDays ?? DEFAULT_EVAL_SETTINGS.retentionDays,
+  };
+}

packages/core/src/index.ts

@@ -1,5 +1,5 @@
 export { COLUMNS, DEFAULT_COLUMN, isColumn, normalizeColumn, COLUMN_LABELS, COLUMN_DESCRIPTIONS, VALID_TRANSITIONS, DEFAULT_SETTINGS, DEFAULT_GLOBAL_SETTINGS, DEFAULT_PROJECT_SETTINGS, GLOBAL_SETTINGS_KEYS, PROJECT_SETTINGS_KEYS, isGlobalSettingsKey, isProjectSettingsKey, THINKING_LEVELS, THEME_MODES, COLOR_THEMES, WORKFLOW_STEP_TEMPLATES, AGENT_PERMISSIONS, agentToConfigSnapshot, diffConfigSnapshots, isEphemeralAgent, hasAgentIdentity, CheckoutConflictError, DEFAULT_HEARTBEAT_PROCEDURE_PATH, getDefaultHeartbeatProcedurePath, EXECUTION_MODES, DEFAULT_EXECUTION_MODE, TASK_PRIORITIES, DEFAULT_TASK_PRIORITY, DASHBOARD_USER_ID, normalizeMessageParticipant, validateMessageMetadata, validateDockerNodeConfig, sanitizeDockerNodeConfigForResponse, normalizeMergeConflictStrategy, buildResearchDocumentKey, PROJECT_AUTH_ROLES, SHARED_STATE_SNAPSHOT_VERSION } from "./types.js";
-export type { Column, IssueInfo, IssueState, TaskSourceIssue, PrInfo, PrStatus, Task, TaskTokenUsage, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, TaskCreateInput, MeshReplicatedTaskCreatePayload, MeshReplicatedTaskApplyResult, TaskSource, SourceType, TaskDetail, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, ProjectAuthRole, ProjectAuthUser, ProjectAuthMembership, ProjectAuthProvider, ProjectAuthSession, ProjectAuthUserCreateInput, ProjectAuthMembershipCreateInput, ProjectAuthProviderCreateInput, ProjectAuthSessionCreateInput, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, DistributedTaskIdReserveInput, DistributedTaskIdReserveResult, DistributedTaskIdCommitInput, DistributedTaskIdCommitResult, DistributedTaskIdAbortInput, DistributedTaskIdAbortResult, DistributedTaskIdStateInput, DistributedTaskIdStateResult, AutostashOutcome, MergeDetails, MergeResult, MergeConflictStrategy, CanonicalMergeConflictStrategy, Settings, GlobalSettings, ProjectSettings, WebSearchBackend, ResearchEnabledSources, ResearchGlobalDefaults, ResearchProjectLimits, ResearchProjectSettings, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, ExecutionMode, TaskPriority, UnavailableNodePolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, MessageReplyReference, Mailbox, CheckoutLease, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter } from "./types.js";
+export type { Column, IssueInfo, IssueState, TaskSourceIssue, PrInfo, PrStatus, Task, TaskTokenUsage, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, TaskCreateInput, MeshReplicatedTaskCreatePayload, MeshReplicatedTaskApplyResult, TaskSource, SourceType, TaskDetail, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, ProjectAuthRole, ProjectAuthUser, ProjectAuthMembership, ProjectAuthProvider, ProjectAuthSession, ProjectAuthUserCreateInput, ProjectAuthMembershipCreateInput, ProjectAuthProviderCreateInput, ProjectAuthSessionCreateInput, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, DistributedTaskIdReserveInput, DistributedTaskIdReserveResult, DistributedTaskIdCommitInput, DistributedTaskIdCommitResult, DistributedTaskIdAbortInput, DistributedTaskIdAbortResult, DistributedTaskIdStateInput, DistributedTaskIdStateResult, AutostashOutcome, MergeDetails, MergeResult, MergeConflictStrategy, CanonicalMergeConflictStrategy, Settings, GlobalSettings, ProjectSettings, WebSearchBackend, ResearchEnabledSources, ResearchGlobalDefaults, ResearchProjectLimits, ResearchProjectSettings, EvalFollowUpPolicy, EvalProjectSettings, ResolvedEvalSettings, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, ExecutionMode, TaskPriority, UnavailableNodePolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, MessageReplyReference, Mailbox, CheckoutLease, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter } from "./types.js";
 export { AGENT_VALID_TRANSITIONS } from "./types.js";
 export * from "./mesh-replication-protocol.js";
 export * from "./mesh-task-replication.js";
@@ -722,6 +722,7 @@ export type {
 
 export { resolveResearchSettings } from "./research-settings.js";
 export type { ResolvedResearchSettings } from "./research-settings.js";
+export { resolveEvalSettings } from "./eval-settings.js";
 
 export { TodoStore } from "./todo-store.js";
 export type { TodoStoreEvents } from "./todo-store.js";

packages/core/src/settings-schema.ts

@@ -289,6 +289,14 @@ export const DEFAULT_PROJECT_SETTINGS = {
       requestTimeoutMs: 30000,
     },
   },
+  evalSettings: {
+    enabled: false,
+    intervalMs: 86_400_000,
+    evaluatorProvider: undefined,
+    evaluatorModelId: undefined,
+    followUpPolicy: "suggest-only",
+    retentionDays: 30,
+  },
   researchEnabled: true,
   researchMaxConcurrentRuns: 3,
   researchDefaultTimeout: 300000,

packages/core/src/types.ts

@@ -1366,6 +1366,26 @@ export interface ResearchProjectSettings {
   limits?: ResearchProjectLimits;
 }
 
+export type EvalFollowUpPolicy = "disabled" | "suggest-only" | "auto-create";
+
+export interface EvalProjectSettings {
+  enabled?: boolean;
+  intervalMs?: number;
+  evaluatorProvider?: string;
+  evaluatorModelId?: string;
+  followUpPolicy?: EvalFollowUpPolicy;
+  retentionDays?: number;
+}
+
+export interface ResolvedEvalSettings {
+  enabled: boolean;
+  intervalMs: number;
+  evaluatorProvider?: string;
+  evaluatorModelId?: string;
+  followUpPolicy: EvalFollowUpPolicy;
+  retentionDays: number;
+}
+
 export interface GlobalSettings {
   /** Theme mode preference: dark, light, or system (follows OS). Default: "dark". */
   themeMode?: ThemeMode;
@@ -1793,6 +1813,8 @@ export interface ProjectSettings {
   unavailableNodePolicy?: UnavailableNodePolicy;
   /** Project-level research configuration overrides. */
   researchSettings?: ResearchProjectSettings;
+  /** Project-level scheduled eval configuration overrides. */
+  evalSettings?: EvalProjectSettings;
   /** Enable scheduled evaluation batches for recently completed tasks. */
   taskEvaluationEnabled?: boolean;
   /** Cron expression for scheduled task-evaluation batches. */