root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-3393): add eval settings with scheduled eval project settings UI

Browse Source 
Adds a scheduled eval settings feature (FN-3393) with a new `evalSettings` contract in core, type-safe numeric validation, a project settings UI section, and an API route for persisting the payload. The bulk of the changes are in the settings modal and supporting test coverage. Two smaller, unrelate

Fusion-Task-Id: FN-3393
This commit is contained in:
Fusion
2026-05-07 07:36:36 -07:00
committed by gsxdsm
parent a1eb3934cd
commit 760621cc8c
11 changed files with 529 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
packages/core/src/__tests__/eval-settings.test.ts Normal file
View File

@@ -0,0 +1,74 @@
import { describe, expect, it } from "vitest";
import { resolveEvalSettings } from "../eval-settings.js";
describe("resolveEvalSettings", () => {
it("returns deterministic defaults when eval settings are unset", () => {
expect(resolveEvalSettings({})).toEqual({
enabled: false,
intervalMs: 86_400_000,
evaluatorProvider: undefined,
evaluatorModelId: undefined,
followUpPolicy: "suggest-only",
retentionDays: 30,
});
});
it("falls back to validator lane model when evaluator model is unset", () => {
expect(
resolveEvalSettings({
validatorProvider: "anthropic",
validatorModelId: "claude-sonnet-4-5",
}),
).toEqual({
enabled: false,
intervalMs: 86_400_000,
evaluatorProvider: "anthropic",
evaluatorModelId: "claude-sonnet-4-5",
followUpPolicy: "suggest-only",
retentionDays: 30,
});
});
it("prefers explicit evalSettings model overrides over validator lane", () => {
expect(
resolveEvalSettings({
validatorProvider: "anthropic",
validatorModelId: "claude-sonnet-4-5",
evalSettings: {
evaluatorProvider: "openai",
evaluatorModelId: "gpt-5",
},
}),
).toEqual({
enabled: false,
intervalMs: 86_400_000,
evaluatorProvider: "openai",
evaluatorModelId: "gpt-5",
followUpPolicy: "suggest-only",
retentionDays: 30,
});
});
it("ignores incomplete evaluator pair and keeps partial override + validator fallback", () => {
expect(
resolveEvalSettings({
validatorProvider: "anthropic",
validatorModelId: "claude-sonnet-4-5",
evalSettings: {
evaluatorProvider: "openai",
intervalMs: 120_000,
enabled: true,
followUpPolicy: "auto-create",
retentionDays: 14,
},
}),
).toEqual({
enabled: true,
intervalMs: 120_000,
evaluatorProvider: "openai",
evaluatorModelId: "claude-sonnet-4-5",
followUpPolicy: "auto-create",
retentionDays: 14,
});
});
});

30
packages/core/src/__tests__/settings-parity.test.ts
View File

@@ -121,24 +121,20 @@ describe("research global key parity regression (FN-3313)", () => {
// ── Model Lane Key Parity Regression Tests (FN-1729) ────────────────────────
describe("task evaluation key parity regression (FN-3514)", () => {
const taskEvaluationDefaults = {
taskEvaluationEnabled: false,
taskEvaluationSchedule: "0 5 * * *",
taskEvaluationProvider: undefined,
taskEvaluationModelId: undefined,
taskEvaluationFollowUpPolicy: "off",
taskEvaluationRetention: undefined,
} as const;
describe("eval settings parity regression (FN-3393)", () => {
it("keeps evalSettings project-scoped with expected defaults", () => {
expect(isProjectSettingsKey("evalSettings")).toBe(true);
expect(isGlobalSettingsKey("evalSettings")).toBe(false);
it.each(Object.entries(taskEvaluationDefaults))(
"%s is project-scoped with expected default",
(key, expectedDefault) => {
expect(isProjectSettingsKey(key)).toBe(true);
expect(isGlobalSettingsKey(key)).toBe(false);
expect((DEFAULT_PROJECT_SETTINGS as Record<string, unknown>)[key]).toBe(expectedDefault);
},
);
expect(DEFAULT_PROJECT_SETTINGS.evalSettings).toEqual({
enabled: false,
intervalMs: 86_400_000,
evaluatorProvider: undefined,
evaluatorModelId: undefined,
followUpPolicy: "suggest-only",
retentionDays: 30,
});
});
});
describe("model lane key parity regression (FN-1729)", () => {

23
packages/core/src/eval-settings.ts Normal file
View File

@@ -0,0 +1,23 @@
import { resolveValidatorSettingsModel } from "./model-resolution.js";
import type { ResolvedEvalSettings, Settings } from "./types.js";
const DEFAULT_EVAL_SETTINGS: Omit<ResolvedEvalSettings, "evaluatorProvider" | "evaluatorModelId"> = {
enabled: false,
intervalMs: 86_400_000,
followUpPolicy: "suggest-only",
retentionDays: 30,
};
export function resolveEvalSettings(settings: Partial<Settings> | undefined): ResolvedEvalSettings {
const scopedSettings = settings?.evalSettings;
const validatorModel = resolveValidatorSettingsModel(settings);
return {
enabled: scopedSettings?.enabled ?? DEFAULT_EVAL_SETTINGS.enabled,
intervalMs: scopedSettings?.intervalMs ?? DEFAULT_EVAL_SETTINGS.intervalMs,
evaluatorProvider: scopedSettings?.evaluatorProvider ?? validatorModel.provider,
evaluatorModelId: scopedSettings?.evaluatorModelId ?? validatorModel.modelId,
followUpPolicy: scopedSettings?.followUpPolicy ?? DEFAULT_EVAL_SETTINGS.followUpPolicy,
retentionDays: scopedSettings?.retentionDays ?? DEFAULT_EVAL_SETTINGS.retentionDays,
};
}

3
packages/core/src/index.ts
View File

@@ -1,5 +1,5 @@
export { COLUMNS, DEFAULT_COLUMN, isColumn, normalizeColumn, COLUMN_LABELS, COLUMN_DESCRIPTIONS, VALID_TRANSITIONS, DEFAULT_SETTINGS, DEFAULT_GLOBAL_SETTINGS, DEFAULT_PROJECT_SETTINGS, GLOBAL_SETTINGS_KEYS, PROJECT_SETTINGS_KEYS, isGlobalSettingsKey, isProjectSettingsKey, THINKING_LEVELS, THEME_MODES, COLOR_THEMES, WORKFLOW_STEP_TEMPLATES, AGENT_PERMISSIONS, agentToConfigSnapshot, diffConfigSnapshots, isEphemeralAgent, hasAgentIdentity, CheckoutConflictError, DEFAULT_HEARTBEAT_PROCEDURE_PATH, getDefaultHeartbeatProcedurePath, EXECUTION_MODES, DEFAULT_EXECUTION_MODE, TASK_PRIORITIES, DEFAULT_TASK_PRIORITY, DASHBOARD_USER_ID, normalizeMessageParticipant, validateMessageMetadata, validateDockerNodeConfig, sanitizeDockerNodeConfigForResponse, normalizeMergeConflictStrategy, buildResearchDocumentKey, PROJECT_AUTH_ROLES, SHARED_STATE_SNAPSHOT_VERSION } from "./types.js";
export type { Column, IssueInfo, IssueState, TaskSourceIssue, PrInfo, PrStatus, Task, TaskTokenUsage, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, TaskCreateInput, MeshReplicatedTaskCreatePayload, MeshReplicatedTaskApplyResult, TaskSource, SourceType, TaskDetail, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, ProjectAuthRole, ProjectAuthUser, ProjectAuthMembership, ProjectAuthProvider, ProjectAuthSession, ProjectAuthUserCreateInput, ProjectAuthMembershipCreateInput, ProjectAuthProviderCreateInput, ProjectAuthSessionCreateInput, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, DistributedTaskIdReserveInput, DistributedTaskIdReserveResult, DistributedTaskIdCommitInput, DistributedTaskIdCommitResult, DistributedTaskIdAbortInput, DistributedTaskIdAbortResult, DistributedTaskIdStateInput, DistributedTaskIdStateResult, AutostashOutcome, MergeDetails, MergeResult, MergeConflictStrategy, CanonicalMergeConflictStrategy, Settings, GlobalSettings, ProjectSettings, WebSearchBackend, ResearchEnabledSources, ResearchGlobalDefaults, ResearchProjectLimits, ResearchProjectSettings, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, ExecutionMode, TaskPriority, UnavailableNodePolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, MessageReplyReference, Mailbox, CheckoutLease, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter } from "./types.js";
export type { Column, IssueInfo, IssueState, TaskSourceIssue, PrInfo, PrStatus, Task, TaskTokenUsage, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, TaskCreateInput, MeshReplicatedTaskCreatePayload, MeshReplicatedTaskApplyResult, TaskSource, SourceType, TaskDetail, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, ProjectAuthRole, ProjectAuthUser, ProjectAuthMembership, ProjectAuthProvider, ProjectAuthSession, ProjectAuthUserCreateInput, ProjectAuthMembershipCreateInput, ProjectAuthProviderCreateInput, ProjectAuthSessionCreateInput, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, DistributedTaskIdReserveInput, DistributedTaskIdReserveResult, DistributedTaskIdCommitInput, DistributedTaskIdCommitResult, DistributedTaskIdAbortInput, DistributedTaskIdAbortResult, DistributedTaskIdStateInput, DistributedTaskIdStateResult, AutostashOutcome, MergeDetails, MergeResult, MergeConflictStrategy, CanonicalMergeConflictStrategy, Settings, GlobalSettings, ProjectSettings, WebSearchBackend, ResearchEnabledSources, ResearchGlobalDefaults, ResearchProjectLimits, ResearchProjectSettings, EvalFollowUpPolicy, EvalProjectSettings, ResolvedEvalSettings, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, ExecutionMode, TaskPriority, UnavailableNodePolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, MessageReplyReference, Mailbox, CheckoutLease, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter } from "./types.js";
export { AGENT_VALID_TRANSITIONS } from "./types.js";
export * from "./mesh-replication-protocol.js";
export * from "./mesh-task-replication.js";
@@ -722,6 +722,7 @@ export type {
export { resolveResearchSettings } from "./research-settings.js";
export type { ResolvedResearchSettings } from "./research-settings.js";
export { resolveEvalSettings } from "./eval-settings.js";
export { TodoStore } from "./todo-store.js";
export type { TodoStoreEvents } from "./todo-store.js";

8
packages/core/src/settings-schema.ts
View File

@@ -289,6 +289,14 @@ export const DEFAULT_PROJECT_SETTINGS = {
requestTimeoutMs: 30000,
},
},
evalSettings: {
enabled: false,
intervalMs: 86_400_000,
evaluatorProvider: undefined,
evaluatorModelId: undefined,
followUpPolicy: "suggest-only",
retentionDays: 30,
},
researchEnabled: true,
researchMaxConcurrentRuns: 3,
researchDefaultTimeout: 300000,

22
packages/core/src/types.ts
View File

@@ -1366,6 +1366,26 @@ export interface ResearchProjectSettings {
limits?: ResearchProjectLimits;
}
export type EvalFollowUpPolicy = "disabled" | "suggest-only" | "auto-create";
export interface EvalProjectSettings {
enabled?: boolean;
intervalMs?: number;
evaluatorProvider?: string;
evaluatorModelId?: string;
followUpPolicy?: EvalFollowUpPolicy;
retentionDays?: number;
}
export interface ResolvedEvalSettings {
enabled: boolean;
intervalMs: number;
evaluatorProvider?: string;
evaluatorModelId?: string;
followUpPolicy: EvalFollowUpPolicy;
retentionDays: number;
}
export interface GlobalSettings {
/** Theme mode preference: dark, light, or system (follows OS). Default: "dark". */
themeMode?: ThemeMode;
@@ -1793,6 +1813,8 @@ export interface ProjectSettings {
unavailableNodePolicy?: UnavailableNodePolicy;
/** Project-level research configuration overrides. */
researchSettings?: ResearchProjectSettings;
/** Project-level scheduled eval configuration overrides. */
evalSettings?: EvalProjectSettings;
/** Enable scheduled evaluation batches for recently completed tasks. */
taskEvaluationEnabled?: boolean;
/** Cron expression for scheduled task-evaluation batches. */

136
packages/dashboard/app/components/SettingsModal.tsx
View File

@@ -222,6 +222,7 @@ const SETTINGS_SECTIONS: SettingsSection[] = [
{ id: "general", label: "Project General", scope: "project" },
{ id: "project-models", label: "Project Models", scope: "project" },
{ id: "scheduling", label: "Scheduling", scope: "project" },
{ id: "scheduled-evals", label: "Scheduled Evals", scope: "project" },
{ id: "node-routing", label: "Node Routing", scope: "project" },
{ id: "worktrees", label: "Worktrees", scope: "project" },
{ id: "commands", label: "Commands", scope: "project" },
@@ -3119,6 +3120,141 @@ export function SettingsModal({
</div>
</>
);
case "scheduled-evals": {
const evalSettings = form.evalSettings ?? {};
const isScheduledEvalEnabled = evalSettings.enabled ?? false;
return (
<>
{renderScopeBanner()}
<h4 className="settings-section-heading">Scheduled Evals</h4>
<div className="form-group">
<label htmlFor="scheduled-evals-enabled" className="checkbox-label">
<input
id="scheduled-evals-enabled"
type="checkbox"
checked={isScheduledEvalEnabled}
onChange={(event) =>
setForm((current) => ({
...current,
evalSettings: {
...(current.evalSettings ?? {}),
enabled: event.target.checked,
},
}))
}
/>
Enable scheduled eval runs for this project
</label>
</div>
<div className="form-group">
<label htmlFor="scheduled-evals-interval">Interval (ms)</label>
<input
id="scheduled-evals-interval"
className="input"
type="number"
min={60000}
max={604800000}
step={1000}
disabled={!isScheduledEvalEnabled}
value={evalSettings.intervalMs ?? 86_400_000}
onChange={(event) =>
setForm((current) => ({
...current,
evalSettings: {
...(current.evalSettings ?? {}),
intervalMs: event.target.value === "" ? undefined : Number(event.target.value),
},
}))
}
/>
</div>
<div className="form-group">
<label htmlFor="scheduled-evals-provider">Evaluator Provider</label>
<input
id="scheduled-evals-provider"
className="input"
value={evalSettings.evaluatorProvider ?? ""}
onChange={(event) =>
setForm((current) => ({
...current,
evalSettings: {
...(current.evalSettings ?? {}),
evaluatorProvider: event.target.value.trim() === "" ? undefined : event.target.value,
},
}))
}
placeholder="openai"
/>
</div>
<div className="form-group">
<label htmlFor="scheduled-evals-model">Evaluator Model</label>
<input
id="scheduled-evals-model"
className="input"
value={evalSettings.evaluatorModelId ?? ""}
onChange={(event) =>
setForm((current) => ({
...current,
evalSettings: {
...(current.evalSettings ?? {}),
evaluatorModelId: event.target.value.trim() === "" ? undefined : event.target.value,
},
}))
}
placeholder="gpt-5"
/>
<small className="form-text text-muted">
Leave provider and model blank to inherit the project validator lane model settings.
</small>
</div>
<div className="form-group">
<label htmlFor="scheduled-evals-follow-up-policy">Follow-up Policy</label>
<select
id="scheduled-evals-follow-up-policy"
className="select"
disabled={!isScheduledEvalEnabled}
value={evalSettings.followUpPolicy ?? "suggest-only"}
onChange={(event) =>
setForm((current) => ({
...current,
evalSettings: {
...(current.evalSettings ?? {}),
followUpPolicy: event.target.value as "disabled" | "suggest-only" | "auto-create",
},
}))
}
>
<option value="disabled">Disabled</option>
<option value="suggest-only">Suggest only</option>
<option value="auto-create">Auto-create tasks</option>
</select>
</div>
<div className="form-group">
<label htmlFor="scheduled-evals-retention-days">Retention (days)</label>
<input
id="scheduled-evals-retention-days"
className="input"
type="number"
min={1}
max={365}
step={1}
disabled={!isScheduledEvalEnabled}
value={evalSettings.retentionDays ?? 30}
onChange={(event) =>
setForm((current) => ({
...current,
evalSettings: {
...(current.evalSettings ?? {}),
retentionDays: event.target.value === "" ? undefined : Number(event.target.value),
},
}))
}
/>
</div>
</>
);
}
case "node-routing":
return (
<>

102
packages/dashboard/app/components/__tests__/SettingsModal.test.tsx
View File

@@ -2766,6 +2766,108 @@ describe("SettingsModal", () => {
});
});
describe("scheduled eval settings section", () => {
const openScheduledEvalsSection = async () => {
await userEvent.click(await screen.findByRole("button", { name: /Scheduled Evals/i }));
};
it("renders controls and disables interval controls when evals are disabled", async () => {
mockFetchSettings.mockResolvedValueOnce({
...defaultSettings,
evalSettings: {
enabled: false,
intervalMs: 86_400_000,
followUpPolicy: "suggest-only",
retentionDays: 30,
},
});
renderModal();
await waitForSettingsModalReady();
await openScheduledEvalsSection();
expect(await screen.findByRole("heading", { name: "Scheduled Evals" })).toBeInTheDocument();
expect(screen.getByLabelText("Enable scheduled eval runs for this project")).toBeInTheDocument();
expect(screen.getByLabelText("Interval (ms)")).toBeDisabled();
expect(screen.getByLabelText("Follow-up Policy")).toBeDisabled();
expect(screen.getByLabelText("Retention (days)")).toBeDisabled();
expect(screen.getByText(/inherit the project validator lane model settings/i)).toBeInTheDocument();
});
it("saves edited project eval settings payload", async () => {
mockFetchSettings.mockResolvedValueOnce({
...defaultSettings,
evalSettings: {
enabled: true,
intervalMs: 86_400_000,
followUpPolicy: "suggest-only",
retentionDays: 30,
},
});
renderModal();
await waitForSettingsModalReady();
await openScheduledEvalsSection();
fireEvent.change(screen.getByLabelText("Interval (ms)"), { target: { value: "120000" } });
await userEvent.type(screen.getByLabelText("Evaluator Provider"), "openai");
await userEvent.type(screen.getByLabelText("Evaluator Model"), "gpt-5");
await userEvent.selectOptions(screen.getByLabelText("Follow-up Policy"), "auto-create");
fireEvent.change(screen.getByLabelText("Retention (days)"), { target: { value: "14" } });
await userEvent.click(screen.getByText("Save"));
await waitFor(() => {
expect(mockUpdateSettings).toHaveBeenCalledWith(
expect.objectContaining({
evalSettings: expect.objectContaining({
enabled: true,
intervalMs: 120000,
evaluatorProvider: "openai",
evaluatorModelId: "gpt-5",
followUpPolicy: "auto-create",
retentionDays: 14,
}),
}),
undefined,
);
});
});
it("clears evaluator provider and model as unset when left blank", async () => {
mockFetchSettings.mockResolvedValueOnce({
...defaultSettings,
evalSettings: {
enabled: true,
intervalMs: 86_400_000,
evaluatorProvider: "openai",
evaluatorModelId: "gpt-5",
followUpPolicy: "suggest-only",
retentionDays: 30,
},
});
renderModal();
await waitForSettingsModalReady();
await openScheduledEvalsSection();
await userEvent.clear(screen.getByLabelText("Evaluator Provider"));
await userEvent.clear(screen.getByLabelText("Evaluator Model"));
await userEvent.click(screen.getByText("Save"));
await waitFor(() => {
expect(mockUpdateSettings).toHaveBeenCalledWith(
expect.objectContaining({
evalSettings: expect.objectContaining({
evaluatorProvider: undefined,
evaluatorModelId: undefined,
}),
}),
undefined,
);
});
});
});
describe("research settings sections", () => {
beforeEach(() => {
mockFetchSettings.mockResolvedValue({

88
packages/dashboard/src/__tests__/routes-settings.test.ts
View File

@@ -713,6 +713,69 @@ describe("PUT /settings", () => {
expect(store.updateSettings).toHaveBeenCalledWith({ maxConcurrent: 8, autoMerge: false });
});
it("accepts valid nested evalSettings payload", async () => {
(store.updateSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
...DEFAULT_SETTINGS,
evalSettings: {
enabled: true,
intervalMs: 300000,
evaluatorProvider: "openai",
evaluatorModelId: "gpt-5",
followUpPolicy: "suggest-only",
retentionDays: 45,
},
});
const payload = {
evalSettings: {
enabled: true,
intervalMs: 300000,
evaluatorProvider: "openai",
evaluatorModelId: "gpt-5",
followUpPolicy: "suggest-only",
retentionDays: 45,
},
};
const res = await REQUEST(buildApp(), "PUT", "/api/settings", JSON.stringify(payload), {
"Content-Type": "application/json",
});
expect(res.status).toBe(200);
expect(store.updateSettings).toHaveBeenCalledWith(payload);
});
it("rejects invalid evalSettings payloads", async () => {
const invalidPayloads = [
{
payload: { evalSettings: { intervalMs: 59_999 } },
message: "evalSettings.intervalMs",
},
{
payload: { evalSettings: { retentionDays: 0 } },
message: "evalSettings.retentionDays",
},
{
payload: { evalSettings: { followUpPolicy: "create" } },
message: "evalSettings.followUpPolicy",
},
{
payload: { evalSettings: { evaluatorProvider: "openai" } },
message: "evalSettings.evaluatorProvider and evalSettings.evaluatorModelId",
},
];
for (const { payload, message } of invalidPayloads) {
const res = await REQUEST(buildApp(), "PUT", "/api/settings", JSON.stringify(payload), {
"Content-Type": "application/json",
});
expect(res.status).toBe(400);
expect(res.body.error).toContain(message);
}
expect(store.updateSettings).not.toHaveBeenCalled();
});
it("accepts partial remoteAccess patches and GET /settings returns merged sibling branches", async () => {
const mergedRemoteAccess = {
enabled: true,
@@ -1251,6 +1314,31 @@ describe("GET /settings/scopes", () => {
expect(res.body.project.titleSummarizerModelId).toBe("gemini-2.5-pro");
});
it("returns evalSettings in project scope only", async () => {
(store.getSettingsByScope as ReturnType<typeof vi.fn>).mockResolvedValue({
global: { themeMode: "dark" },
project: {
evalSettings: {
enabled: true,
intervalMs: 300000,
followUpPolicy: "auto-create",
retentionDays: 14,
},
},
});
const res = await GET(buildApp(), "/api/settings/scopes");
expect(res.status).toBe(200);
expect(res.body.project.evalSettings).toEqual({
enabled: true,
intervalMs: 300000,
followUpPolicy: "auto-create",
retentionDays: 14,
});
expect(res.body.global.evalSettings).toBeUndefined();
});
it("returns remoteAccess only under project scope", async () => {
(store.getSettingsByScope as ReturnType<typeof vi.fn>).mockResolvedValue({
global: { themeMode: "dark" },

33
packages/dashboard/src/routes/register-settings-memory-routes.ts
View File

@@ -439,6 +439,39 @@ export function registerSettingsMemoryRoutes(ctx: ApiRoutesContext, deps: Settin
clientSettings.unavailableNodePolicy = validatedUnavailableNodePolicy;
}
const evalSettings = clientSettings.evalSettings as Record<string, unknown> | null | undefined;
if (evalSettings !== undefined && evalSettings !== null) {
if (typeof evalSettings !== "object" || Array.isArray(evalSettings)) {
throw badRequest("evalSettings must be an object");
}
const allowedFollowUpPolicies = ["disabled", "suggest-only", "auto-create"];
const intervalMs = evalSettings.intervalMs;
if (intervalMs !== undefined && intervalMs !== null) {
if (typeof intervalMs !== "number" || !Number.isInteger(intervalMs) || intervalMs < 60_000 || intervalMs > 604_800_000) {
throw badRequest("evalSettings.intervalMs must be an integer between 60000 and 604800000");
}
}
const retentionDays = evalSettings.retentionDays;
if (retentionDays !== undefined && retentionDays !== null) {
if (typeof retentionDays !== "number" || !Number.isInteger(retentionDays) || retentionDays < 1 || retentionDays > 365) {
throw badRequest("evalSettings.retentionDays must be an integer between 1 and 365");
}
}
const followUpPolicy = evalSettings.followUpPolicy;
if (followUpPolicy !== undefined && !allowedFollowUpPolicies.includes(String(followUpPolicy))) {
throw badRequest("evalSettings.followUpPolicy must be one of: disabled, suggest-only, auto-create");
}
const hasEvaluatorProvider = evalSettings.evaluatorProvider !== undefined && evalSettings.evaluatorProvider !== null && String(evalSettings.evaluatorProvider).trim() !== "";
const hasEvaluatorModelId = evalSettings.evaluatorModelId !== undefined && evalSettings.evaluatorModelId !== null && String(evalSettings.evaluatorModelId).trim() !== "";
if (hasEvaluatorProvider !== hasEvaluatorModelId) {
throw badRequest("evalSettings.evaluatorProvider and evalSettings.evaluatorModelId must be provided together or both omitted");
}
}
// Validate memoryBackendType if provided - must be string or null (for explicit clear)
// Unknown backend IDs are accepted and persisted verbatim (for custom backend compatibility)
// Fallback-to-file is runtime resolution behavior only