feat(FN-3393): add eval settings with scheduled eval project settings UI
Adds a scheduled eval settings feature (FN-3393) with a new `evalSettings` contract in core, type-safe numeric validation, a project settings UI section, and an API route for persisting the payload. The bulk of the changes are in the settings modal and supporting test coverage. Two smaller, unrelate Fusion-Task-Id: FN-3393
This commit is contained in:
23
packages/core/src/eval-settings.ts
Normal file
23
packages/core/src/eval-settings.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import { resolveValidatorSettingsModel } from "./model-resolution.js";
|
||||
import type { ResolvedEvalSettings, Settings } from "./types.js";
|
||||
|
||||
const DEFAULT_EVAL_SETTINGS: Omit<ResolvedEvalSettings, "evaluatorProvider" | "evaluatorModelId"> = {
|
||||
enabled: false,
|
||||
intervalMs: 86_400_000,
|
||||
followUpPolicy: "suggest-only",
|
||||
retentionDays: 30,
|
||||
};
|
||||
|
||||
export function resolveEvalSettings(settings: Partial<Settings> | undefined): ResolvedEvalSettings {
|
||||
const scopedSettings = settings?.evalSettings;
|
||||
const validatorModel = resolveValidatorSettingsModel(settings);
|
||||
|
||||
return {
|
||||
enabled: scopedSettings?.enabled ?? DEFAULT_EVAL_SETTINGS.enabled,
|
||||
intervalMs: scopedSettings?.intervalMs ?? DEFAULT_EVAL_SETTINGS.intervalMs,
|
||||
evaluatorProvider: scopedSettings?.evaluatorProvider ?? validatorModel.provider,
|
||||
evaluatorModelId: scopedSettings?.evaluatorModelId ?? validatorModel.modelId,
|
||||
followUpPolicy: scopedSettings?.followUpPolicy ?? DEFAULT_EVAL_SETTINGS.followUpPolicy,
|
||||
retentionDays: scopedSettings?.retentionDays ?? DEFAULT_EVAL_SETTINGS.retentionDays,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user