feat(FN-3388): document scheduled eval batch architecture
Added 3 lines documenting the scheduled eval batch architecture in the architecture docs as part of FN-3388 Step 4. Fusion-Task-Id: FN-3388
This commit is contained in:
Fusion
138
packages/core/src/__tests__/eval-automation.test.ts
Normal file
138
packages/core/src/__tests__/eval-automation.test.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createDatabase } from "../db.js";
|
||||
import { EvalStore } from "../eval-store.js";
|
||||
import {
|
||||
DEFAULT_TASK_EVALUATION_SCHEDULE,
|
||||
createScheduledEvalBatchAutomation,
|
||||
resolveTaskEvaluationSettings,
|
||||
runScheduledEvalBatch,
|
||||
syncScheduledEvalBatchAutomation,
|
||||
} from "../eval-automation.js";
|
||||
|
||||
function task(id: string, column: "done" | "todo" | "archived", completedAt: string, createdAt = "2026-01-01T00:00:00.000Z") {
|
||||
return {
|
||||
id,
|
||||
column,
|
||||
createdAt,
|
||||
updatedAt: createdAt,
|
||||
executionCompletedAt: completedAt,
|
||||
title: id,
|
||||
summary: id,
|
||||
} as any;
|
||||
}
|
||||
|
||||
describe("eval-automation", () => {
|
||||
it("resolves task evaluation settings defaults", () => {
|
||||
const resolved = resolveTaskEvaluationSettings({});
|
||||
expect(resolved.taskEvaluationEnabled).toBe(false);
|
||||
expect(resolved.taskEvaluationSchedule).toBe(DEFAULT_TASK_EVALUATION_SCHEDULE);
|
||||
expect(resolved.taskEvaluationFollowUpPolicy).toBe("off");
|
||||
});
|
||||
|
||||
it("creates scheduled eval automation", () => {
|
||||
const input = createScheduledEvalBatchAutomation({ taskEvaluationSchedule: "0 9 * * *" });
|
||||
expect(input.name).toBe("Scheduled Task Evaluation");
|
||||
expect(input.cronExpression).toBe("0 9 * * *");
|
||||
expect(input.scope).toBe("project");
|
||||
});
|
||||
|
||||
it("syncs schedule create/delete based on enabled flag", async () => {
|
||||
const schedules: any[] = [];
|
||||
const automationStore = {
|
||||
listSchedules: async () => schedules,
|
||||
createSchedule: async (input: any) => ({ ...input, id: "S-1" }),
|
||||
deleteSchedule: async () => true,
|
||||
updateSchedule: async () => undefined,
|
||||
} as any;
|
||||
|
||||
const created = await syncScheduledEvalBatchAutomation(automationStore, { taskEvaluationEnabled: true });
|
||||
expect(created?.name).toBe("Scheduled Task Evaluation");
|
||||
|
||||
schedules.push({ id: "S-1", name: "Scheduled Task Evaluation" });
|
||||
const deleted = await syncScheduledEvalBatchAutomation(automationStore, { taskEvaluationEnabled: false });
|
||||
expect(deleted).toBeUndefined();
|
||||
});
|
||||
|
||||
it("selects done tasks on first run and orders deterministically", async () => {
|
||||
const db = createDatabase("/tmp/fn-eval-automation-1", { inMemory: true });
|
||||
db.init();
|
||||
const evalStore = new EvalStore(db);
|
||||
const tasks = [
|
||||
task("FN-2", "done", "2026-05-01T01:00:00.000Z", "2026-01-02T00:00:00.000Z"),
|
||||
task("FN-1", "done", "2026-05-01T01:00:00.000Z", "2026-01-01T00:00:00.000Z"),
|
||||
task("FN-3", "done", "2026-05-01T02:00:00.000Z"),
|
||||
task("FN-4", "todo", "2026-05-01T03:00:00.000Z"),
|
||||
task("FN-5", "archived", "2026-05-01T04:00:00.000Z"),
|
||||
];
|
||||
|
||||
const result = await runScheduledEvalBatch({
|
||||
projectId: "proj",
|
||||
store: {
|
||||
listTasks: async () => tasks,
|
||||
getEvalStore: () => evalStore,
|
||||
} as any,
|
||||
startedAt: "2026-05-01T05:00:00.000Z",
|
||||
evaluator: async ({ task }) => ({ status: "scored", categoryScores: [], evidence: [], deterministicSignals: [], followUps: [], summary: task.id }),
|
||||
});
|
||||
|
||||
expect(result.status).toBe("completed");
|
||||
expect(result.selectedTaskIds).toEqual(["FN-1", "FN-2", "FN-3"]);
|
||||
|
||||
const run = evalStore.getRun(result.runId)!;
|
||||
expect(run.counts.totalTasks).toBe(3);
|
||||
expect(run.metadata?.windowEndInclusive).toBe("2026-05-01T05:00:00.000Z");
|
||||
const results = evalStore.listTaskResults({ runId: run.id });
|
||||
expect(results).toHaveLength(3);
|
||||
expect(results[0]?.metadata?.windowEndInclusive).toBe("2026-05-01T05:00:00.000Z");
|
||||
});
|
||||
|
||||
it("uses previous windowEndInclusive cursor for incremental selection", async () => {
|
||||
const db = createDatabase("/tmp/fn-eval-automation-2", { inMemory: true });
|
||||
db.init();
|
||||
const evalStore = new EvalStore(db);
|
||||
|
||||
evalStore.createRun({
|
||||
projectId: "proj",
|
||||
trigger: "schedule",
|
||||
scope: "completed-tasks",
|
||||
window: { until: "2026-05-01T05:00:00.000Z" },
|
||||
metadata: { windowEndInclusive: "2026-05-01T05:00:00.000Z" },
|
||||
});
|
||||
const run = evalStore.listRuns({ projectId: "proj", trigger: "schedule" })[0]!;
|
||||
evalStore.updateRun(run.id, { status: "completed", completedAt: "2026-05-01T05:05:00.000Z" });
|
||||
|
||||
const tasks = [
|
||||
task("FN-1", "done", "2026-05-01T05:00:00.000Z"),
|
||||
task("FN-2", "done", "2026-05-01T05:00:00.001Z"),
|
||||
task("FN-3", "done", "2026-05-01T06:00:00.000Z"),
|
||||
];
|
||||
|
||||
const result = await runScheduledEvalBatch({
|
||||
projectId: "proj",
|
||||
store: { listTasks: async () => tasks, getEvalStore: () => evalStore } as any,
|
||||
startedAt: "2026-05-01T06:00:00.000Z",
|
||||
evaluator: async () => ({ status: "skipped", categoryScores: [], evidence: [], deterministicSignals: [], followUps: [] }),
|
||||
});
|
||||
|
||||
expect(result.windowStartExclusive).toBe("2026-05-01T05:00:00.000Z");
|
||||
expect(result.selectedTaskIds).toEqual(["FN-2", "FN-3"]);
|
||||
});
|
||||
|
||||
it("completes no-op batch when no tasks are eligible", async () => {
|
||||
const db = createDatabase("/tmp/fn-eval-automation-3", { inMemory: true });
|
||||
db.init();
|
||||
const evalStore = new EvalStore(db);
|
||||
|
||||
const result = await runScheduledEvalBatch({
|
||||
projectId: "proj",
|
||||
store: { listTasks: async () => [task("FN-1", "todo", "2026-05-01T01:00:00.000Z")], getEvalStore: () => evalStore } as any,
|
||||
startedAt: "2026-05-02T01:00:00.000Z",
|
||||
evaluator: async () => ({ status: "scored", categoryScores: [], evidence: [], deterministicSignals: [], followUps: [] }),
|
||||
});
|
||||
|
||||
expect(result.tasksSelected).toBe(0);
|
||||
const run = evalStore.getRun(result.runId)!;
|
||||
expect(run.status).toBe("completed");
|
||||
expect(run.counts.totalTasks).toBe(0);
|
||||
});
|
||||
});
|
||||
335
packages/core/src/eval-automation.ts
Normal file
335
packages/core/src/eval-automation.ts
Normal file
@@ -0,0 +1,335 @@
|
||||
import type { AutomationStore } from "./automation-store.js";
|
||||
import type { ScheduledTask, ScheduledTaskCreateInput } from "./automation.js";
|
||||
import type { EvalRun, EvalTaskResultCreateInput } from "./eval-types.js";
|
||||
import { EvalLifecycleError } from "./eval-store.js";
|
||||
import type { ProjectSettings, Task } from "./types.js";
|
||||
|
||||
export const TASK_EVALUATION_SCHEDULE_NAME = "Scheduled Task Evaluation";
|
||||
export const DEFAULT_TASK_EVALUATION_SCHEDULE = "0 5 * * *";
|
||||
export const TASK_EVALUATION_SCHEDULE_COMMAND = "fn eval --scheduled-batch";
|
||||
|
||||
export interface ResolvedTaskEvaluationSettings {
|
||||
taskEvaluationEnabled: boolean;
|
||||
taskEvaluationSchedule: string;
|
||||
taskEvaluationProvider?: string;
|
||||
taskEvaluationModelId?: string;
|
||||
taskEvaluationFollowUpPolicy: "off" | "suggest" | "create";
|
||||
taskEvaluationRetention?: number;
|
||||
}
|
||||
|
||||
export function resolveTaskEvaluationSettings(
|
||||
settings: Partial<ProjectSettings>,
|
||||
): ResolvedTaskEvaluationSettings {
|
||||
return {
|
||||
taskEvaluationEnabled: settings.taskEvaluationEnabled ?? false,
|
||||
taskEvaluationSchedule: settings.taskEvaluationSchedule ?? DEFAULT_TASK_EVALUATION_SCHEDULE,
|
||||
taskEvaluationProvider: settings.taskEvaluationProvider,
|
||||
taskEvaluationModelId: settings.taskEvaluationModelId,
|
||||
taskEvaluationFollowUpPolicy: settings.taskEvaluationFollowUpPolicy ?? "off",
|
||||
taskEvaluationRetention: settings.taskEvaluationRetention,
|
||||
};
|
||||
}
|
||||
|
||||
export function createScheduledEvalBatchAutomation(
|
||||
settings: Partial<ProjectSettings>,
|
||||
): ScheduledTaskCreateInput {
|
||||
const resolved = resolveTaskEvaluationSettings(settings);
|
||||
return {
|
||||
name: TASK_EVALUATION_SCHEDULE_NAME,
|
||||
description: "Evaluates tasks completed since the previous scheduled evaluation batch",
|
||||
scheduleType: "custom",
|
||||
cronExpression: resolved.taskEvaluationSchedule,
|
||||
command: TASK_EVALUATION_SCHEDULE_COMMAND,
|
||||
enabled: true,
|
||||
scope: "project",
|
||||
};
|
||||
}
|
||||
|
||||
export async function syncScheduledEvalBatchAutomation(
|
||||
automationStore: AutomationStore,
|
||||
settings: Partial<ProjectSettings>,
|
||||
): Promise<ScheduledTask | undefined> {
|
||||
const { AutomationStore } = await import("./automation-store.js");
|
||||
const resolved = resolveTaskEvaluationSettings(settings);
|
||||
const schedules = await automationStore.listSchedules();
|
||||
const existing = schedules.find((s) => s.name === TASK_EVALUATION_SCHEDULE_NAME);
|
||||
|
||||
if (!resolved.taskEvaluationEnabled) {
|
||||
if (existing) await automationStore.deleteSchedule(existing.id);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (!AutomationStore.isValidCron(resolved.taskEvaluationSchedule)) {
|
||||
throw new Error(`Invalid task evaluation schedule: ${resolved.taskEvaluationSchedule}`);
|
||||
}
|
||||
|
||||
const input = createScheduledEvalBatchAutomation(settings);
|
||||
if (existing) {
|
||||
return automationStore.updateSchedule(existing.id, {
|
||||
scheduleType: "custom",
|
||||
cronExpression: input.cronExpression,
|
||||
command: input.command,
|
||||
enabled: true,
|
||||
scope: "project",
|
||||
});
|
||||
}
|
||||
|
||||
return automationStore.createSchedule(input);
|
||||
}
|
||||
|
||||
export interface EvalBatchWindow {
|
||||
windowStartExclusive?: string;
|
||||
windowEndInclusive: string;
|
||||
}
|
||||
|
||||
export interface CompletedTaskEvaluationContext {
|
||||
run: EvalRun;
|
||||
task: Task;
|
||||
taskIndex: number;
|
||||
totalTasks: number;
|
||||
window: EvalBatchWindow;
|
||||
}
|
||||
|
||||
export type CompletedTaskEvaluator = (
|
||||
context: CompletedTaskEvaluationContext,
|
||||
) => Promise<Omit<EvalTaskResultCreateInput, "taskId" | "taskSnapshot">>;
|
||||
|
||||
export interface EvalBatchTaskStore {
|
||||
listTasks(options?: { column?: string }): Promise<Task[]>;
|
||||
getEvalStore(): import("./eval-store.js").EvalStore;
|
||||
}
|
||||
|
||||
export interface RunScheduledEvalBatchParams {
|
||||
store: EvalBatchTaskStore;
|
||||
projectId: string;
|
||||
evaluator: CompletedTaskEvaluator;
|
||||
startedAt?: string;
|
||||
}
|
||||
|
||||
export interface ScheduledEvalBatchResult {
|
||||
runId: string;
|
||||
status: "completed" | "failed";
|
||||
windowStartExclusive?: string;
|
||||
windowEndInclusive: string;
|
||||
selectedTaskIds: string[];
|
||||
tasksSelected: number;
|
||||
}
|
||||
|
||||
export async function runScheduledEvalBatch(
|
||||
params: RunScheduledEvalBatchParams,
|
||||
): Promise<ScheduledEvalBatchResult> {
|
||||
const startedAt = params.startedAt ?? new Date().toISOString();
|
||||
const evalStore = params.store.getEvalStore();
|
||||
const priorRuns = evalStore
|
||||
.listRuns({ projectId: params.projectId, trigger: "schedule" })
|
||||
.filter((run) => run.status === "completed")
|
||||
.sort((a, b) => {
|
||||
const aWindowEnd = (a.metadata?.windowEndInclusive as string | undefined) ?? a.window.until ?? "";
|
||||
const bWindowEnd = (b.metadata?.windowEndInclusive as string | undefined) ?? b.window.until ?? "";
|
||||
if (aWindowEnd !== bWindowEnd) return aWindowEnd.localeCompare(bWindowEnd);
|
||||
return a.id.localeCompare(b.id);
|
||||
});
|
||||
|
||||
const previousScheduledBatch = priorRuns.at(-1);
|
||||
const windowStartExclusive =
|
||||
(previousScheduledBatch?.metadata?.windowEndInclusive as string | undefined) ??
|
||||
previousScheduledBatch?.window.until;
|
||||
const windowEndInclusive = startedAt;
|
||||
|
||||
let run: EvalRun;
|
||||
try {
|
||||
run = evalStore.createRun({
|
||||
projectId: params.projectId,
|
||||
trigger: "schedule",
|
||||
scope: "completed-tasks",
|
||||
window: {
|
||||
since: windowStartExclusive,
|
||||
until: windowEndInclusive,
|
||||
windowStartExclusive,
|
||||
windowEndInclusive,
|
||||
},
|
||||
metadata: {
|
||||
windowStartExclusive,
|
||||
windowEndInclusive,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof EvalLifecycleError && error.code === "active_run_conflict") {
|
||||
throw error;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
evalStore.appendRunEvent(run.id, {
|
||||
type: "info",
|
||||
message: "Scheduled eval batch started",
|
||||
status: "pending",
|
||||
metadata: { windowStartExclusive, windowEndInclusive },
|
||||
});
|
||||
|
||||
evalStore.updateRun(run.id, { status: "running", startedAt });
|
||||
|
||||
try {
|
||||
const doneTasks = (await params.store.listTasks({ column: "done" })).filter((task) =>
|
||||
task.column === "done"
|
||||
&& Boolean(task.executionCompletedAt)
|
||||
&& (!windowStartExclusive || task.executionCompletedAt! > windowStartExclusive)
|
||||
&& task.executionCompletedAt! <= windowEndInclusive,
|
||||
);
|
||||
|
||||
doneTasks.sort((a, b) => {
|
||||
const byCompletedAt = (a.executionCompletedAt ?? "").localeCompare(b.executionCompletedAt ?? "");
|
||||
if (byCompletedAt !== 0) return byCompletedAt;
|
||||
const byCreatedAt = a.createdAt.localeCompare(b.createdAt);
|
||||
if (byCreatedAt !== 0) return byCreatedAt;
|
||||
return a.id.localeCompare(b.id);
|
||||
});
|
||||
|
||||
const selectedTaskIds = doneTasks.map((task) => task.id);
|
||||
evalStore.updateRun(run.id, {
|
||||
counts: { totalTasks: selectedTaskIds.length, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 },
|
||||
metadata: {
|
||||
windowStartExclusive,
|
||||
windowEndInclusive,
|
||||
selectedTaskIds,
|
||||
tasksSelected: selectedTaskIds.length,
|
||||
},
|
||||
});
|
||||
|
||||
if (doneTasks.length === 0) {
|
||||
evalStore.appendRunEvent(run.id, {
|
||||
type: "info",
|
||||
status: "completed",
|
||||
message: "Scheduled eval batch completed with no newly done tasks",
|
||||
metadata: { tasksSelected: 0 },
|
||||
});
|
||||
evalStore.updateRun(run.id, {
|
||||
status: "completed",
|
||||
completedAt: new Date().toISOString(),
|
||||
summary: "No newly completed tasks found in evaluation window",
|
||||
});
|
||||
return {
|
||||
runId: run.id,
|
||||
status: "completed",
|
||||
windowStartExclusive,
|
||||
windowEndInclusive,
|
||||
selectedTaskIds: [],
|
||||
tasksSelected: 0,
|
||||
};
|
||||
}
|
||||
|
||||
let scoredTasks = 0;
|
||||
let skippedTasks = 0;
|
||||
let erroredTasks = 0;
|
||||
const evaluatedTaskIds: string[] = [];
|
||||
|
||||
for (const [index, task] of doneTasks.entries()) {
|
||||
try {
|
||||
const result = await params.evaluator({
|
||||
run,
|
||||
task,
|
||||
taskIndex: index,
|
||||
totalTasks: doneTasks.length,
|
||||
window: { windowStartExclusive, windowEndInclusive },
|
||||
});
|
||||
|
||||
evalStore.createTaskResult(run.id, {
|
||||
...result,
|
||||
taskId: task.id,
|
||||
taskSnapshot: {
|
||||
taskId: task.id,
|
||||
title: task.title,
|
||||
column: task.column,
|
||||
createdAt: task.createdAt,
|
||||
updatedAt: task.updatedAt,
|
||||
executionCompletedAt: task.executionCompletedAt,
|
||||
summary: task.summary,
|
||||
},
|
||||
metadata: {
|
||||
...(result.metadata ?? {}),
|
||||
windowEndInclusive,
|
||||
},
|
||||
});
|
||||
|
||||
evaluatedTaskIds.push(task.id);
|
||||
if (result.status === "scored") scoredTasks += 1;
|
||||
else if (result.status === "skipped") skippedTasks += 1;
|
||||
else erroredTasks += 1;
|
||||
|
||||
evalStore.appendRunEvent(run.id, {
|
||||
type: "task_evaluated",
|
||||
message: `Evaluated task ${task.id}`,
|
||||
taskId: task.id,
|
||||
metadata: { status: result.status },
|
||||
});
|
||||
} catch (error) {
|
||||
erroredTasks += 1;
|
||||
evalStore.appendRunEvent(run.id, {
|
||||
type: "error",
|
||||
message: `Failed evaluating task ${task.id}`,
|
||||
taskId: task.id,
|
||||
metadata: { error: error instanceof Error ? error.message : String(error) },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
evalStore.updateRun(run.id, {
|
||||
status: "completed",
|
||||
evaluatedTaskIds,
|
||||
counts: {
|
||||
totalTasks: doneTasks.length,
|
||||
scoredTasks,
|
||||
skippedTasks,
|
||||
erroredTasks,
|
||||
},
|
||||
completedAt: new Date().toISOString(),
|
||||
summary: `Scheduled eval batch completed for ${doneTasks.length} task(s)`,
|
||||
metadata: {
|
||||
windowStartExclusive,
|
||||
windowEndInclusive,
|
||||
selectedTaskIds,
|
||||
tasksSelected: selectedTaskIds.length,
|
||||
},
|
||||
});
|
||||
|
||||
evalStore.appendRunEvent(run.id, {
|
||||
type: "status_changed",
|
||||
status: "completed",
|
||||
message: `Scheduled eval batch completed (${doneTasks.length} tasks selected)`,
|
||||
metadata: { scoredTasks, skippedTasks, erroredTasks },
|
||||
});
|
||||
|
||||
return {
|
||||
runId: run.id,
|
||||
status: "completed",
|
||||
windowStartExclusive,
|
||||
windowEndInclusive,
|
||||
selectedTaskIds,
|
||||
tasksSelected: selectedTaskIds.length,
|
||||
};
|
||||
} catch (error) {
|
||||
evalStore.updateRun(run.id, {
|
||||
status: "failed",
|
||||
completedAt: new Date().toISOString(),
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
metadata: {
|
||||
windowStartExclusive,
|
||||
windowEndInclusive,
|
||||
},
|
||||
});
|
||||
evalStore.appendRunEvent(run.id, {
|
||||
type: "error",
|
||||
status: "failed",
|
||||
message: "Scheduled eval batch failed",
|
||||
metadata: { error: error instanceof Error ? error.message : String(error) },
|
||||
});
|
||||
return {
|
||||
runId: run.id,
|
||||
status: "failed",
|
||||
windowStartExclusive,
|
||||
windowEndInclusive,
|
||||
selectedTaskIds: [],
|
||||
tasksSelected: 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user