feat(FN-3389): fix scheduled evaluator integration types

Completes typing for the scheduled evaluator integration in the cron runner and project engine, with corresponding test updates in the evaluator test file.

Fusion-Task-Id: FN-3389

packages/engine/src/__tests__/cron-runner.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
-import { CronRunner, createAiPromptExecutor, isInProcessBackupCommand } from "../cron-runner.js";
+import { CronRunner, createAiPromptExecutor, isInProcessBackupCommand, isInProcessScheduledEvalCommand } from "../cron-runner.js";
 import type { AiPromptExecutor } from "../cron-runner.js";
 import type { TaskStore, AutomationStore, ScheduledTask, AutomationRunResult, AutomationStep, Settings } from "@fusion/core";
 import { randomUUID } from "node:crypto";
@@ -1842,4 +1842,17 @@ describe("CronRunner", () => {
       });
     }
   });
+
+  describe("isInProcessScheduledEvalCommand", () => {
+    it("matches canonical scheduled eval command", () => {
+      expect(isInProcessScheduledEvalCommand("fn eval --scheduled-batch")).toBe(true);
+      expect(isInProcessScheduledEvalCommand("fusion eval --scheduled-batch --flag")).toBe(true);
+    });
+
+    it("rejects non-canonical and shell-metachar commands", () => {
+      expect(isInProcessScheduledEvalCommand("fn eval")).toBe(false);
+      expect(isInProcessScheduledEvalCommand("fn eval --scheduled-batch && echo x")).toBe(false);
+      expect(isInProcessScheduledEvalCommand("echo fn eval --scheduled-batch")).toBe(false);
+    });
+  });
 });

packages/engine/src/__tests__/evaluator.test.ts

@@ -0,0 +1,105 @@
+import { describe, expect, it } from "vitest";
+import { createDatabase, EvalStore, runScheduledEvalBatch, type TaskDetail } from "@fusion/core";
+import { HybridEvaluatorService, buildEvaluationPrompt, parseAiResponse, resolveEvaluatorModel } from "../evaluator.js";
+
+function makeTask(overrides: Partial<TaskDetail> = {}): TaskDetail {
+  return {
+    id: "FN-101",
+    description: "desc",
+    column: "done",
+    dependencies: [],
+    steps: [],
+    currentStep: 0,
+    log: [{ timestamp: "1", action: "[timing] build in 50ms" }],
+    createdAt: "2026-05-01T00:00:00.000Z",
+    updatedAt: "2026-05-01T01:00:00.000Z",
+    prompt: "prompt",
+    ...overrides,
+  } as TaskDetail;
+}
+
+describe("evaluator", () => {
+  it("resolves explicit complete model override before validator lane", () => {
+    expect(resolveEvaluatorModel({ validatorProvider: "anthropic", validatorModelId: "claude" }, { provider: "openai", modelId: "gpt-4o" }))
+      .toEqual({ provider: "openai", modelId: "gpt-4o" });
+  });
+
+  it("ignores partial override and falls back to validator lane", () => {
+    expect(resolveEvaluatorModel({ validatorProvider: "anthropic", validatorModelId: "claude" }, { provider: "openai" }))
+      .toEqual({ provider: "anthropic", modelId: "claude" });
+  });
+
+  it("parses strict AI JSON response", () => {
+    const parsed = parseAiResponse('{"overallScore":0.8,"categoryScores":{"quality":0.8},"rationale":"ok","evidence":[],"followUpDrafts":[]}');
+    expect(parsed.overallScore).toBe(0.8);
+    expect(parsed.rationale).toBe("ok");
+  });
+
+  it("throws on malformed AI JSON response", () => {
+    expect(() => parseAiResponse("not-json")).toThrow(/not valid JSON/);
+  });
+
+  it("builds prompt with deterministic signal bundle", () => {
+    const task = makeTask();
+    const prompt = buildEvaluationPrompt(task, { runId: "ER-1", startedAt: "2026-05-02T00:00:00.000Z" }, {
+      taskId: task.id,
+      column: "done",
+      workflowSummary: { total: 0, passed: 0, failed: 0, pending: 0 },
+      commitSummary: { commitCount: 0 },
+      logSummary: { errorCount: 0, warningCount: 0, timingEntries: 1 },
+      evidence: [],
+    });
+    expect(prompt).toContain("Deterministic signals");
+    expect(prompt).toContain("ER-1");
+  });
+
+  it("returns merged evaluation payload shape for persistence", async () => {
+    const service = new HybridEvaluatorService({
+      cwd: process.cwd(),
+      runPrompt: async () => '{"overallScore":0.9,"categoryScores":{"quality":0.9},"rationale":"Great","evidence":[{"kind":"task","label":"done"}],"followUpDrafts":[{"title":"Add tests","description":"More tests","reason":"coverage","evidenceRefs":["task:done"]}]}'
+    });
+    const result = await service.evaluateTask(makeTask(), { runId: "ER-1", startedAt: "2026-05-01T00:00:00.000Z" }, {});
+    expect(result.status).toBe("scored");
+    expect(result.overallScore).toBe(0.9);
+    expect(result.categoryScores?.[0]?.category).toBe("quality");
+    expect(result.followUps?.[0]?.title).toBe("Add tests");
+    expect((result.metadata as any).hybridEvaluation).toBeDefined();
+  });
+
+  it("integrates scheduled batch with evaluator and persists one result per run/task", async () => {
+    const db = createDatabase("/tmp/fn-evaluator-integration", { inMemory: true });
+    db.init();
+    const evalStore = new EvalStore(db);
+    const doneTask = makeTask({ executionCompletedAt: "2026-05-01T00:04:00.000Z" });
+
+    const service = new HybridEvaluatorService({
+      cwd: process.cwd(),
+      runPrompt: async () => '{"overallScore":0.7,"categoryScores":{"quality":0.7},"rationale":"Solid","evidence":[],"followUpDrafts":[]}'
+    });
+
+    const mockStore = {
+      listTasks: async () => [doneTask],
+      getEvalStore: () => evalStore,
+    };
+
+    const run1 = await runScheduledEvalBatch({
+      store: mockStore,
+      projectId: "proj-1",
+      startedAt: "2026-05-02T00:00:00.000Z",
+      evaluator: async ({ task, run }) => service.evaluateTask(task as TaskDetail, { runId: run.id, startedAt: run.startedAt ?? "" }, {}),
+    });
+
+    const run2 = await runScheduledEvalBatch({
+      store: mockStore,
+      projectId: "proj-1",
+      startedAt: "2026-05-02T00:00:00.000Z",
+      evaluator: async ({ task, run }) => service.evaluateTask(task as TaskDetail, { runId: run.id, startedAt: run.startedAt ?? "" }, {}),
+    });
+
+    expect(run1.status).toBe("completed");
+    expect(run2.tasksSelected).toBe(0);
+    const all = evalStore.listTaskResults({ taskId: doneTask.id });
+    expect(all).toHaveLength(1);
+    expect(all[0]?.overallScore).toBe(0.7);
+  });
+});

packages/engine/src/__tests__/project-engine.test.ts

@@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({
   syncInsightExtractionAutomation: vi.fn(),
   syncAutoSummarizeAutomation: vi.fn(),
   syncMemoryDreamsAutomation: vi.fn(),
+  syncScheduledEvalBatchAutomation: vi.fn(),
   automationStoreInit: vi.fn(async () => undefined),
   createAiPromptExecutor: vi.fn(async () => vi.fn()),
   cronRunnerStart: vi.fn(),
@@ -39,6 +40,7 @@ vi.mock("@fusion/core", async (importOriginal) => {
     syncInsightExtractionAutomation: mocks.syncInsightExtractionAutomation,
     syncAutoSummarizeAutomation: mocks.syncAutoSummarizeAutomation,
     syncMemoryDreamsAutomation: mocks.syncMemoryDreamsAutomation,
+    syncScheduledEvalBatchAutomation: mocks.syncScheduledEvalBatchAutomation,
   });
 });
 
@@ -340,12 +342,15 @@ describe("ProjectEngine auto-summarize wiring", () => {
     expect(mocks.syncInsightExtractionAutomation).toHaveBeenCalledTimes(1);
     expect(mocks.syncAutoSummarizeAutomation).toHaveBeenCalledTimes(1);
     expect(mocks.syncMemoryDreamsAutomation).toHaveBeenCalledTimes(1);
+    expect(mocks.syncScheduledEvalBatchAutomation).toHaveBeenCalledTimes(1);
 
     const insightSettings = mocks.syncInsightExtractionAutomation.mock.calls[0][1];
     const autoSummarizeSettings = mocks.syncAutoSummarizeAutomation.mock.calls[0][1];
     const memoryDreamsSettings = mocks.syncMemoryDreamsAutomation.mock.calls[0][1];
+    const scheduledEvalSettings = mocks.syncScheduledEvalBatchAutomation.mock.calls[0][1];
     expect(autoSummarizeSettings).toBe(insightSettings);
     expect(memoryDreamsSettings).toBe(insightSettings);
+    expect(scheduledEvalSettings).toBe(insightSettings);
 
     const cronRunnerStartOrder = mocks.cronRunnerStart.mock.invocationCallOrder[0];
     expect(mocks.syncInsightExtractionAutomation.mock.invocationCallOrder[0]).toBeLessThan(
@@ -357,6 +362,9 @@ describe("ProjectEngine auto-summarize wiring", () => {
     expect(mocks.syncMemoryDreamsAutomation.mock.invocationCallOrder[0]).toBeLessThan(
       cronRunnerStartOrder,
     );
+    expect(mocks.syncScheduledEvalBatchAutomation.mock.invocationCallOrder[0]).toBeLessThan(
+      cronRunnerStartOrder,
+    );
 
     await engine.stop();
   });

packages/engine/src/cron-runner.ts

@@ -2,6 +2,8 @@ import { exec } from "node:child_process";
 
 import {
   resolveProjectDefaultModel,
+  runScheduledEvalBatch,
+  resolveTaskEvaluationSettings,
   type TaskStore,
   type AutomationStore,
   type ScheduledTask,
@@ -10,10 +12,12 @@ import {
   type AutomationStepResult,
   type Column,
   type TaskCreateInput,
+  type TaskDetail,
 } from "@fusion/core";
 import { createLogger } from "./logger.js";
 import { defaultShell } from "./shell-utils.js";
 import { createFnAgent, promptWithFallback } from "./pi.js";
+import { HybridEvaluatorService } from "./evaluator.js";
 
 const log = createLogger("cron-runner");
 
@@ -128,6 +132,18 @@ export function isInProcessBackupCommand(command: string | undefined): boolean {
   return true;
 }
 
+export function isInProcessScheduledEvalCommand(command: string | undefined): boolean {
+  if (!command) return false;
+  const trimmed = command.trim();
+  if (!trimmed || SHELL_METACHARACTERS_REGEX.test(trimmed)) return false;
+  const tokens = trimmed.split(/\s+/).map((tok) => tok.toLowerCase());
+  return tokens.length >= 3
+    && FUSION_BINARY_TOKENS.has(tokens[0] ?? "")
+    && tokens[1] === "eval"
+    && tokens[2] === "--scheduled-batch"
+    && tokens.slice(3).every((tok) => tok.startsWith("-"));
+}
+
 /** Default execution timeout: 5 minutes. */
 const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000;
 /** Maximum output buffer: 1 MB. */
@@ -150,6 +166,10 @@ export type AiPromptExecutor = (
 ) => Promise<string>;
 
 export interface CronRunnerOptions {
+  /** Project working directory used for in-process evaluator sessions */
+  workingDirectory?: string;
+  /** Project id for scheduled eval batch persistence */
+  projectId?: string;
   /** Polling interval in milliseconds. Default: 60000 (60s). Minimum: 10000 (10s). */
   pollIntervalMs?: number;
   /** Optional AI prompt executor. When not provided, ai-prompt steps return a configuration error. */
@@ -391,6 +411,10 @@ export class CronRunner {
       return this.executeBackupInProcess(schedule, startedAt);
     }
 
+    if (isInProcessScheduledEvalCommand(schedule.command)) {
+      return this.executeScheduledEvalInProcess(schedule, startedAt);
+    }
+
     try {
       const timeoutMs = schedule.timeoutMs ?? DEFAULT_TIMEOUT_MS;
       const { stdout, stderr } = await execCommand(schedule.command, {
@@ -458,6 +482,39 @@ export class CronRunner {
    * and the command-step path. Returns the success/output/error tuple in
    * a shape that callers can wrap into either a run or a step result.
    */
+  private async executeScheduledEvalInProcess(
+    schedule: ScheduledTask,
+    startedAt: string,
+  ): Promise<AutomationRunResult> {
+    const settings = await this.store.getSettings();
+    const evalSettings = resolveTaskEvaluationSettings(settings);
+    const evaluator = new HybridEvaluatorService({ cwd: this.options.workingDirectory ?? process.cwd() });
+
+    const result = await runScheduledEvalBatch({
+      store: this.store,
+      projectId: this.options.projectId ?? "default-project",
+      startedAt,
+      evaluator: async ({ task, run }) => {
+        const taskDetail = await this.store.getTask(task.id);
+        if (!taskDetail) {
+          throw new Error(`Task not found for evaluation: ${task.id}`);
+        }
+        return evaluator.evaluateTask(taskDetail as TaskDetail, { runId: run.id, startedAt: run.startedAt ?? startedAt }, settings, {
+          provider: evalSettings.taskEvaluationProvider,
+          modelId: evalSettings.taskEvaluationModelId,
+        });
+      },
+    });
+
+    return {
+      success: result.status === "completed",
+      output: JSON.stringify(result),
+      error: result.status === "failed" ? "Scheduled eval batch failed" : undefined,
+      startedAt,
+      completedAt: new Date().toISOString(),
+    };
+  }
+
   private async runBackupActionInProcess(): Promise<{
     success: boolean;
     output: string;

packages/engine/src/evaluator.ts

@@ -0,0 +1,201 @@
+import {
+  collectDeterministicSignals,
+  resolveValidatorSettingsModel,
+  type DeterministicSignals,
+  type EvalTaskResultCreateInput,
+  type EvaluationEvidenceRef,
+  type FollowUpDraft,
+  type Settings,
+  type TaskDetail,
+} from "@fusion/core";
+import { createFnAgent, promptWithFallback } from "./pi.js";
+import { createLogger } from "./logger.js";
+
+const log = createLogger("evaluator");
+
+export interface EvalRunContext {
+  runId: string;
+  startedAt: string;
+}
+
+export interface EvaluatorModelOverride {
+  provider?: string;
+  modelId?: string;
+}
+
+export interface EvaluatorDeps {
+  cwd: string;
+  runPrompt?: (prompt: string, provider?: string, modelId?: string) => Promise<string>;
+}
+
+interface EvaluatorAiResponse {
+  overallScore: number;
+  categoryScores: Record<string, number>;
+  rationale: string;
+  evidence: EvaluationEvidenceRef[];
+  followUpDrafts: FollowUpDraft[];
+}
+
+export function resolveEvaluatorModel(
+  settings: Partial<Settings>,
+  override?: EvaluatorModelOverride,
+): { provider?: string; modelId?: string } {
+  if (override?.provider && override?.modelId) {
+    return { provider: override.provider, modelId: override.modelId };
+  }
+  // Temporary fallback until FN-3393 introduces dedicated evaluator settings.
+  return resolveValidatorSettingsModel(settings);
+}
+
+export class HybridEvaluatorService {
+  constructor(private readonly deps: EvaluatorDeps) {}
+
+  async evaluateTask(
+    task: TaskDetail,
+    run: EvalRunContext,
+    settings: Partial<Settings>,
+    modelOverride?: EvaluatorModelOverride,
+  ): Promise<Omit<EvalTaskResultCreateInput, "taskId" | "taskSnapshot">> {
+    const deterministicSignals = collectDeterministicSignals(task, run);
+    const model = resolveEvaluatorModel(settings, modelOverride);
+    const prompt = buildEvaluationPrompt(task, run, deterministicSignals);
+    const responseText = await this.runPrompt(prompt, model.provider, model.modelId);
+    const ai = parseAiResponse(responseText);
+
+    return {
+      status: "scored",
+      overallScore: ai.overallScore,
+      categoryScores: Object.entries(ai.categoryScores).map(([category, score]) => ({ category, score })),
+      rationale: ai.rationale,
+      summary: ai.rationale,
+      evidence: ai.evidence.map((ev) => ({ type: "other", ref: `${ev.kind}:${ev.label}`, excerpt: ev.value })),
+      deterministicSignals: deterministicSignalsToEvalSignals(deterministicSignals),
+      followUps: ai.followUpDrafts.map((draft) => ({
+        title: draft.title,
+        description: draft.description,
+        metadata: { reason: draft.reason, evidenceRefs: draft.evidenceRefs },
+      })),
+      metadata: {
+        runId: run.runId,
+        evaluatorModel: model,
+        evaluatorRationale: ai.rationale,
+        hybridEvaluation: {
+          deterministicSignals,
+          ai,
+        },
+      },
+    };
+  }
+
+  private async runPrompt(prompt: string, provider?: string, modelId?: string): Promise<string> {
+    if (this.deps.runPrompt) {
+      return this.deps.runPrompt(prompt, provider, modelId);
+    }
+
+    let text = "";
+    const { session } = await createFnAgent({
+      cwd: this.deps.cwd,
+      systemPrompt: "You are a strict evaluator. Reply with JSON only.",
+      tools: "readonly",
+      defaultProvider: provider,
+      defaultModelId: modelId,
+      onText: (delta) => {
+        text += delta;
+      },
+    });
+
+    try {
+      await promptWithFallback(session, prompt);
+      return text;
+    } finally {
+      try {
+        session.dispose();
+      } catch (error) {
+        log.warn(`Evaluator session disposal failed: ${error instanceof Error ? error.message : String(error)}`);
+      }
+    }
+  }
+}
+
+function deterministicSignalsToEvalSignals(signals: DeterministicSignals): Array<{ signalId: string; kind: string; name: string; value?: string | number; passed?: boolean }> {
+  return [
+    {
+      signalId: "workflow-summary",
+      kind: "workflow",
+      name: "workflow-summary",
+      value: `${signals.workflowSummary.passed}/${signals.workflowSummary.total}`,
+      passed: signals.workflowSummary.failed === 0,
+    },
+    {
+      signalId: "timing-ms",
+      kind: "timing",
+      name: "timed-execution-ms",
+      value: signals.timedExecutionMs,
+    },
+    {
+      signalId: "commit-count",
+      kind: "commit",
+      name: "commit-count",
+      value: signals.commitSummary.commitCount,
+    },
+  ];
+}
+
+export function buildEvaluationPrompt(task: TaskDetail, run: EvalRunContext, deterministicSignals: DeterministicSignals): string {
+  return [
+    "Evaluate the completed task and respond with strict JSON.",
+    `Run: ${run.runId}`,
+    "Schema:",
+    JSON.stringify({
+      overallScore: 0,
+      categoryScores: { quality: 0, reliability: 0, testing: 0 },
+      rationale: "",
+      evidence: [{ kind: "task", label: "", value: "", source: "" }],
+      followUpDrafts: [{ title: "", description: "", reason: "", evidenceRefs: [] }],
+    }, null, 2),
+    "Task:",
+    JSON.stringify({
+      id: task.id,
+      title: task.title,
+      column: task.column,
+      status: task.status,
+      summary: task.summary,
+    }, null, 2),
+    "Deterministic signals:",
+    JSON.stringify(deterministicSignals, null, 2),
+  ].join("\n\n");
+}
+
+export function parseAiResponse(raw: string): EvaluatorAiResponse {
+  const candidate = extractJson(raw);
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(candidate);
+  } catch (error) {
+    throw new Error(`Evaluator AI response was not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
+  }
+
+  const record = parsed as Partial<EvaluatorAiResponse>;
+  if (typeof record.overallScore !== "number") throw new Error("Evaluator response missing numeric overallScore");
+  if (!record.categoryScores || typeof record.categoryScores !== "object") throw new Error("Evaluator response missing categoryScores");
+  if (typeof record.rationale !== "string" || !record.rationale.trim()) throw new Error("Evaluator response missing rationale");
+
+  return {
+    overallScore: record.overallScore,
+    categoryScores: record.categoryScores as Record<string, number>,
+    rationale: record.rationale,
+    evidence: Array.isArray(record.evidence) ? record.evidence : [],
+    followUpDrafts: Array.isArray(record.followUpDrafts) ? record.followUpDrafts : [],
+  };
+}
+
+function extractJson(raw: string): string {
+  const trimmed = raw.trim();
+  if (trimmed.startsWith("```")) {
+    return trimmed.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, "").trim();
+  }
+  const first = trimmed.indexOf("{");
+  const last = trimmed.lastIndexOf("}");
+  if (first >= 0 && last > first) return trimmed.slice(first, last + 1);
+  return trimmed;
+}

packages/engine/src/project-engine.ts

@@ -320,6 +320,8 @@ export class ProjectEngine {
       this.cronRunner = new CronRunner(store, this.automationStore, {
         aiPromptExecutor,
         onScheduleRunProcessed: this.buildInsightRunHandler(cwd),
+        workingDirectory: cwd,
+        projectId: this.config.projectId,
         scope: "project", // Project-scoped execution — global schedules run separately
       });
 
@@ -365,6 +367,19 @@ export class ProjectEngine {
         runtimeLog.warn("syncMemoryDreamsAutomation is unavailable; skipping startup sync");
       }
 
+      // Sync scheduled eval batch automation on startup
+      if (typeof coreAutomationModule.syncScheduledEvalBatchAutomation === "function") {
+        try {
+          await coreAutomationModule.syncScheduledEvalBatchAutomation(this.automationStore, settings);
+        } catch (err) {
+          const { message, detail } = formatErrorDetails(err);
+          startupSyncFailures.push(`scheduled eval: ${message}`);
+          runtimeLog.warn(`Scheduled eval automation startup sync failed:\n${detail}`);
+        }
+      } else {
+        runtimeLog.warn("syncScheduledEvalBatchAutomation is unavailable; skipping startup sync");
+      }
+
       this.cronRunner.start();
 
       if (startupSyncFailures.length > 0) {
@@ -2101,6 +2116,40 @@ export class ProjectEngine {
     };
     store.on("settings:updated", onAutoSummarizeSettingsChange);
     this.settingsHandlers.push(onAutoSummarizeSettingsChange);
+
+    // 7. Scheduled eval settings change — sync automation
+    const onScheduledEvalSettingsChange = async ({
+      settings: s,
+      previous: prev,
+    }: {
+      settings: Settings;
+      previous: Settings;
+    }) => {
+      const evalKeys = [
+        "taskEvaluationEnabled",
+        "taskEvaluationSchedule",
+      ] as const;
+
+      const changed = evalKeys.some((key) => (s as any)[key] !== (prev as any)[key]);
+      if (!changed || !this.automationStore) return;
+
+      try {
+        const { syncScheduledEvalBatchAutomation } = await import("@fusion/core");
+        if (typeof syncScheduledEvalBatchAutomation === "function") {
+          await syncScheduledEvalBatchAutomation(this.automationStore, s);
+          runtimeLog.log("Scheduled eval automation synced with settings");
+        }
+      } catch (err) {
+        const { message, detail } = formatErrorDetails(err);
+        this.setAutomationSubsystemHealth(
+          "degraded",
+          `Failed to sync scheduled eval automation: ${message}`,
+        );
+        runtimeLog.warn(`Failed to sync scheduled eval automation:\n${detail}`);
+      }
+    };
+    store.on("settings:updated", onScheduledEvalSettingsChange);
+    this.settingsHandlers.push(onScheduledEvalSettingsChange);
   }
 
   /**