feat(FN-3391): persist evaluator evidence with score categories

- Add eval score category types and exports in core with store support and coverage - Implement engine evaluator evidence extraction and persistence with dedicated tests - Update evaluator flow and cron wiring to record evidence alongside eval runs - Refresh architecture, storage, and eval docs for evidence and categorization behavior Fusion-Task-Id: FN-3391
2026-05-06 13:33:58 -07:00
parent 91f187f9ff
commit 2449472a63
13 changed files with 926 additions and 12 deletions
--- a/packages/core/src/tests/eval-store.test.ts
+++ b/packages/core/src/tests/eval-store.test.ts
@@ -1,6 +1,7 @@
 import { beforeEach, describe, expect, it } from "vitest";
 import { createDatabase, type Database } from "../db.js";
 import { EvalLifecycleError, EvalStore } from "../eval-store.js";
+import { EVIDENCE_EXCERPT_TRUNCATION_MARKER, EVIDENCE_LIMITS, TASK_EVALUATION_EVIDENCE_SOURCE_ORDER } from "../eval-types.js";

 let db: Database;
 let store: EvalStore;
@@ -127,6 +128,145 @@ describe("EvalStore", () => {
    expect(second.id).toBe(first.id);
  });

+  it("persists evidence bundles via metadata and preserves stable source ordering", () => {
+    const run = store.createRun({ projectId: "p1", scope: "window" });
+    const created = store.createTaskResult(run.id, {
+      taskId: "FN-evidence",
+      taskSnapshot: { taskId: "FN-evidence", title: "Evidence task" },
+      status: "scored",
+      evidenceBundle: {
+        taskId: "FN-evidence",
+        runId: run.id,
+        sourceOrder: [...TASK_EVALUATION_EVIDENCE_SOURCE_ORDER],
+        taskMetadata: [{ id: "tm-1", source: "taskMetadata", label: "task snapshot", taskId: "FN-evidence", runId: run.id }],
+        commits: [{ id: "c-1", source: "commits", label: "commit", sha: "abc123", taskId: "FN-evidence", runId: run.id }],
+        workflow: [],
+        reviews: [],
+        documents: [],
+        taskActivity: [],
+        agentLogs: [],
+        runAudit: [],
+      },
+    });
+
+    const fetched = store.getTaskResult(created.id);
+    expect(fetched?.evidenceBundle?.sourceOrder).toEqual(TASK_EVALUATION_EVIDENCE_SOURCE_ORDER);
+    expect(fetched?.evidenceBundle?.taskMetadata[0]?.id).toBe("tm-1");
+    expect(fetched?.metadata?.__taskEvaluationEvidenceBundle).toBeDefined();
+  });
+
+  it("rejects evidence bundles that exceed per-source limits", () => {
+    const run = store.createRun({ projectId: "p1", scope: "window" });
+    expect(() => store.createTaskResult(run.id, {
+      taskId: "FN-over-limit",
+      taskSnapshot: { taskId: "FN-over-limit" },
+      status: "scored",
+      evidenceBundle: {
+        taskId: "FN-over-limit",
+        runId: run.id,
+        sourceOrder: [...TASK_EVALUATION_EVIDENCE_SOURCE_ORDER],
+        taskMetadata: [],
+        commits: Array.from({ length: EVIDENCE_LIMITS.commits + 1 }, (_, i) => ({
+          id: `c-${i}`,
+          source: "commits" as const,
+          label: `commit ${i}`,
+          sha: `${i}`,
+          taskId: "FN-over-limit",
+          runId: run.id,
+        })),
+        workflow: [],
+        reviews: [],
+        documents: [],
+        taskActivity: [],
+        agentLogs: [],
+        runAudit: [],
+      },
+    })).toThrow(/commits exceeds limit/);
+  });
+
+  it("truncates overlong evidence excerpts to bounded persisted size", () => {
+    const run = store.createRun({ projectId: "p1", scope: "window" });
+    const result = store.createTaskResult(run.id, {
+      taskId: "FN-truncate",
+      taskSnapshot: { taskId: "FN-truncate" },
+      status: "scored",
+      evidenceBundle: {
+        taskId: "FN-truncate",
+        runId: run.id,
+        sourceOrder: [...TASK_EVALUATION_EVIDENCE_SOURCE_ORDER],
+        taskMetadata: [{
+          id: "tm-1",
+          source: "taskMetadata",
+          label: "summary",
+          taskId: "FN-truncate",
+          runId: run.id,
+          excerpt: "x".repeat(800),
+        }],
+        commits: [],
+        workflow: [],
+        reviews: [],
+        documents: [],
+        taskActivity: [],
+        agentLogs: [],
+        runAudit: [],
+      },
+    });
+
+    const fetched = store.getTaskResult(result.id);
+    const excerpt = fetched?.evidenceBundle?.taskMetadata[0]?.excerpt ?? "";
+    expect(excerpt.length).toBeLessThanOrEqual(500);
+    expect(excerpt.endsWith(EVIDENCE_EXCERPT_TRUNCATION_MARKER)).toBe(true);
+    expect(fetched?.evidenceBundle?.taskMetadata[0]?.truncated).toBe(true);
+  });
+
+  it("rejects evidence bundles with incorrect sourceOrder", () => {
+    const run = store.createRun({ projectId: "p1", scope: "window" });
+    expect(() => store.createTaskResult(run.id, {
+      taskId: "FN-wrong-order",
+      taskSnapshot: { taskId: "FN-wrong-order" },
+      status: "scored",
+      evidenceBundle: {
+        taskId: "FN-wrong-order",
+        runId: run.id,
+        sourceOrder: ["commits", "taskMetadata", "workflow", "reviews", "documents", "taskActivity", "agentLogs", "runAudit"],
+        taskMetadata: [],
+        commits: [],
+        workflow: [],
+        reviews: [],
+        documents: [],
+        taskActivity: [],
+        agentLogs: [],
+        runAudit: [],
+      },
+    })).toThrow(/sourceOrder must match/);
+  });
+
+  it("round-trips optional empty evidence source groups", () => {
+    const run = store.createRun({ projectId: "p1", scope: "window" });
+    const result = store.createTaskResult(run.id, {
+      taskId: "FN-empty-sources",
+      taskSnapshot: { taskId: "FN-empty-sources" },
+      status: "scored",
+      evidenceBundle: {
+        taskId: "FN-empty-sources",
+        runId: run.id,
+        sourceOrder: [...TASK_EVALUATION_EVIDENCE_SOURCE_ORDER],
+        taskMetadata: [],
+        commits: [],
+        workflow: [],
+        reviews: [],
+        documents: [],
+        taskActivity: [],
+        agentLogs: [],
+        runAudit: [],
+      },
+    });
+
+    const fetched = store.getTaskResult(result.id);
+    expect(fetched?.evidenceBundle?.commits).toEqual([]);
+    expect(fetched?.evidenceBundle?.runAudit).toEqual([]);
+  });
+
  it("appends run events with sequential ordering", () => {
    const run = store.createRun({ projectId: "p1", scope: "window" });
    const evt1 = store.appendRunEvent(run.id, { type: "info", message: "started" });
--- a/packages/core/src/eval-store.ts
+++ b/packages/core/src/eval-store.ts
@@ -2,6 +2,12 @@ import { EventEmitter } from "node:events";
 import { randomUUID } from "node:crypto";
 import type { Database } from "./db.js";
 import { fromJson, toJson, toJsonNullable } from "./db.js";
+import {
+  EVIDENCE_EXCERPT_TRUNCATION_MARKER,
+  EVIDENCE_LIMITS,
+  MAX_EVIDENCE_EXCERPT_LENGTH,
+  TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
+} from "./eval-types.js";
 import type {
  EvalRun,
  EvalRunCreateInput,
@@ -14,10 +20,13 @@ import type {
  EvalTaskResultCreateInput,
  EvalTaskResultListOptions,
  EvalTaskResultUpdateInput,
+  TaskEvaluationEvidenceBundle,
+  TaskEvidenceEntryBase,
 } from "./eval-types.js";

 const TERMINAL_STATUSES = new Set<EvalRunStatus>(["completed", "failed", "cancelled"]);
 const ACTIVE_STATUSES = new Set<EvalRunStatus>(["pending", "running"]);
+const EVIDENCE_BUNDLE_METADATA_KEY = "__taskEvaluationEvidenceBundle";
 const VALID_TRANSITIONS: Record<EvalRunStatus, EvalRunStatus[]> = {
  pending: ["running", "completed", "failed", "cancelled"],
  running: ["completed", "failed", "cancelled"],
@@ -45,6 +54,77 @@ function generateEventId(): string {
  return `ERE-${randomUUID()}`;
 }

+function withEvidenceBundleMetadata(
+  metadata: Record<string, unknown> | undefined,
+  evidenceBundle: EvalTaskResult["evidenceBundle"],
+): Record<string, unknown> | undefined {
+  if (!evidenceBundle) return metadata;
+  return {
+    ...(metadata ?? {}),
+    [EVIDENCE_BUNDLE_METADATA_KEY]: evidenceBundle,
+  };
+}
+
+function readEvidenceBundleFromMetadata(metadata: Record<string, unknown> | undefined): EvalTaskResult["evidenceBundle"] {
+  if (!metadata) return undefined;
+  return metadata[EVIDENCE_BUNDLE_METADATA_KEY] as EvalTaskResult["evidenceBundle"] | undefined;
+}
+
+function sortEvidenceEntries<T extends TaskEvidenceEntryBase>(entries: T[]): T[] {
+  return [...entries].sort((a, b) => {
+    const timeOrder = (a.timestamp ?? "").localeCompare(b.timestamp ?? "");
+    if (timeOrder !== 0) return timeOrder;
+    return a.id.localeCompare(b.id);
+  });
+}
+
+function truncateEvidenceExcerpt<T extends TaskEvidenceEntryBase>(entry: T): T {
+  if (!entry.excerpt || entry.excerpt.length <= MAX_EVIDENCE_EXCERPT_LENGTH) {
+    return entry;
+  }
+  const maxPrefix = Math.max(0, MAX_EVIDENCE_EXCERPT_LENGTH - EVIDENCE_EXCERPT_TRUNCATION_MARKER.length);
+  return {
+    ...entry,
+    excerpt: `${entry.excerpt.slice(0, maxPrefix)}${EVIDENCE_EXCERPT_TRUNCATION_MARKER}`,
+    truncated: true,
+  };
+}
+
+function validateEvidenceBundle(bundle: TaskEvaluationEvidenceBundle | undefined): TaskEvaluationEvidenceBundle | undefined {
+  if (!bundle) return undefined;
+  if (bundle.sourceOrder.join("|") !== TASK_EVALUATION_EVIDENCE_SOURCE_ORDER.join("|")) {
+    throw new Error("evidenceBundle.sourceOrder must match TASK_EVALUATION_EVIDENCE_SOURCE_ORDER");
+  }
+  const groupLimits: Array<[keyof TaskEvaluationEvidenceBundle, number]> = [
+    ["taskMetadata", EVIDENCE_LIMITS.taskMetadata],
+    ["commits", EVIDENCE_LIMITS.commits],
+    ["workflow", EVIDENCE_LIMITS.workflow],
+    ["reviews", EVIDENCE_LIMITS.reviews],
+    ["documents", EVIDENCE_LIMITS.documents],
+    ["taskActivity", EVIDENCE_LIMITS.taskActivity],
+    ["agentLogs", EVIDENCE_LIMITS.agentLogs],
+    ["runAudit", EVIDENCE_LIMITS.runAudit],
+  ];
+  for (const [group, limit] of groupLimits) {
+    const entries = bundle[group];
+    if (Array.isArray(entries) && entries.length > limit) {
+      throw new Error(`evidenceBundle.${group} exceeds limit ${limit}`);
+    }
+  }
+
+  return {
+    ...bundle,
+    taskMetadata: sortEvidenceEntries(bundle.taskMetadata).map(truncateEvidenceExcerpt),
+    commits: sortEvidenceEntries(bundle.commits).map(truncateEvidenceExcerpt),
+    workflow: sortEvidenceEntries(bundle.workflow).map(truncateEvidenceExcerpt),
+    reviews: sortEvidenceEntries(bundle.reviews).map(truncateEvidenceExcerpt),
+    documents: sortEvidenceEntries(bundle.documents).map(truncateEvidenceExcerpt),
+    taskActivity: sortEvidenceEntries(bundle.taskActivity).map(truncateEvidenceExcerpt),
+    agentLogs: sortEvidenceEntries(bundle.agentLogs).map(truncateEvidenceExcerpt),
+    runAudit: sortEvidenceEntries(bundle.runAudit).map(truncateEvidenceExcerpt),
+  };
+}
+
 export class EvalStore extends EventEmitter<EvalStoreEvents> {
  constructor(private readonly db: Database) {
    super();
@@ -188,6 +268,8 @@ export class EvalStore extends EventEmitter<EvalStoreEvents> {
    if (!run) throw new Error(`Eval run not found: ${runId}`);

    const now = new Date().toISOString();
+    const evidenceBundle = validateEvidenceBundle(input.evidenceBundle);
+    const metadata = withEvidenceBundleMetadata(input.metadata, evidenceBundle);
    const result: EvalTaskResult = {
      id: generateResultId(),
      runId,
@@ -200,11 +282,12 @@ export class EvalStore extends EventEmitter<EvalStoreEvents> {
      rationale: input.rationale,
      summary: input.summary,
      evidence: input.evidence ?? [],
+      evidenceBundle,
      deterministicSignals: input.deterministicSignals ?? [],
      aiSignals: input.aiSignals,
      followUps: input.followUps ?? [],
      provenance: input.provenance,
-      metadata: input.metadata,
+      metadata,
      createdAt: now,
      updatedAt: now,
    };
@@ -301,10 +384,13 @@ export class EvalStore extends EventEmitter<EvalStoreEvents> {
    if (!existing) return undefined;

    const now = new Date().toISOString();
+    const mergedMetadata = input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata;
+    const evidenceBundle = validateEvidenceBundle(input.evidenceBundle ?? existing.evidenceBundle);
    const updated: EvalTaskResult = {
      ...existing,
      ...input,
-      metadata: input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata,
+      evidenceBundle,
+      metadata: withEvidenceBundleMetadata(mergedMetadata, evidenceBundle),
      provenance: input.provenance ? { ...(existing.provenance ?? {}), ...input.provenance } : existing.provenance,
      updatedAt: now,
    };
@@ -437,6 +523,8 @@ export class EvalStore extends EventEmitter<EvalStoreEvents> {
  }

  private rowToResult(row: Record<string, unknown>): EvalTaskResult {
+    const metadata = fromJson<Record<string, unknown>>(row.metadata as string);
+    const evidenceBundle = readEvidenceBundleFromMetadata(metadata);
    return {
      id: String(row.id),
      runId: String(row.runId),
@@ -449,11 +537,12 @@ export class EvalStore extends EventEmitter<EvalStoreEvents> {
      rationale: (row.rationale as string | null) ?? undefined,
      summary: (row.summary as string | null) ?? undefined,
      evidence: fromJson(row.evidence as string) ?? [],
+      evidenceBundle,
      deterministicSignals: fromJson(row.deterministicSignals as string) ?? [],
      aiSignals: fromJson(row.aiSignals as string),
      followUps: fromJson(row.followUps as string) ?? [],
      provenance: fromJson(row.provenance as string),
-      metadata: fromJson(row.metadata as string),
+      metadata,
      createdAt: String(row.createdAt),
      updatedAt: String(row.updatedAt),
    };
--- a/packages/core/src/eval-types.ts
+++ b/packages/core/src/eval-types.ts
@@ -92,6 +92,144 @@ export interface EvalEvidenceReference {
  metadata?: Record<string, unknown>;
 }

+export const TASK_EVALUATION_EVIDENCE_SOURCE_ORDER = [
+  "taskMetadata",
+  "commits",
+  "workflow",
+  "reviews",
+  "documents",
+  "taskActivity",
+  "agentLogs",
+  "runAudit",
+] as const;
+
+export const EVIDENCE_LIMITS = {
+  taskMetadata: 25,
+  commits: 20,
+  workflow: 25,
+  reviews: 25,
+  documents: 25,
+  taskActivity: 25,
+  agentLogs: 25,
+  runAudit: 25,
+} as const;
+
+export const MAX_EVIDENCE_EXCERPT_LENGTH = 500;
+export const EVIDENCE_EXCERPT_TRUNCATION_MARKER = "… [truncated]";
+
+export type TaskEvaluationEvidenceSource = typeof TASK_EVALUATION_EVIDENCE_SOURCE_ORDER[number];
+
+export interface TaskEvidenceEntryBase {
+  id: string;
+  source: TaskEvaluationEvidenceSource;
+  label: string;
+  timestamp?: string;
+  excerpt?: string;
+  truncated?: boolean;
+}
+
+export interface TaskMetadataEvidence extends TaskEvidenceEntryBase {
+  source: "taskMetadata";
+  taskId: string;
+  runId: string;
+  summary?: string;
+  references?: {
+    prNumber?: number;
+    prUrl?: string;
+    mergeCommitSha?: string;
+    mergeCompletedAt?: string;
+    executionStartedAt?: string;
+    executionCompletedAt?: string;
+  };
+  retryMetrics?: {
+    mergeRetries: number;
+    workflowStepRetries: number;
+    stuckKillCount: number;
+    postReviewFixCount: number;
+    recoveryRetryCount: number;
+    taskDoneRetryCount: number;
+    verificationFailureCount: number;
+    mergeConflictBounceCount: number;
+  };
+}
+
+export interface CommitEvidence extends TaskEvidenceEntryBase {
+  source: "commits";
+  sha: string;
+  taskId: string;
+  runId: string;
+  authoredAt?: string;
+  authorName?: string;
+  subject?: string;
+}
+
+export interface WorkflowEvidence extends TaskEvidenceEntryBase {
+  source: "workflow";
+  taskId: string;
+  runId: string;
+  workflowStepId?: string;
+  stepName?: string;
+  status?: string;
+  command?: string;
+}
+
+export interface ReviewEvidence extends TaskEvidenceEntryBase {
+  source: "reviews";
+  taskId: string;
+  runId: string;
+  reviewStep?: number;
+  reviewType?: string;
+  verdict?: string;
+}
+
+export interface DocumentEvidence extends TaskEvidenceEntryBase {
+  source: "documents";
+  taskId: string;
+  runId: string;
+  documentKey: string;
+  revision?: number;
+  author?: string;
+}
+
+export interface TaskActivityEvidence extends TaskEvidenceEntryBase {
+  source: "taskActivity";
+  taskId: string;
+  runId: string;
+  activityType?: string;
+}
+
+export interface AgentLogEvidence extends TaskEvidenceEntryBase {
+  source: "agentLogs";
+  taskId: string;
+  runId: string;
+  logType?: string;
+  agentId?: string;
+}
+
+export interface RunAuditEvidence extends TaskEvidenceEntryBase {
+  source: "runAudit";
+  taskId: string;
+  runId: string;
+  eventId: string;
+  domain?: string;
+  mutationType?: string;
+  target?: string;
+}
+
+export interface TaskEvaluationEvidenceBundle {
+  taskId: string;
+  runId: string;
+  sourceOrder: readonly TaskEvaluationEvidenceSource[];
+  taskMetadata: TaskMetadataEvidence[];
+  commits: CommitEvidence[];
+  workflow: WorkflowEvidence[];
+  reviews: ReviewEvidence[];
+  documents: DocumentEvidence[];
+  taskActivity: TaskActivityEvidence[];
+  agentLogs: AgentLogEvidence[];
+  runAudit: RunAuditEvidence[];
+}
+
 export interface EvalCategoryScore {
  category: EvalScoreCategory;
  deterministicScore: number;
@@ -123,6 +261,7 @@ export interface EvalTaskResult {
  rationale?: string;
  summary?: string;
  evidence: EvalEvidenceReference[];
+  evidenceBundle?: TaskEvaluationEvidenceBundle;
  deterministicSignals: EvalSignal[];
  aiSignals?: EvalSignal[];
  followUps: EvalFollowUpSuggestion[];
@@ -215,6 +354,7 @@ export interface EvalTaskResultCreateInput {
  rationale?: string;
  summary?: string;
  evidence?: EvalEvidenceReference[];
+  evidenceBundle?: TaskEvaluationEvidenceBundle;
  deterministicSignals?: EvalSignal[];
  aiSignals?: EvalSignal[];
  followUps?: EvalFollowUpSuggestion[];
@@ -230,6 +370,7 @@ export interface EvalTaskResultUpdateInput {
  rationale?: string;
  summary?: string;
  evidence?: EvalEvidenceReference[];
+  evidenceBundle?: TaskEvaluationEvidenceBundle;
  deterministicSignals?: EvalSignal[];
  aiSignals?: EvalSignal[];
  followUps?: EvalFollowUpSuggestion[];
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -729,6 +729,17 @@ export type {
  EvalScoreCategory,
  EvalCategoryScore,
  EvalEvidenceReference,
+  TaskEvaluationEvidenceSource,
+  TaskEvidenceEntryBase,
+  TaskMetadataEvidence,
+  CommitEvidence,
+  WorkflowEvidence,
+  ReviewEvidence,
+  DocumentEvidence,
+  TaskActivityEvidence,
+  AgentLogEvidence,
+  RunAuditEvidence,
+  TaskEvaluationEvidenceBundle,
  EvalSignal,
  EvalFollowUpSuggestion,
  EvalProvenance,
@@ -745,6 +756,10 @@ export {
  EVAL_SCORE_BANDS,
  EVAL_SCORE_SCALE_MIN,
  EVAL_SCORE_SCALE_MAX,
+  TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
+  EVIDENCE_LIMITS,
+  MAX_EVIDENCE_EXCERPT_LENGTH,
+  EVIDENCE_EXCERPT_TRUNCATION_MARKER,
 } from "./eval-types.js";
 export {
  EVAL_CATEGORY_WEIGHTS,
--- a/packages/engine/src/tests/evaluator-evidence.test.ts
+++ b/packages/engine/src/tests/evaluator-evidence.test.ts
@@ -0,0 +1,160 @@
+import { describe, expect, it, vi } from "vitest";
+import * as core from "@fusion/core";
+import { collectTaskEvaluationEvidence } from "../evaluator-evidence.js";
+
+const truncationMarker = core.EVIDENCE_EXCERPT_TRUNCATION_MARKER;
+
+function makeTask(overrides: Record<string, unknown> = {}): core.TaskDetail {
+  return {
+    id: "FN-1",
+    description: "desc",
+    column: "done",
+    dependencies: [],
+    steps: [],
+    currentStep: 0,
+    log: [],
+    createdAt: "2026-01-01T00:00:00.000Z",
+    updatedAt: "2026-01-01T01:00:00.000Z",
+    prompt: "prompt",
+    ...overrides,
+  } as core.TaskDetail;
+}
+
+function makeStore(overrides: Partial<core.TaskStore> = {}): core.TaskStore {
+  return {
+    getTaskDocuments: vi.fn().mockResolvedValue([]),
+    getAgentLogs: vi.fn().mockResolvedValue([]),
+    getRunAuditEvents: vi.fn().mockReturnValue([]),
+    ...overrides,
+  } as unknown as core.TaskStore;
+}
+
+describe("collectTaskEvaluationEvidence", () => {
+  it("collects fixed source groups with bounded excerpts", async () => {
+    const store = makeStore({
+      getTaskDocuments: vi.fn().mockResolvedValue([{ key: "plan", content: "x".repeat(900), revision: 1, author: "agent", updatedAt: "2026-01-01T00:01:00.000Z" }]),
+      getAgentLogs: vi.fn().mockResolvedValue([{ timestamp: "2026-01-01T00:01:30.000Z", taskId: "FN-1", text: "run", type: "tool_result", detail: "ok" }]),
+      getRunAuditEvents: vi.fn().mockReturnValue([{ id: "ra-1", timestamp: "2026-01-01T00:01:31.000Z", runId: "ER-1", agentId: "executor", taskId: "FN-1", domain: "git", mutationType: "git:commit", target: "HEAD" }]),
+    });
+    const task = makeTask({ summary: "summary", log: [{ timestamp: "2026-01-01T00:01:29.000Z", action: "Review step", outcome: "APPROVE" }] });
+
+    const evidence = await collectTaskEvaluationEvidence({ store, task, runId: "ER-1", cwd: process.cwd() });
+
+    expect(evidence.sourceOrder).toEqual(core.TASK_EVALUATION_EVIDENCE_SOURCE_ORDER);
+    expect(evidence.documents[0]?.excerpt?.length).toBeLessThanOrEqual(500);
+    expect(evidence.documents[0]?.truncated).toBe(true);
+    expect(evidence.documents[0]?.excerpt?.endsWith(truncationMarker)).toBe(true);
+    expect(evidence.taskMetadata[0]?.references?.executionCompletedAt).toBeUndefined();
+    expect(evidence.taskMetadata[0]?.retryMetrics?.mergeRetries).toBe(0);
+  });
+
+  it("gracefully handles absent optional sources", async () => {
+    const evidence = await collectTaskEvaluationEvidence({
+      store: makeStore(),
+      task: makeTask({ workflowStepResults: undefined, log: undefined }),
+      runId: "ER-2",
+      cwd: process.cwd(),
+    });
+
+    expect(evidence.workflow).toEqual([]);
+    expect(evidence.reviews).toEqual([]);
+    expect(evidence.agentLogs).toEqual([]);
+    expect(evidence.runAudit).toEqual([]);
+  });
+
+  it("handles git read failures by returning empty commit evidence", async () => {
+    const spy = vi.spyOn(core, "runCommandAsync").mockResolvedValue({
+      stdout: "",
+      stderr: "timeout",
+      exitCode: 1,
+      signal: null,
+      bufferExceeded: false,
+      timedOut: true,
+    });
+
+    const evidence = await collectTaskEvaluationEvidence({
+      store: makeStore(),
+      task: makeTask({ mergeDetails: { commitSha: "abc" } }),
+      runId: "ER-3",
+      cwd: process.cwd(),
+    });
+
+    expect(evidence.commits).toEqual([]);
+    spy.mockRestore();
+  });
+
+  it("caps agent logs and run-audit/task-activity to configured limits", async () => {
+    const agentLogs = Array.from({ length: 30 }, (_, i) => ({
+      timestamp: `2026-01-01T00:00:${String(i).padStart(2, "0")}.000Z`,
+      taskId: "FN-1",
+      text: `entry-${i}`,
+      type: "text" as const,
+    }));
+    const runAudit = Array.from({ length: 30 }, (_, i) => ({
+      id: `ra-${i}`,
+      timestamp: `2026-01-01T00:01:${String(i).padStart(2, "0")}.000Z`,
+      runId: "ER-4",
+      agentId: "executor",
+      taskId: "FN-1",
+      domain: "git",
+      mutationType: `mutation-${i}`,
+      target: `target-${i}`,
+    }));
+    const taskLog = Array.from({ length: 30 }, (_, i) => ({
+      timestamp: `2026-01-01T00:02:${String(i).padStart(2, "0")}.000Z`,
+      action: `action-${i}`,
+      outcome: "ok",
+    }));
+
+    const evidence = await collectTaskEvaluationEvidence({
+      store: makeStore({
+        getAgentLogs: vi.fn().mockResolvedValue(agentLogs),
+        getRunAuditEvents: vi.fn().mockReturnValue(runAudit),
+      }),
+      task: makeTask({ log: taskLog }),
+      runId: "ER-4",
+      cwd: process.cwd(),
+    });
+
+    expect(evidence.agentLogs).toHaveLength(core.EVIDENCE_LIMITS.agentLogs);
+    expect(evidence.runAudit).toHaveLength(core.EVIDENCE_LIMITS.runAudit);
+    expect(evidence.taskActivity).toHaveLength(core.EVIDENCE_LIMITS.taskActivity);
+    expect(evidence.agentLogs[0]?.excerpt).toContain("entry-5");
+    expect(evidence.agentLogs.at(-1)?.excerpt).toContain("entry-29");
+  });
+
+  it("truncates task metadata summary when oversized", async () => {
+    const evidence = await collectTaskEvaluationEvidence({
+      store: makeStore(),
+      task: makeTask({
+        summary: "s".repeat(700),
+        mergeRetries: 2,
+        workflowStepRetries: 3,
+        stuckKillCount: 1,
+        postReviewFixCount: 4,
+        recoveryRetryCount: 5,
+        taskDoneRetryCount: 6,
+        verificationFailureCount: 7,
+        mergeConflictBounceCount: 8,
+      }),
+      runId: "ER-5",
+      cwd: process.cwd(),
+    });
+
+    expect(evidence.taskMetadata[0]?.retryMetrics).toEqual({
+      mergeRetries: 2,
+      workflowStepRetries: 3,
+      stuckKillCount: 1,
+      postReviewFixCount: 4,
+      recoveryRetryCount: 5,
+      taskDoneRetryCount: 6,
+      verificationFailureCount: 7,
+      mergeConflictBounceCount: 8,
+    });
+
+    const summary = evidence.taskMetadata[0]?.summary ?? "";
+    expect(summary.length).toBeLessThanOrEqual(500);
+    expect(summary.endsWith(truncationMarker)).toBe(true);
+    expect(evidence.taskMetadata[0]?.truncated).toBe(true);
+  });
+});
--- a/packages/engine/src/tests/evaluator.test.ts
+++ b/packages/engine/src/tests/evaluator.test.ts
@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { computeOverallScore, createDatabase, EvalStore, runScheduledEvalBatch, type TaskDetail } from "@fusion/core";
+import { TASK_EVALUATION_EVIDENCE_SOURCE_ORDER, computeOverallScore, createDatabase, EvalStore, runScheduledEvalBatch, type TaskDetail, type TaskEvaluationEvidenceBundle } from "@fusion/core";
 import { HybridEvaluatorService, buildEvaluationPrompt, parseAiResponse, resolveEvaluatorModel } from "../evaluator.js";

 function makeTask(overrides: Partial<TaskDetail> = {}): TaskDetail {
@@ -110,13 +110,56 @@ describe("evaluator", () => {
      evidence: [],
    });
    expect(prompt).toContain("Deterministic signals");
+    expect(prompt).toContain("## Evidence");
    expect(prompt).toContain("ER-1");
  });

+  it("injects evidence bundle payload into prompt", () => {
+    const task = makeTask();
+    const bundle: TaskEvaluationEvidenceBundle = {
+      taskId: task.id,
+      runId: "ER-1",
+      sourceOrder: TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
+      taskMetadata: [{ id: "tm-1", source: "taskMetadata", label: "snapshot", taskId: task.id, runId: "ER-1" }],
+      commits: [{ id: "commit-abc123", source: "commits", label: "subject", sha: "abc123", taskId: task.id, runId: "ER-1" }],
+      workflow: [],
+      reviews: [],
+      documents: [],
+      taskActivity: [],
+      agentLogs: [],
+      runAudit: [],
+    };
+    const prompt = buildEvaluationPrompt(task, { runId: "ER-1", startedAt: "2026-05-02T00:00:00.000Z" }, {
+      taskId: task.id,
+      column: "done",
+      workflowSummary: { total: 0, passed: 0, failed: 0, pending: 0 },
+      commitSummary: { commitCount: 0 },
+      logSummary: { errorCount: 0, warningCount: 0, timingEntries: 1 },
+      evidence: [],
+    }, bundle);
+    expect(prompt).toContain("## Evidence");
+    expect(prompt).toContain("tm-1");
+    expect(prompt).toContain("commit-abc123");
+  });
+
  it("returns merged evaluation payload shape for persistence", async () => {
    const service = new HybridEvaluatorService({
      cwd: process.cwd(),
      runPrompt: async () => makeAiResponse(),
+      store: {} as any,
+      collectEvidence: async ({ task, runId }) => ({
+        taskId: task.id,
+        runId,
+        sourceOrder: TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
+        taskMetadata: [],
+        commits: [],
+        workflow: [],
+        reviews: [],
+        documents: [],
+        taskActivity: [],
+        agentLogs: [{ id: "agent-log-1", source: "agentLogs", label: "tool_result", taskId: task.id, runId, excerpt: "summary only" }],
+        runAudit: [],
+      }),
    });
    const result = await service.evaluateTask(makeTask(), { runId: "ER-1", startedAt: "2026-05-01T00:00:00.000Z" }, {});
    expect(result.status).toBe("scored");
@@ -130,6 +173,9 @@ describe("evaluator", () => {
      "processCompliance",
    ]);
    expect((result.metadata as any).hybridEvaluation).toBeDefined();
+    expect(result.evidenceBundle?.sourceOrder).toEqual(TASK_EVALUATION_EVIDENCE_SOURCE_ORDER);
+    expect(result.evidenceBundle?.agentLogs[0]?.excerpt).toBe("summary only");
+    expect((result.evidenceBundle?.agentLogs[0] as any)?.detail).toBeUndefined();
  });

  it("integrates scheduled batch with evaluator and persists one result per run/task", async () => {
--- a/packages/engine/src/cron-runner.ts
+++ b/packages/engine/src/cron-runner.ts
@@ -488,7 +488,7 @@ export class CronRunner {
  ): Promise<AutomationRunResult> {
    const settings = await this.store.getSettings();
    const evalSettings = resolveTaskEvaluationSettings(settings);
-    const evaluator = new HybridEvaluatorService({ cwd: this.options.workingDirectory ?? process.cwd() });
+    const evaluator = new HybridEvaluatorService({ cwd: this.options.workingDirectory ?? process.cwd(), store: this.store });

    const result = await runScheduledEvalBatch({
      store: this.store,
--- a/packages/engine/src/evaluator-evidence.ts
+++ b/packages/engine/src/evaluator-evidence.ts
@@ -0,0 +1,250 @@
+import type { AgentLogEntry, RunAuditEvent, TaskDetail, TaskDocument, TaskEvaluationEvidenceBundle, TaskLogEntry, TaskStore } from "@fusion/core";
+import {
+  EVIDENCE_LIMITS,
+  EVIDENCE_EXCERPT_TRUNCATION_MARKER,
+  MAX_EVIDENCE_EXCERPT_LENGTH,
+  TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
+  runCommandAsync,
+} from "@fusion/core";
+
+const COMMIT_SUBJECT_LIMIT = 160;
+
+function truncateExcerpt(text: string | undefined): { excerpt?: string; truncated?: boolean } {
+  if (!text) return {};
+  if (text.length <= MAX_EVIDENCE_EXCERPT_LENGTH) return { excerpt: text };
+  const prefixLength = Math.max(0, MAX_EVIDENCE_EXCERPT_LENGTH - EVIDENCE_EXCERPT_TRUNCATION_MARKER.length);
+  return { excerpt: `${text.slice(0, prefixLength)}${EVIDENCE_EXCERPT_TRUNCATION_MARKER}`, truncated: true };
+}
+
+function truncateSubject(text: string | undefined): string | undefined {
+  if (!text) return undefined;
+  return text.length <= COMMIT_SUBJECT_LIMIT ? text : text.slice(0, COMMIT_SUBJECT_LIMIT);
+}
+
+function byChronologicalThenId<T extends { timestamp?: string; id: string }>(a: T, b: T): number {
+  const timeOrder = (a.timestamp ?? "").localeCompare(b.timestamp ?? "");
+  if (timeOrder !== 0) return timeOrder;
+  return a.id.localeCompare(b.id);
+}
+
+async function collectCommitEvidence(task: TaskDetail, runId: string, cwd: string): Promise<TaskEvaluationEvidenceBundle["commits"]> {
+  const mergeSha = task.mergeDetails?.commitSha;
+  if (!mergeSha) return [];
+
+  const command = [
+    "git log",
+    "--pretty=format:%H%x09%an%x09%aI%x09%s",
+    `-n ${EVIDENCE_LIMITS.commits}`,
+    mergeSha,
+  ].join(" ");
+  const res = await runCommandAsync(command, { cwd, timeoutMs: 7_500, maxBuffer: 1024 * 1024 });
+  if (res.exitCode !== 0 || !res.stdout.trim()) return [];
+
+  return res.stdout
+    .trim()
+    .split("\n")
+    .map((line, index) => {
+      const [sha, authorName, authoredAt, subject] = line.split("\t");
+      const { excerpt, truncated } = truncateExcerpt(subject);
+      return {
+        id: `commit-${sha || index + 1}`,
+        source: "commits" as const,
+        label: truncateSubject(subject) ?? `commit ${index + 1}`,
+        taskId: task.id,
+        runId,
+        timestamp: authoredAt,
+        authoredAt,
+        authorName,
+        sha,
+        subject: truncateSubject(subject),
+        excerpt,
+        truncated,
+      };
+    })
+    .sort(byChronologicalThenId)
+    .slice(-EVIDENCE_LIMITS.commits);
+}
+
+function collectDocumentEvidence(taskId: string, runId: string, docs: TaskDocument[]): TaskEvaluationEvidenceBundle["documents"] {
+  return docs
+    .map((doc, index) => {
+      const { excerpt, truncated } = truncateExcerpt(doc.content);
+      return {
+        id: `doc-${index + 1}`,
+        source: "documents" as const,
+        label: doc.key,
+        taskId,
+        runId,
+        timestamp: doc.updatedAt,
+        documentKey: doc.key,
+        revision: doc.revision,
+        author: doc.author,
+        excerpt,
+        truncated,
+      };
+    })
+    .sort(byChronologicalThenId)
+    .slice(-EVIDENCE_LIMITS.documents);
+}
+
+function collectTaskActivityEvidence(taskId: string, runId: string, entries: TaskLogEntry[]): TaskEvaluationEvidenceBundle["taskActivity"] {
+  return entries
+    .map((entry, index) => {
+      const text = [entry.action, entry.outcome].filter(Boolean).join(" — ");
+      const { excerpt, truncated } = truncateExcerpt(text);
+      return {
+        id: `task-activity-${index + 1}`,
+        source: "taskActivity" as const,
+        label: entry.action,
+        taskId,
+        runId,
+        timestamp: entry.timestamp,
+        activityType: entry.action,
+        excerpt,
+        truncated,
+      };
+    })
+    .sort(byChronologicalThenId)
+    .slice(-EVIDENCE_LIMITS.taskActivity);
+}
+
+function collectAgentLogEvidence(taskId: string, runId: string, entries: AgentLogEntry[]): TaskEvaluationEvidenceBundle["agentLogs"] {
+  return entries
+    .map((entry, index) => {
+      const text = `${entry.text}${entry.detail ? ` — ${entry.detail}` : ""}`;
+      const { excerpt, truncated } = truncateExcerpt(text);
+      return {
+        id: `agent-log-${index + 1}`,
+        source: "agentLogs" as const,
+        label: entry.type,
+        taskId,
+        runId,
+        timestamp: entry.timestamp,
+        logType: entry.type,
+        agentId: entry.agent,
+        excerpt,
+        truncated,
+      };
+    })
+    .sort(byChronologicalThenId)
+    .slice(-EVIDENCE_LIMITS.agentLogs);
+}
+
+function collectRunAuditEvidence(taskId: string, runId: string, events: RunAuditEvent[]): TaskEvaluationEvidenceBundle["runAudit"] {
+  return events
+    .map((event, index) => {
+      const { excerpt, truncated } = truncateExcerpt(`${event.mutationType} ${event.target}`);
+      return {
+        id: `run-audit-${index + 1}`,
+        source: "runAudit" as const,
+        label: event.mutationType,
+        taskId,
+        runId,
+        timestamp: event.timestamp,
+        eventId: event.id,
+        domain: event.domain,
+        mutationType: event.mutationType,
+        target: event.target,
+        excerpt,
+        truncated,
+      };
+    })
+    .sort(byChronologicalThenId)
+    .slice(-EVIDENCE_LIMITS.runAudit);
+}
+
+export async function collectTaskEvaluationEvidence(params: {
+  store: TaskStore;
+  task: TaskDetail;
+  runId: string;
+  cwd: string;
+}): Promise<TaskEvaluationEvidenceBundle> {
+  const { store, task, runId, cwd } = params;
+  const [documents, agentLogs] = await Promise.all([
+    store.getTaskDocuments(task.id),
+    store.getAgentLogs(task.id, { limit: EVIDENCE_LIMITS.agentLogs }),
+  ]);
+  const commitEvidence = await collectCommitEvidence(task, runId, cwd);
+  const workflow = (task.workflowStepResults ?? [])
+    .map((step, index) => {
+      const { excerpt, truncated } = truncateExcerpt(step.output);
+      return {
+        id: `workflow-${index + 1}`,
+        source: "workflow" as const,
+        label: step.workflowStepName,
+        taskId: task.id,
+        runId,
+        timestamp: step.completedAt ?? step.startedAt,
+        workflowStepId: step.workflowStepId,
+        stepName: step.workflowStepName,
+        status: step.status,
+        excerpt,
+        truncated,
+      };
+    })
+    .sort(byChronologicalThenId)
+    .slice(-EVIDENCE_LIMITS.workflow);
+
+  const reviews = (task.log ?? [])
+    .filter((entry) => /review/i.test(entry.action))
+    .map((entry, index) => {
+      const { excerpt, truncated } = truncateExcerpt(entry.outcome);
+      return {
+        id: `review-${index + 1}`,
+        source: "reviews" as const,
+        label: entry.action,
+        taskId: task.id,
+        runId,
+        timestamp: entry.timestamp,
+        verdict: entry.outcome,
+        excerpt,
+        truncated,
+      };
+    })
+    .sort(byChronologicalThenId)
+    .slice(-EVIDENCE_LIMITS.reviews);
+
+  const taskMetadataExcerpt = truncateExcerpt(task.summary ?? task.description);
+
+  return {
+    taskId: task.id,
+    runId,
+    sourceOrder: TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
+    taskMetadata: [{
+      id: "task-metadata-1",
+      source: "taskMetadata",
+      label: task.title ?? task.id,
+      taskId: task.id,
+      runId,
+      timestamp: task.updatedAt,
+      summary: taskMetadataExcerpt.excerpt,
+      excerpt: taskMetadataExcerpt.excerpt,
+      truncated: taskMetadataExcerpt.truncated,
+      references: {
+        prNumber: task.prInfo?.number,
+        prUrl: task.prInfo?.url,
+        mergeCommitSha: task.mergeDetails?.commitSha,
+        mergeCompletedAt: task.mergeDetails?.mergedAt,
+        executionStartedAt: task.executionStartedAt,
+        executionCompletedAt: task.executionCompletedAt,
+      },
+      retryMetrics: {
+        mergeRetries: task.mergeRetries ?? 0,
+        workflowStepRetries: task.workflowStepRetries ?? 0,
+        stuckKillCount: task.stuckKillCount ?? 0,
+        postReviewFixCount: task.postReviewFixCount ?? 0,
+        recoveryRetryCount: task.recoveryRetryCount ?? 0,
+        taskDoneRetryCount: task.taskDoneRetryCount ?? 0,
+        verificationFailureCount: task.verificationFailureCount ?? 0,
+        mergeConflictBounceCount: task.mergeConflictBounceCount ?? 0,
+      },
+    }],
+    commits: commitEvidence,
+    workflow,
+    reviews,
+    documents: collectDocumentEvidence(task.id, runId, documents),
+    taskActivity: collectTaskActivityEvidence(task.id, runId, task.log ?? []),
+    agentLogs: collectAgentLogEvidence(task.id, runId, agentLogs),
+    runAudit: collectRunAuditEvidence(task.id, runId, store.getRunAuditEvents({ taskId: task.id, limit: EVIDENCE_LIMITS.runAudit })),
+  };
+}
--- a/packages/engine/src/evaluator.ts
+++ b/packages/engine/src/evaluator.ts
@@ -11,7 +11,10 @@ import {
  type FollowUpDraft,
  type Settings,
  type TaskDetail,
+  type TaskEvaluationEvidenceBundle,
+  type TaskStore,
 } from "@fusion/core";
+import { collectTaskEvaluationEvidence } from "./evaluator-evidence.js";
 import { createFnAgent, promptWithFallback } from "./pi.js";
 import { createLogger } from "./logger.js";

@@ -29,7 +32,9 @@ export interface EvaluatorModelOverride {

 export interface EvaluatorDeps {
  cwd: string;
+  store?: TaskStore;
  runPrompt?: (prompt: string, provider?: string, modelId?: string) => Promise<string>;
+  collectEvidence?: (params: { task: TaskDetail; runId: string; cwd: string; store: TaskStore }) => Promise<TaskEvaluationEvidenceBundle>;
 }

 interface EvaluatorAiCategoryResponse {
@@ -66,7 +71,15 @@ export class HybridEvaluatorService {
  ): Promise<Omit<EvalTaskResultCreateInput, "taskId" | "taskSnapshot">> {
    const deterministicSignals = collectDeterministicSignals(task, run);
    const model = resolveEvaluatorModel(settings, modelOverride);
-    const prompt = buildEvaluationPrompt(task, run, deterministicSignals);
+    const evidenceBundle = this.deps.store
+      ? await (this.deps.collectEvidence ?? collectTaskEvaluationEvidence)({
+        store: this.deps.store,
+        task,
+        runId: run.runId,
+        cwd: this.deps.cwd,
+      })
+      : undefined;
+    const prompt = buildEvaluationPrompt(task, run, deterministicSignals, evidenceBundle);
    const responseText = await this.runPrompt(prompt, model.provider, model.modelId);
    const ai = parseAiResponse(responseText);

@@ -95,6 +108,7 @@ export class HybridEvaluatorService {
      rationale: ai.overallRationale,
      summary: ai.overallRationale,
      evidence: categoryScores.flatMap((categoryScore) => categoryScore.evidence),
+      evidenceBundle,
      deterministicSignals: deterministicSignalsToEvalSignals(deterministicSignals),
      followUps: ai.followUpDrafts.map((draft) => ({
        title: draft.title,
@@ -188,10 +202,30 @@ function deterministicSignalsToEvalSignals(signals: DeterministicSignals): Array
  ];
 }

-export function buildEvaluationPrompt(task: TaskDetail, run: EvalRunContext, deterministicSignals: DeterministicSignals): string {
+function formatEvidenceForPrompt(evidenceBundle: TaskEvaluationEvidenceBundle): string {
+  return JSON.stringify({
+    sourceOrder: evidenceBundle.sourceOrder,
+    taskMetadata: evidenceBundle.taskMetadata,
+    commits: evidenceBundle.commits,
+    workflow: evidenceBundle.workflow,
+    reviews: evidenceBundle.reviews,
+    documents: evidenceBundle.documents,
+    taskActivity: evidenceBundle.taskActivity,
+    agentLogs: evidenceBundle.agentLogs,
+    runAudit: evidenceBundle.runAudit,
+  }, null, 2);
+}
+
+export function buildEvaluationPrompt(
+  task: TaskDetail,
+  run: EvalRunContext,
+  deterministicSignals: DeterministicSignals,
+  evidenceBundle?: TaskEvaluationEvidenceBundle,
+): string {
  return [
    "Evaluate the completed task and respond with strict JSON.",
    "Scores must be integers between 0 and 100.",
+    "When citing evidence, use labels that include evidence IDs from the ## Evidence section.",
    `Run: ${run.runId}`,
    "Schema:",
    JSON.stringify({
@@ -213,6 +247,8 @@ export function buildEvaluationPrompt(task: TaskDetail, run: EvalRunContext, det
    }, null, 2),
    "Deterministic signals:",
    JSON.stringify(deterministicSignals, null, 2),
+    "## Evidence",
+    evidenceBundle ? formatEvidenceForPrompt(evidenceBundle) : JSON.stringify({ sourceOrder: [], note: "No evidence bundle available" }, null, 2),
  ].join("\n\n");
 }

--- a/packages/engine/src/index.ts
+++ b/packages/engine/src/index.ts
@@ -12,6 +12,7 @@ export {
 export { AgentSemaphore, PRIORITY_MERGE, PRIORITY_EXECUTE, PRIORITY_SPECIFY } from "./concurrency.js";
 export { TriageProcessor, type TriageProcessorOptions } from "./triage.js";
 export { TaskExecutor, type TaskExecutorOptions } from "./executor.js";
+export { collectTaskEvaluationEvidence } from "./evaluator-evidence.js";
 export { Scheduler, type SchedulerOptions } from "./scheduler.js";
 export { MissionAutopilot, type MissionAutopilotOptions } from "./mission-autopilot.js";
 export { MissionExecutionLoop, type MissionExecutionLoopOptions, type ValidationResult, loopLog } from "./mission-execution-loop.js";