feat(FN-3391): persist evaluator evidence with score categories
- Add eval score category types and exports in core with store support and coverage - Implement engine evaluator evidence extraction and persistence with dedicated tests - Update evaluator flow and cron wiring to record evidence alongside eval runs - Refresh architecture, storage, and eval docs for evidence and categorization behavior Fusion-Task-Id: FN-3391
This commit is contained in:
Fusion
160
packages/engine/src/__tests__/evaluator-evidence.test.ts
Normal file
160
packages/engine/src/__tests__/evaluator-evidence.test.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import * as core from "@fusion/core";
|
||||
import { collectTaskEvaluationEvidence } from "../evaluator-evidence.js";
|
||||
|
||||
const truncationMarker = core.EVIDENCE_EXCERPT_TRUNCATION_MARKER;
|
||||
|
||||
function makeTask(overrides: Record<string, unknown> = {}): core.TaskDetail {
|
||||
return {
|
||||
id: "FN-1",
|
||||
description: "desc",
|
||||
column: "done",
|
||||
dependencies: [],
|
||||
steps: [],
|
||||
currentStep: 0,
|
||||
log: [],
|
||||
createdAt: "2026-01-01T00:00:00.000Z",
|
||||
updatedAt: "2026-01-01T01:00:00.000Z",
|
||||
prompt: "prompt",
|
||||
...overrides,
|
||||
} as core.TaskDetail;
|
||||
}
|
||||
|
||||
function makeStore(overrides: Partial<core.TaskStore> = {}): core.TaskStore {
|
||||
return {
|
||||
getTaskDocuments: vi.fn().mockResolvedValue([]),
|
||||
getAgentLogs: vi.fn().mockResolvedValue([]),
|
||||
getRunAuditEvents: vi.fn().mockReturnValue([]),
|
||||
...overrides,
|
||||
} as unknown as core.TaskStore;
|
||||
}
|
||||
|
||||
describe("collectTaskEvaluationEvidence", () => {
|
||||
it("collects fixed source groups with bounded excerpts", async () => {
|
||||
const store = makeStore({
|
||||
getTaskDocuments: vi.fn().mockResolvedValue([{ key: "plan", content: "x".repeat(900), revision: 1, author: "agent", updatedAt: "2026-01-01T00:01:00.000Z" }]),
|
||||
getAgentLogs: vi.fn().mockResolvedValue([{ timestamp: "2026-01-01T00:01:30.000Z", taskId: "FN-1", text: "run", type: "tool_result", detail: "ok" }]),
|
||||
getRunAuditEvents: vi.fn().mockReturnValue([{ id: "ra-1", timestamp: "2026-01-01T00:01:31.000Z", runId: "ER-1", agentId: "executor", taskId: "FN-1", domain: "git", mutationType: "git:commit", target: "HEAD" }]),
|
||||
});
|
||||
const task = makeTask({ summary: "summary", log: [{ timestamp: "2026-01-01T00:01:29.000Z", action: "Review step", outcome: "APPROVE" }] });
|
||||
|
||||
const evidence = await collectTaskEvaluationEvidence({ store, task, runId: "ER-1", cwd: process.cwd() });
|
||||
|
||||
expect(evidence.sourceOrder).toEqual(core.TASK_EVALUATION_EVIDENCE_SOURCE_ORDER);
|
||||
expect(evidence.documents[0]?.excerpt?.length).toBeLessThanOrEqual(500);
|
||||
expect(evidence.documents[0]?.truncated).toBe(true);
|
||||
expect(evidence.documents[0]?.excerpt?.endsWith(truncationMarker)).toBe(true);
|
||||
expect(evidence.taskMetadata[0]?.references?.executionCompletedAt).toBeUndefined();
|
||||
expect(evidence.taskMetadata[0]?.retryMetrics?.mergeRetries).toBe(0);
|
||||
});
|
||||
|
||||
it("gracefully handles absent optional sources", async () => {
|
||||
const evidence = await collectTaskEvaluationEvidence({
|
||||
store: makeStore(),
|
||||
task: makeTask({ workflowStepResults: undefined, log: undefined }),
|
||||
runId: "ER-2",
|
||||
cwd: process.cwd(),
|
||||
});
|
||||
|
||||
expect(evidence.workflow).toEqual([]);
|
||||
expect(evidence.reviews).toEqual([]);
|
||||
expect(evidence.agentLogs).toEqual([]);
|
||||
expect(evidence.runAudit).toEqual([]);
|
||||
});
|
||||
|
||||
it("handles git read failures by returning empty commit evidence", async () => {
|
||||
const spy = vi.spyOn(core, "runCommandAsync").mockResolvedValue({
|
||||
stdout: "",
|
||||
stderr: "timeout",
|
||||
exitCode: 1,
|
||||
signal: null,
|
||||
bufferExceeded: false,
|
||||
timedOut: true,
|
||||
});
|
||||
|
||||
const evidence = await collectTaskEvaluationEvidence({
|
||||
store: makeStore(),
|
||||
task: makeTask({ mergeDetails: { commitSha: "abc" } }),
|
||||
runId: "ER-3",
|
||||
cwd: process.cwd(),
|
||||
});
|
||||
|
||||
expect(evidence.commits).toEqual([]);
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
it("caps agent logs and run-audit/task-activity to configured limits", async () => {
|
||||
const agentLogs = Array.from({ length: 30 }, (_, i) => ({
|
||||
timestamp: `2026-01-01T00:00:${String(i).padStart(2, "0")}.000Z`,
|
||||
taskId: "FN-1",
|
||||
text: `entry-${i}`,
|
||||
type: "text" as const,
|
||||
}));
|
||||
const runAudit = Array.from({ length: 30 }, (_, i) => ({
|
||||
id: `ra-${i}`,
|
||||
timestamp: `2026-01-01T00:01:${String(i).padStart(2, "0")}.000Z`,
|
||||
runId: "ER-4",
|
||||
agentId: "executor",
|
||||
taskId: "FN-1",
|
||||
domain: "git",
|
||||
mutationType: `mutation-${i}`,
|
||||
target: `target-${i}`,
|
||||
}));
|
||||
const taskLog = Array.from({ length: 30 }, (_, i) => ({
|
||||
timestamp: `2026-01-01T00:02:${String(i).padStart(2, "0")}.000Z`,
|
||||
action: `action-${i}`,
|
||||
outcome: "ok",
|
||||
}));
|
||||
|
||||
const evidence = await collectTaskEvaluationEvidence({
|
||||
store: makeStore({
|
||||
getAgentLogs: vi.fn().mockResolvedValue(agentLogs),
|
||||
getRunAuditEvents: vi.fn().mockReturnValue(runAudit),
|
||||
}),
|
||||
task: makeTask({ log: taskLog }),
|
||||
runId: "ER-4",
|
||||
cwd: process.cwd(),
|
||||
});
|
||||
|
||||
expect(evidence.agentLogs).toHaveLength(core.EVIDENCE_LIMITS.agentLogs);
|
||||
expect(evidence.runAudit).toHaveLength(core.EVIDENCE_LIMITS.runAudit);
|
||||
expect(evidence.taskActivity).toHaveLength(core.EVIDENCE_LIMITS.taskActivity);
|
||||
expect(evidence.agentLogs[0]?.excerpt).toContain("entry-5");
|
||||
expect(evidence.agentLogs.at(-1)?.excerpt).toContain("entry-29");
|
||||
});
|
||||
|
||||
it("truncates task metadata summary when oversized", async () => {
|
||||
const evidence = await collectTaskEvaluationEvidence({
|
||||
store: makeStore(),
|
||||
task: makeTask({
|
||||
summary: "s".repeat(700),
|
||||
mergeRetries: 2,
|
||||
workflowStepRetries: 3,
|
||||
stuckKillCount: 1,
|
||||
postReviewFixCount: 4,
|
||||
recoveryRetryCount: 5,
|
||||
taskDoneRetryCount: 6,
|
||||
verificationFailureCount: 7,
|
||||
mergeConflictBounceCount: 8,
|
||||
}),
|
||||
runId: "ER-5",
|
||||
cwd: process.cwd(),
|
||||
});
|
||||
|
||||
expect(evidence.taskMetadata[0]?.retryMetrics).toEqual({
|
||||
mergeRetries: 2,
|
||||
workflowStepRetries: 3,
|
||||
stuckKillCount: 1,
|
||||
postReviewFixCount: 4,
|
||||
recoveryRetryCount: 5,
|
||||
taskDoneRetryCount: 6,
|
||||
verificationFailureCount: 7,
|
||||
mergeConflictBounceCount: 8,
|
||||
});
|
||||
|
||||
const summary = evidence.taskMetadata[0]?.summary ?? "";
|
||||
expect(summary.length).toBeLessThanOrEqual(500);
|
||||
expect(summary.endsWith(truncationMarker)).toBe(true);
|
||||
expect(evidence.taskMetadata[0]?.truncated).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { computeOverallScore, createDatabase, EvalStore, runScheduledEvalBatch, type TaskDetail } from "@fusion/core";
|
||||
import { TASK_EVALUATION_EVIDENCE_SOURCE_ORDER, computeOverallScore, createDatabase, EvalStore, runScheduledEvalBatch, type TaskDetail, type TaskEvaluationEvidenceBundle } from "@fusion/core";
|
||||
import { HybridEvaluatorService, buildEvaluationPrompt, parseAiResponse, resolveEvaluatorModel } from "../evaluator.js";
|
||||
|
||||
function makeTask(overrides: Partial<TaskDetail> = {}): TaskDetail {
|
||||
@@ -110,13 +110,56 @@ describe("evaluator", () => {
|
||||
evidence: [],
|
||||
});
|
||||
expect(prompt).toContain("Deterministic signals");
|
||||
expect(prompt).toContain("## Evidence");
|
||||
expect(prompt).toContain("ER-1");
|
||||
});
|
||||
|
||||
it("injects evidence bundle payload into prompt", () => {
|
||||
const task = makeTask();
|
||||
const bundle: TaskEvaluationEvidenceBundle = {
|
||||
taskId: task.id,
|
||||
runId: "ER-1",
|
||||
sourceOrder: TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
|
||||
taskMetadata: [{ id: "tm-1", source: "taskMetadata", label: "snapshot", taskId: task.id, runId: "ER-1" }],
|
||||
commits: [{ id: "commit-abc123", source: "commits", label: "subject", sha: "abc123", taskId: task.id, runId: "ER-1" }],
|
||||
workflow: [],
|
||||
reviews: [],
|
||||
documents: [],
|
||||
taskActivity: [],
|
||||
agentLogs: [],
|
||||
runAudit: [],
|
||||
};
|
||||
const prompt = buildEvaluationPrompt(task, { runId: "ER-1", startedAt: "2026-05-02T00:00:00.000Z" }, {
|
||||
taskId: task.id,
|
||||
column: "done",
|
||||
workflowSummary: { total: 0, passed: 0, failed: 0, pending: 0 },
|
||||
commitSummary: { commitCount: 0 },
|
||||
logSummary: { errorCount: 0, warningCount: 0, timingEntries: 1 },
|
||||
evidence: [],
|
||||
}, bundle);
|
||||
expect(prompt).toContain("## Evidence");
|
||||
expect(prompt).toContain("tm-1");
|
||||
expect(prompt).toContain("commit-abc123");
|
||||
});
|
||||
|
||||
it("returns merged evaluation payload shape for persistence", async () => {
|
||||
const service = new HybridEvaluatorService({
|
||||
cwd: process.cwd(),
|
||||
runPrompt: async () => makeAiResponse(),
|
||||
store: {} as any,
|
||||
collectEvidence: async ({ task, runId }) => ({
|
||||
taskId: task.id,
|
||||
runId,
|
||||
sourceOrder: TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
|
||||
taskMetadata: [],
|
||||
commits: [],
|
||||
workflow: [],
|
||||
reviews: [],
|
||||
documents: [],
|
||||
taskActivity: [],
|
||||
agentLogs: [{ id: "agent-log-1", source: "agentLogs", label: "tool_result", taskId: task.id, runId, excerpt: "summary only" }],
|
||||
runAudit: [],
|
||||
}),
|
||||
});
|
||||
const result = await service.evaluateTask(makeTask(), { runId: "ER-1", startedAt: "2026-05-01T00:00:00.000Z" }, {});
|
||||
expect(result.status).toBe("scored");
|
||||
@@ -130,6 +173,9 @@ describe("evaluator", () => {
|
||||
"processCompliance",
|
||||
]);
|
||||
expect((result.metadata as any).hybridEvaluation).toBeDefined();
|
||||
expect(result.evidenceBundle?.sourceOrder).toEqual(TASK_EVALUATION_EVIDENCE_SOURCE_ORDER);
|
||||
expect(result.evidenceBundle?.agentLogs[0]?.excerpt).toBe("summary only");
|
||||
expect((result.evidenceBundle?.agentLogs[0] as any)?.detail).toBeUndefined();
|
||||
});
|
||||
|
||||
it("integrates scheduled batch with evaluator and persists one result per run/task", async () => {
|
||||
|
||||
@@ -488,7 +488,7 @@ export class CronRunner {
|
||||
): Promise<AutomationRunResult> {
|
||||
const settings = await this.store.getSettings();
|
||||
const evalSettings = resolveTaskEvaluationSettings(settings);
|
||||
const evaluator = new HybridEvaluatorService({ cwd: this.options.workingDirectory ?? process.cwd() });
|
||||
const evaluator = new HybridEvaluatorService({ cwd: this.options.workingDirectory ?? process.cwd(), store: this.store });
|
||||
|
||||
const result = await runScheduledEvalBatch({
|
||||
store: this.store,
|
||||
|
||||
250
packages/engine/src/evaluator-evidence.ts
Normal file
250
packages/engine/src/evaluator-evidence.ts
Normal file
@@ -0,0 +1,250 @@
|
||||
import type { AgentLogEntry, RunAuditEvent, TaskDetail, TaskDocument, TaskEvaluationEvidenceBundle, TaskLogEntry, TaskStore } from "@fusion/core";
|
||||
import {
|
||||
EVIDENCE_LIMITS,
|
||||
EVIDENCE_EXCERPT_TRUNCATION_MARKER,
|
||||
MAX_EVIDENCE_EXCERPT_LENGTH,
|
||||
TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
|
||||
runCommandAsync,
|
||||
} from "@fusion/core";
|
||||
|
||||
const COMMIT_SUBJECT_LIMIT = 160;
|
||||
|
||||
function truncateExcerpt(text: string | undefined): { excerpt?: string; truncated?: boolean } {
|
||||
if (!text) return {};
|
||||
if (text.length <= MAX_EVIDENCE_EXCERPT_LENGTH) return { excerpt: text };
|
||||
const prefixLength = Math.max(0, MAX_EVIDENCE_EXCERPT_LENGTH - EVIDENCE_EXCERPT_TRUNCATION_MARKER.length);
|
||||
return { excerpt: `${text.slice(0, prefixLength)}${EVIDENCE_EXCERPT_TRUNCATION_MARKER}`, truncated: true };
|
||||
}
|
||||
|
||||
function truncateSubject(text: string | undefined): string | undefined {
|
||||
if (!text) return undefined;
|
||||
return text.length <= COMMIT_SUBJECT_LIMIT ? text : text.slice(0, COMMIT_SUBJECT_LIMIT);
|
||||
}
|
||||
|
||||
function byChronologicalThenId<T extends { timestamp?: string; id: string }>(a: T, b: T): number {
|
||||
const timeOrder = (a.timestamp ?? "").localeCompare(b.timestamp ?? "");
|
||||
if (timeOrder !== 0) return timeOrder;
|
||||
return a.id.localeCompare(b.id);
|
||||
}
|
||||
|
||||
async function collectCommitEvidence(task: TaskDetail, runId: string, cwd: string): Promise<TaskEvaluationEvidenceBundle["commits"]> {
|
||||
const mergeSha = task.mergeDetails?.commitSha;
|
||||
if (!mergeSha) return [];
|
||||
|
||||
const command = [
|
||||
"git log",
|
||||
"--pretty=format:%H%x09%an%x09%aI%x09%s",
|
||||
`-n ${EVIDENCE_LIMITS.commits}`,
|
||||
mergeSha,
|
||||
].join(" ");
|
||||
const res = await runCommandAsync(command, { cwd, timeoutMs: 7_500, maxBuffer: 1024 * 1024 });
|
||||
if (res.exitCode !== 0 || !res.stdout.trim()) return [];
|
||||
|
||||
return res.stdout
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line, index) => {
|
||||
const [sha, authorName, authoredAt, subject] = line.split("\t");
|
||||
const { excerpt, truncated } = truncateExcerpt(subject);
|
||||
return {
|
||||
id: `commit-${sha || index + 1}`,
|
||||
source: "commits" as const,
|
||||
label: truncateSubject(subject) ?? `commit ${index + 1}`,
|
||||
taskId: task.id,
|
||||
runId,
|
||||
timestamp: authoredAt,
|
||||
authoredAt,
|
||||
authorName,
|
||||
sha,
|
||||
subject: truncateSubject(subject),
|
||||
excerpt,
|
||||
truncated,
|
||||
};
|
||||
})
|
||||
.sort(byChronologicalThenId)
|
||||
.slice(-EVIDENCE_LIMITS.commits);
|
||||
}
|
||||
|
||||
function collectDocumentEvidence(taskId: string, runId: string, docs: TaskDocument[]): TaskEvaluationEvidenceBundle["documents"] {
|
||||
return docs
|
||||
.map((doc, index) => {
|
||||
const { excerpt, truncated } = truncateExcerpt(doc.content);
|
||||
return {
|
||||
id: `doc-${index + 1}`,
|
||||
source: "documents" as const,
|
||||
label: doc.key,
|
||||
taskId,
|
||||
runId,
|
||||
timestamp: doc.updatedAt,
|
||||
documentKey: doc.key,
|
||||
revision: doc.revision,
|
||||
author: doc.author,
|
||||
excerpt,
|
||||
truncated,
|
||||
};
|
||||
})
|
||||
.sort(byChronologicalThenId)
|
||||
.slice(-EVIDENCE_LIMITS.documents);
|
||||
}
|
||||
|
||||
function collectTaskActivityEvidence(taskId: string, runId: string, entries: TaskLogEntry[]): TaskEvaluationEvidenceBundle["taskActivity"] {
|
||||
return entries
|
||||
.map((entry, index) => {
|
||||
const text = [entry.action, entry.outcome].filter(Boolean).join(" — ");
|
||||
const { excerpt, truncated } = truncateExcerpt(text);
|
||||
return {
|
||||
id: `task-activity-${index + 1}`,
|
||||
source: "taskActivity" as const,
|
||||
label: entry.action,
|
||||
taskId,
|
||||
runId,
|
||||
timestamp: entry.timestamp,
|
||||
activityType: entry.action,
|
||||
excerpt,
|
||||
truncated,
|
||||
};
|
||||
})
|
||||
.sort(byChronologicalThenId)
|
||||
.slice(-EVIDENCE_LIMITS.taskActivity);
|
||||
}
|
||||
|
||||
function collectAgentLogEvidence(taskId: string, runId: string, entries: AgentLogEntry[]): TaskEvaluationEvidenceBundle["agentLogs"] {
|
||||
return entries
|
||||
.map((entry, index) => {
|
||||
const text = `${entry.text}${entry.detail ? ` — ${entry.detail}` : ""}`;
|
||||
const { excerpt, truncated } = truncateExcerpt(text);
|
||||
return {
|
||||
id: `agent-log-${index + 1}`,
|
||||
source: "agentLogs" as const,
|
||||
label: entry.type,
|
||||
taskId,
|
||||
runId,
|
||||
timestamp: entry.timestamp,
|
||||
logType: entry.type,
|
||||
agentId: entry.agent,
|
||||
excerpt,
|
||||
truncated,
|
||||
};
|
||||
})
|
||||
.sort(byChronologicalThenId)
|
||||
.slice(-EVIDENCE_LIMITS.agentLogs);
|
||||
}
|
||||
|
||||
function collectRunAuditEvidence(taskId: string, runId: string, events: RunAuditEvent[]): TaskEvaluationEvidenceBundle["runAudit"] {
|
||||
return events
|
||||
.map((event, index) => {
|
||||
const { excerpt, truncated } = truncateExcerpt(`${event.mutationType} ${event.target}`);
|
||||
return {
|
||||
id: `run-audit-${index + 1}`,
|
||||
source: "runAudit" as const,
|
||||
label: event.mutationType,
|
||||
taskId,
|
||||
runId,
|
||||
timestamp: event.timestamp,
|
||||
eventId: event.id,
|
||||
domain: event.domain,
|
||||
mutationType: event.mutationType,
|
||||
target: event.target,
|
||||
excerpt,
|
||||
truncated,
|
||||
};
|
||||
})
|
||||
.sort(byChronologicalThenId)
|
||||
.slice(-EVIDENCE_LIMITS.runAudit);
|
||||
}
|
||||
|
||||
export async function collectTaskEvaluationEvidence(params: {
|
||||
store: TaskStore;
|
||||
task: TaskDetail;
|
||||
runId: string;
|
||||
cwd: string;
|
||||
}): Promise<TaskEvaluationEvidenceBundle> {
|
||||
const { store, task, runId, cwd } = params;
|
||||
const [documents, agentLogs] = await Promise.all([
|
||||
store.getTaskDocuments(task.id),
|
||||
store.getAgentLogs(task.id, { limit: EVIDENCE_LIMITS.agentLogs }),
|
||||
]);
|
||||
const commitEvidence = await collectCommitEvidence(task, runId, cwd);
|
||||
const workflow = (task.workflowStepResults ?? [])
|
||||
.map((step, index) => {
|
||||
const { excerpt, truncated } = truncateExcerpt(step.output);
|
||||
return {
|
||||
id: `workflow-${index + 1}`,
|
||||
source: "workflow" as const,
|
||||
label: step.workflowStepName,
|
||||
taskId: task.id,
|
||||
runId,
|
||||
timestamp: step.completedAt ?? step.startedAt,
|
||||
workflowStepId: step.workflowStepId,
|
||||
stepName: step.workflowStepName,
|
||||
status: step.status,
|
||||
excerpt,
|
||||
truncated,
|
||||
};
|
||||
})
|
||||
.sort(byChronologicalThenId)
|
||||
.slice(-EVIDENCE_LIMITS.workflow);
|
||||
|
||||
const reviews = (task.log ?? [])
|
||||
.filter((entry) => /review/i.test(entry.action))
|
||||
.map((entry, index) => {
|
||||
const { excerpt, truncated } = truncateExcerpt(entry.outcome);
|
||||
return {
|
||||
id: `review-${index + 1}`,
|
||||
source: "reviews" as const,
|
||||
label: entry.action,
|
||||
taskId: task.id,
|
||||
runId,
|
||||
timestamp: entry.timestamp,
|
||||
verdict: entry.outcome,
|
||||
excerpt,
|
||||
truncated,
|
||||
};
|
||||
})
|
||||
.sort(byChronologicalThenId)
|
||||
.slice(-EVIDENCE_LIMITS.reviews);
|
||||
|
||||
const taskMetadataExcerpt = truncateExcerpt(task.summary ?? task.description);
|
||||
|
||||
return {
|
||||
taskId: task.id,
|
||||
runId,
|
||||
sourceOrder: TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
|
||||
taskMetadata: [{
|
||||
id: "task-metadata-1",
|
||||
source: "taskMetadata",
|
||||
label: task.title ?? task.id,
|
||||
taskId: task.id,
|
||||
runId,
|
||||
timestamp: task.updatedAt,
|
||||
summary: taskMetadataExcerpt.excerpt,
|
||||
excerpt: taskMetadataExcerpt.excerpt,
|
||||
truncated: taskMetadataExcerpt.truncated,
|
||||
references: {
|
||||
prNumber: task.prInfo?.number,
|
||||
prUrl: task.prInfo?.url,
|
||||
mergeCommitSha: task.mergeDetails?.commitSha,
|
||||
mergeCompletedAt: task.mergeDetails?.mergedAt,
|
||||
executionStartedAt: task.executionStartedAt,
|
||||
executionCompletedAt: task.executionCompletedAt,
|
||||
},
|
||||
retryMetrics: {
|
||||
mergeRetries: task.mergeRetries ?? 0,
|
||||
workflowStepRetries: task.workflowStepRetries ?? 0,
|
||||
stuckKillCount: task.stuckKillCount ?? 0,
|
||||
postReviewFixCount: task.postReviewFixCount ?? 0,
|
||||
recoveryRetryCount: task.recoveryRetryCount ?? 0,
|
||||
taskDoneRetryCount: task.taskDoneRetryCount ?? 0,
|
||||
verificationFailureCount: task.verificationFailureCount ?? 0,
|
||||
mergeConflictBounceCount: task.mergeConflictBounceCount ?? 0,
|
||||
},
|
||||
}],
|
||||
commits: commitEvidence,
|
||||
workflow,
|
||||
reviews,
|
||||
documents: collectDocumentEvidence(task.id, runId, documents),
|
||||
taskActivity: collectTaskActivityEvidence(task.id, runId, task.log ?? []),
|
||||
agentLogs: collectAgentLogEvidence(task.id, runId, agentLogs),
|
||||
runAudit: collectRunAuditEvidence(task.id, runId, store.getRunAuditEvents({ taskId: task.id, limit: EVIDENCE_LIMITS.runAudit })),
|
||||
};
|
||||
}
|
||||
@@ -11,7 +11,10 @@ import {
|
||||
type FollowUpDraft,
|
||||
type Settings,
|
||||
type TaskDetail,
|
||||
type TaskEvaluationEvidenceBundle,
|
||||
type TaskStore,
|
||||
} from "@fusion/core";
|
||||
import { collectTaskEvaluationEvidence } from "./evaluator-evidence.js";
|
||||
import { createFnAgent, promptWithFallback } from "./pi.js";
|
||||
import { createLogger } from "./logger.js";
|
||||
|
||||
@@ -29,7 +32,9 @@ export interface EvaluatorModelOverride {
|
||||
|
||||
export interface EvaluatorDeps {
|
||||
cwd: string;
|
||||
store?: TaskStore;
|
||||
runPrompt?: (prompt: string, provider?: string, modelId?: string) => Promise<string>;
|
||||
collectEvidence?: (params: { task: TaskDetail; runId: string; cwd: string; store: TaskStore }) => Promise<TaskEvaluationEvidenceBundle>;
|
||||
}
|
||||
|
||||
interface EvaluatorAiCategoryResponse {
|
||||
@@ -66,7 +71,15 @@ export class HybridEvaluatorService {
|
||||
): Promise<Omit<EvalTaskResultCreateInput, "taskId" | "taskSnapshot">> {
|
||||
const deterministicSignals = collectDeterministicSignals(task, run);
|
||||
const model = resolveEvaluatorModel(settings, modelOverride);
|
||||
const prompt = buildEvaluationPrompt(task, run, deterministicSignals);
|
||||
const evidenceBundle = this.deps.store
|
||||
? await (this.deps.collectEvidence ?? collectTaskEvaluationEvidence)({
|
||||
store: this.deps.store,
|
||||
task,
|
||||
runId: run.runId,
|
||||
cwd: this.deps.cwd,
|
||||
})
|
||||
: undefined;
|
||||
const prompt = buildEvaluationPrompt(task, run, deterministicSignals, evidenceBundle);
|
||||
const responseText = await this.runPrompt(prompt, model.provider, model.modelId);
|
||||
const ai = parseAiResponse(responseText);
|
||||
|
||||
@@ -95,6 +108,7 @@ export class HybridEvaluatorService {
|
||||
rationale: ai.overallRationale,
|
||||
summary: ai.overallRationale,
|
||||
evidence: categoryScores.flatMap((categoryScore) => categoryScore.evidence),
|
||||
evidenceBundle,
|
||||
deterministicSignals: deterministicSignalsToEvalSignals(deterministicSignals),
|
||||
followUps: ai.followUpDrafts.map((draft) => ({
|
||||
title: draft.title,
|
||||
@@ -188,10 +202,30 @@ function deterministicSignalsToEvalSignals(signals: DeterministicSignals): Array
|
||||
];
|
||||
}
|
||||
|
||||
export function buildEvaluationPrompt(task: TaskDetail, run: EvalRunContext, deterministicSignals: DeterministicSignals): string {
|
||||
function formatEvidenceForPrompt(evidenceBundle: TaskEvaluationEvidenceBundle): string {
|
||||
return JSON.stringify({
|
||||
sourceOrder: evidenceBundle.sourceOrder,
|
||||
taskMetadata: evidenceBundle.taskMetadata,
|
||||
commits: evidenceBundle.commits,
|
||||
workflow: evidenceBundle.workflow,
|
||||
reviews: evidenceBundle.reviews,
|
||||
documents: evidenceBundle.documents,
|
||||
taskActivity: evidenceBundle.taskActivity,
|
||||
agentLogs: evidenceBundle.agentLogs,
|
||||
runAudit: evidenceBundle.runAudit,
|
||||
}, null, 2);
|
||||
}
|
||||
|
||||
export function buildEvaluationPrompt(
|
||||
task: TaskDetail,
|
||||
run: EvalRunContext,
|
||||
deterministicSignals: DeterministicSignals,
|
||||
evidenceBundle?: TaskEvaluationEvidenceBundle,
|
||||
): string {
|
||||
return [
|
||||
"Evaluate the completed task and respond with strict JSON.",
|
||||
"Scores must be integers between 0 and 100.",
|
||||
"When citing evidence, use labels that include evidence IDs from the ## Evidence section.",
|
||||
`Run: ${run.runId}`,
|
||||
"Schema:",
|
||||
JSON.stringify({
|
||||
@@ -213,6 +247,8 @@ export function buildEvaluationPrompt(task: TaskDetail, run: EvalRunContext, det
|
||||
}, null, 2),
|
||||
"Deterministic signals:",
|
||||
JSON.stringify(deterministicSignals, null, 2),
|
||||
"## Evidence",
|
||||
evidenceBundle ? formatEvidenceForPrompt(evidenceBundle) : JSON.stringify({ sourceOrder: [], note: "No evidence bundle available" }, null, 2),
|
||||
].join("\n\n");
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ export {
|
||||
export { AgentSemaphore, PRIORITY_MERGE, PRIORITY_EXECUTE, PRIORITY_SPECIFY } from "./concurrency.js";
|
||||
export { TriageProcessor, type TriageProcessorOptions } from "./triage.js";
|
||||
export { TaskExecutor, type TaskExecutorOptions } from "./executor.js";
|
||||
export { collectTaskEvaluationEvidence } from "./evaluator-evidence.js";
|
||||
export { Scheduler, type SchedulerOptions } from "./scheduler.js";
|
||||
export { MissionAutopilot, type MissionAutopilotOptions } from "./mission-autopilot.js";
|
||||
export { MissionExecutionLoop, type MissionExecutionLoopOptions, type ValidationResult, loopLog } from "./mission-execution-loop.js";
|
||||
|
||||
Reference in New Issue
Block a user