root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-3390): export eval score category type and harden evaluator switch

Browse Source 
Exported the eval score category type from `@fusion/core` and added a defensive guard in the evaluator to prevent edge-case failures in the score evaluation switch.

Fusion-Task-Id: FN-3390
This commit is contained in:
Fusion
2026-05-06 11:09:05 -07:00
committed by gsxdsm
parent 828ab8a3bb
commit 8d3ceb9dd0
10 changed files with 484 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
packages/engine/src/__tests__/evaluator.test.ts
View File

@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { createDatabase, EvalStore, runScheduledEvalBatch, type TaskDetail } from "@fusion/core";
import { computeOverallScore, createDatabase, EvalStore, runScheduledEvalBatch, type TaskDetail } from "@fusion/core";
import { HybridEvaluatorService, buildEvaluationPrompt, parseAiResponse, resolveEvaluatorModel } from "../evaluator.js";
function makeTask(overrides: Partial<TaskDetail> = {}): TaskDetail {
@@ -18,6 +18,31 @@ function makeTask(overrides: Partial<TaskDetail> = {}): TaskDetail {
} as TaskDetail;
}
function makeAiResponse(overrides: Partial<Record<string, unknown>> = {}): string {
return JSON.stringify({
categories: {
agentPerformance: {
score: 80,
rationale: "Strong execution decisions.",
evidence: [{ kind: "task", label: "status", value: "done", source: "task" }],
},
taskOutcomeQuality: {
score: 90,
rationale: "Shipped result is complete and correct.",
evidence: [{ kind: "workflow", label: "tests", value: "pass", source: "workflow" }],
},
processCompliance: {
score: 70,
rationale: "Workflow mostly followed.",
evidence: [{ kind: "review", label: "review", value: "approved", source: "review" }],
},
},
overallRationale: "Solid result with complete verification.",
followUpDrafts: [],
...overrides,
});
}
describe("evaluator", () => {
it("resolves explicit complete model override before validator lane", () => {
expect(resolveEvaluatorModel({ validatorProvider: "anthropic", validatorModelId: "claude" }, { provider: "openai", modelId: "gpt-4o" }))
@@ -29,16 +54,51 @@ describe("evaluator", () => {
.toEqual({ provider: "anthropic", modelId: "claude" });
});
it("parses strict AI JSON response", () => {
const parsed = parseAiResponse('{"overallScore":0.8,"categoryScores":{"quality":0.8},"rationale":"ok","evidence":[],"followUpDrafts":[]}');
expect(parsed.overallScore).toBe(0.8);
expect(parsed.rationale).toBe("ok");
it("parses strict AI JSON response with canonical categories", () => {
const parsed = parseAiResponse(makeAiResponse());
expect(parsed.overallRationale).toBe("Solid result with complete verification.");
expect(parsed.categories.agentPerformance.score).toBe(80);
expect(parsed.categories.taskOutcomeQuality.score).toBe(90);
expect(parsed.categories.processCompliance.score).toBe(70);
});
it("throws on malformed AI JSON response", () => {
expect(() => parseAiResponse("not-json")).toThrow(/not valid JSON/);
});
it("rejects invalid evaluator payloads (out of range, missing rationale/evidence, missing category)", () => {
expect(() => parseAiResponse(makeAiResponse({
categories: {
agentPerformance: { score: 101, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
taskOutcomeQuality: { score: 90, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
processCompliance: { score: 80, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
},
}))).toThrow(/agentPerformance score must be an integer in 0..100/);
expect(() => parseAiResponse(makeAiResponse({
categories: {
agentPerformance: { score: 80, rationale: "", evidence: [{ kind: "task", label: "l" }] },
taskOutcomeQuality: { score: 90, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
processCompliance: { score: 80, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
},
}))).toThrow(/agentPerformance rationale is required/);
expect(() => parseAiResponse(makeAiResponse({
categories: {
agentPerformance: { score: 80, rationale: "x", evidence: [] },
taskOutcomeQuality: { score: 90, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
processCompliance: { score: 80, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
},
}))).toThrow(/agentPerformance evidence is required/);
expect(() => parseAiResponse(makeAiResponse({
categories: {
taskOutcomeQuality: { score: 90, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
processCompliance: { score: 80, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
},
}))).toThrow(/missing category agentPerformance/);
});
it("builds prompt with deterministic signal bundle", () => {
const task = makeTask();
const prompt = buildEvaluationPrompt(task, { runId: "ER-1", startedAt: "2026-05-02T00:00:00.000Z" }, {
@@ -56,13 +116,19 @@ describe("evaluator", () => {
it("returns merged evaluation payload shape for persistence", async () => {
const service = new HybridEvaluatorService({
cwd: process.cwd(),
runPrompt: async () => '{"overallScore":0.9,"categoryScores":{"quality":0.9},"rationale":"Great","evidence":[{"kind":"task","label":"done"}],"followUpDrafts":[{"title":"Add tests","description":"More tests","reason":"coverage","evidenceRefs":["task:done"]}]}'
runPrompt: async () => makeAiResponse(),
});
const result = await service.evaluateTask(makeTask(), { runId: "ER-1", startedAt: "2026-05-01T00:00:00.000Z" }, {});
expect(result.status).toBe("scored");
expect(result.overallScore).toBe(0.9);
expect(result.categoryScores?.[0]?.category).toBe("quality");
expect(result.followUps?.[0]?.title).toBe("Add tests");
expect(result.overallScore).toBeGreaterThanOrEqual(0);
expect(result.overallScore).toBeLessThanOrEqual(100);
expect(result.categoryScores).toHaveLength(3);
expect(result.overallScore).toBe(computeOverallScore(result.categoryScores ?? []));
expect(result.categoryScores?.map((score) => score.category)).toEqual([
"agentPerformance",
"taskOutcomeQuality",
"processCompliance",
]);
expect((result.metadata as any).hybridEvaluation).toBeDefined();
});
@@ -74,7 +140,7 @@ describe("evaluator", () => {
const service = new HybridEvaluatorService({
cwd: process.cwd(),
runPrompt: async () => '{"overallScore":0.7,"categoryScores":{"quality":0.7},"rationale":"Solid","evidence":[],"followUpDrafts":[]}'
runPrompt: async () => makeAiResponse(),
});
const mockStore = {
@@ -100,6 +166,9 @@ describe("evaluator", () => {
expect(run2.tasksSelected).toBe(0);
const all = evalStore.listTaskResults({ taskId: doneTask.id });
expect(all).toHaveLength(1);
expect(all[0]?.overallScore).toBe(0.7);
expect(all[0]?.overallScore).toBeGreaterThanOrEqual(0);
expect(all[0]?.overallScore).toBeLessThanOrEqual(100);
expect(all[0]?.categoryScores).toHaveLength(3);
expect(all[0]?.overallScore).toBe(computeOverallScore(all[0]?.categoryScores ?? []));
});
});

103
packages/engine/src/evaluator.ts
View File

@@ -1,7 +1,11 @@
import {
collectDeterministicSignals,
computeOverallScore,
normalizeCategoryScore,
resolveValidatorSettingsModel,
EVAL_SCORE_CATEGORIES,
type DeterministicSignals,
type EvalScoreCategory,
type EvalTaskResultCreateInput,
type EvaluationEvidenceRef,
type FollowUpDraft,
@@ -28,11 +32,15 @@ export interface EvaluatorDeps {
runPrompt?: (prompt: string, provider?: string, modelId?: string) => Promise<string>;
}
interface EvaluatorAiResponse {
overallScore: number;
categoryScores: Record<string, number>;
interface EvaluatorAiCategoryResponse {
score: number;
rationale: string;
evidence: EvaluationEvidenceRef[];
}
interface EvaluatorAiResponse {
categories: Record<EvalScoreCategory, EvaluatorAiCategoryResponse>;
overallRationale: string;
followUpDrafts: FollowUpDraft[];
}
@@ -62,13 +70,31 @@ export class HybridEvaluatorService {
const responseText = await this.runPrompt(prompt, model.provider, model.modelId);
const ai = parseAiResponse(responseText);
const categoryScores = EVAL_SCORE_CATEGORIES.map((category) => {
const aiCategory = ai.categories[category];
return normalizeCategoryScore({
category,
deterministicScore: deriveDeterministicCategoryScore(category, deterministicSignals),
aiScore: aiCategory.score,
rationale: aiCategory.rationale,
evidence: aiCategory.evidence.map((ev) => ({
type: "other",
ref: `${ev.kind}:${ev.label}`,
excerpt: ev.value,
metadata: { source: ev.source },
})),
});
});
const overallScore = computeOverallScore(categoryScores);
return {
status: "scored",
overallScore: ai.overallScore,
categoryScores: Object.entries(ai.categoryScores).map(([category, score]) => ({ category, score })),
rationale: ai.rationale,
summary: ai.rationale,
evidence: ai.evidence.map((ev) => ({ type: "other", ref: `${ev.kind}:${ev.label}`, excerpt: ev.value })),
overallScore,
categoryScores,
rationale: ai.overallRationale,
summary: ai.overallRationale,
evidence: categoryScores.flatMap((categoryScore) => categoryScore.evidence),
deterministicSignals: deterministicSignalsToEvalSignals(deterministicSignals),
followUps: ai.followUpDrafts.map((draft) => ({
title: draft.title,
@@ -78,7 +104,7 @@ export class HybridEvaluatorService {
metadata: {
runId: run.runId,
evaluatorModel: model,
evaluatorRationale: ai.rationale,
evaluatorRationale: ai.overallRationale,
hybridEvaluation: {
deterministicSignals,
ai,
@@ -117,6 +143,27 @@ export class HybridEvaluatorService {
}
}
function deriveDeterministicCategoryScore(category: EvalScoreCategory, signals: DeterministicSignals): number {
const workflowPassRate = signals.workflowSummary.total > 0
? (signals.workflowSummary.passed / signals.workflowSummary.total) * 100
: 50;
const errorPenalty = Math.min(signals.logSummary.errorCount * 20, 60);
const warningPenalty = Math.min(signals.logSummary.warningCount * 5, 25);
const commitScore = Math.min(signals.commitSummary.commitCount * 15, 100);
const reviewScore = signals.reviewStatus === "approved" ? 100 : signals.reviewStatus ? 70 : 50;
switch (category) {
case "agentPerformance":
return Math.round(Math.max(0, Math.min(100, (workflowPassRate * 0.5) + (reviewScore * 0.3) + (commitScore * 0.2) - warningPenalty)));
case "taskOutcomeQuality":
return Math.round(Math.max(0, Math.min(100, (workflowPassRate * 0.6) + (commitScore * 0.2) + (100 - errorPenalty) * 0.2)));
case "processCompliance":
return Math.round(Math.max(0, Math.min(100, (workflowPassRate * 0.5) + (reviewScore * 0.3) + ((signals.logSummary.timingEntries > 0 ? 100 : 50) * 0.2) - errorPenalty)));
default:
throw new Error(`Unsupported eval score category: ${String(category)}`);
}
}
function deterministicSignalsToEvalSignals(signals: DeterministicSignals): Array<{ signalId: string; kind: string; name: string; value?: string | number; passed?: boolean }> {
return [
{
@@ -144,13 +191,16 @@ function deterministicSignalsToEvalSignals(signals: DeterministicSignals): Array
export function buildEvaluationPrompt(task: TaskDetail, run: EvalRunContext, deterministicSignals: DeterministicSignals): string {
return [
"Evaluate the completed task and respond with strict JSON.",
"Scores must be integers between 0 and 100.",
`Run: ${run.runId}`,
"Schema:",
JSON.stringify({
overallScore: 0,
categoryScores: { quality: 0, reliability: 0, testing: 0 },
rationale: "",
evidence: [{ kind: "task", label: "", value: "", source: "" }],
categories: {
agentPerformance: { score: 0, rationale: "", evidence: [{ kind: "task", label: "", value: "", source: "" }] },
taskOutcomeQuality: { score: 0, rationale: "", evidence: [{ kind: "task", label: "", value: "", source: "" }] },
processCompliance: { score: 0, rationale: "", evidence: [{ kind: "task", label: "", value: "", source: "" }] },
},
overallRationale: "",
followUpDrafts: [{ title: "", description: "", reason: "", evidenceRefs: [] }],
}, null, 2),
"Task:",
@@ -176,15 +226,28 @@ export function parseAiResponse(raw: string): EvaluatorAiResponse {
}
const record = parsed as Partial<EvaluatorAiResponse>;
if (typeof record.overallScore !== "number") throw new Error("Evaluator response missing numeric overallScore");
if (!record.categoryScores || typeof record.categoryScores !== "object") throw new Error("Evaluator response missing categoryScores");
if (typeof record.rationale !== "string" || !record.rationale.trim()) throw new Error("Evaluator response missing rationale");
if (!record.categories || typeof record.categories !== "object") throw new Error("Evaluator response missing categories");
if (typeof record.overallRationale !== "string" || !record.overallRationale.trim()) throw new Error("Evaluator response missing overallRationale");
const categories = {} as Record<EvalScoreCategory, EvaluatorAiCategoryResponse>;
for (const category of EVAL_SCORE_CATEGORIES) {
const entry = (record.categories as Record<string, EvaluatorAiCategoryResponse>)[category];
if (!entry) throw new Error(`Evaluator response missing category ${category}`);
if (!Number.isInteger(entry.score) || entry.score < 0 || entry.score > 100) {
throw new Error(`Evaluator category ${category} score must be an integer in 0..100`);
}
if (typeof entry.rationale !== "string" || !entry.rationale.trim()) {
throw new Error(`Evaluator category ${category} rationale is required`);
}
if (!Array.isArray(entry.evidence) || entry.evidence.length === 0) {
throw new Error(`Evaluator category ${category} evidence is required`);
}
categories[category] = entry;
}
return {
overallScore: record.overallScore,
categoryScores: record.categoryScores as Record<string, number>,
rationale: record.rationale,
evidence: Array.isArray(record.evidence) ? record.evidence : [],
categories,
overallRationale: record.overallRationale,
followUpDrafts: Array.isArray(record.followUpDrafts) ? record.followUpDrafts : [],
};
}