root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-3390): export eval score category type and harden evaluator switch

Browse Source 
Exported the eval score category type from `@fusion/core` and added a defensive guard in the evaluator to prevent edge-case failures in the score evaluation switch.

Fusion-Task-Id: FN-3390
This commit is contained in:
Fusion
2026-05-06 11:09:05 -07:00
committed by gsxdsm
parent 828ab8a3bb
commit 8d3ceb9dd0
10 changed files with 484 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
packages/core/src/__tests__/eval-scoring.test.ts Normal file
View File

@@ -0,0 +1,61 @@
import { describe, expect, it } from "vitest";
import {
computeCategoryFinalScore,
computeOverallScore,
normalizeCategoryScore,
resolveScoreBand,
} from "../eval-scoring.js";
describe("eval-scoring", () => {
it("resolves score bands at boundaries", () => {
expect(resolveScoreBand(39)).toBe("failing");
expect(resolveScoreBand(40)).toBe("weak");
expect(resolveScoreBand(59)).toBe("weak");
expect(resolveScoreBand(60)).toBe("acceptable");
expect(resolveScoreBand(74)).toBe("acceptable");
expect(resolveScoreBand(75)).toBe("strong");
expect(resolveScoreBand(89)).toBe("strong");
expect(resolveScoreBand(90)).toBe("excellent");
expect(resolveScoreBand(100)).toBe("excellent");
});
it("computes deterministic/AI blend with canonical 70/30 rule", () => {
expect(computeCategoryFinalScore(0, 100)).toBe(30);
expect(computeCategoryFinalScore(100, 0)).toBe(70);
expect(computeCategoryFinalScore(73, 89)).toBe(78);
});
it("computes overall weighted score with canonical category weights", () => {
const categories = [
normalizeCategoryScore({
category: "agentPerformance",
deterministicScore: 80,
aiScore: 80,
rationale: "solid execution",
evidence: [],
}),
normalizeCategoryScore({
category: "taskOutcomeQuality",
deterministicScore: 90,
aiScore: 90,
rationale: "high quality",
evidence: [],
}),
normalizeCategoryScore({
category: "processCompliance",
deterministicScore: 70,
aiScore: 70,
rationale: "mostly compliant",
evidence: [],
}),
];
expect(computeOverallScore(categories)).toBe(82);
});
it("rejects invalid input scores", () => {
expect(() => computeCategoryFinalScore(-1, 50)).toThrow(/deterministicScore/);
expect(() => computeCategoryFinalScore(50, 101)).toThrow(/aiScore/);
expect(() => resolveScoreBand(101)).toThrow(/integer between/);
});
});

42
packages/core/src/__tests__/eval-store.test.ts
View File

@@ -40,8 +40,35 @@ describe("EvalStore", () => {
taskId: "FN-123",
taskSnapshot: { taskId: "FN-123", title: "Snapshot title", status: "done", summary: "task summary" },
status: "scored",
overallScore: 0.8,
categoryScores: [{ category: "quality", score: 0.8 }],
overallScore: 80,
categoryScores: [{
category: "agentPerformance",
deterministicScore: 78,
aiScore: 82,
finalScore: 79,
weight: 0.3,
band: "strong",
rationale: "handled execution well",
evidence: [{ type: "task_log", ref: "log:1" }],
}, {
category: "taskOutcomeQuality",
deterministicScore: 80,
aiScore: 80,
finalScore: 80,
weight: 0.45,
band: "strong",
rationale: "good",
evidence: [{ type: "test", ref: "test:all" }],
}, {
category: "processCompliance",
deterministicScore: 72,
aiScore: 76,
finalScore: 73,
weight: 0.25,
band: "acceptable",
rationale: "mostly compliant",
evidence: [{ type: "other", ref: "workflow:review" }],
}],
evidence: [{ type: "task_log", ref: "log:1" }],
deterministicSignals: [{ signalId: "s1", kind: "test", name: "tests-pass", passed: true }],
});
@@ -51,6 +78,11 @@ describe("EvalStore", () => {
const fetched = store.getTaskResult(result.id);
expect(fetched?.taskSnapshot.title).toBe("Snapshot title");
expect(fetched?.taskId).toBe("FN-123");
expect(fetched?.categoryScores).toHaveLength(3);
expect(fetched?.categoryScores[0]?.category).toBe("agentPerformance");
expect(fetched?.categoryScores[0]?.deterministicScore).toBe(78);
expect(fetched?.categoryScores[1]?.weight).toBe(0.45);
expect(fetched?.categoryScores[2]?.band).toBe("acceptable");
});
it("persists run window boundaries and evaluated task rollups", () => {
@@ -79,18 +111,18 @@ describe("EvalStore", () => {
taskId: "FN-dup",
taskSnapshot: { taskId: "FN-dup", title: "A" },
status: "scored",
overallScore: 0.2,
overallScore: 20,
});
const second = store.createTaskResult(run.id, {
taskId: "FN-dup",
taskSnapshot: { taskId: "FN-dup", title: "B" },
status: "scored",
overallScore: 0.9,
overallScore: 90,
});
const rows = store.listTaskResults({ runId: run.id, taskId: "FN-dup" });
expect(rows).toHaveLength(1);
expect(rows[0]?.overallScore).toBe(0.9);
expect(rows[0]?.overallScore).toBe(90);
expect(rows[0]?.taskSnapshot.title).toBe("B");
expect(second.id).toBe(first.id);
});

98
packages/core/src/eval-scoring.ts Normal file
View File

@@ -0,0 +1,98 @@
import {
EVAL_SCORE_BANDS,
EVAL_SCORE_CATEGORIES,
EVAL_SCORE_SCALE_MAX,
EVAL_SCORE_SCALE_MIN,
type EvalCategoryScore,
type EvalScoreBand,
type EvalScoreCategory,
} from "./eval-types.js";
export const EVAL_CATEGORY_WEIGHTS: Record<EvalScoreCategory, number> = {
agentPerformance: 0.30,
taskOutcomeQuality: 0.45,
processCompliance: 0.25,
};
const DETERMINISTIC_WEIGHT = 0.7;
const AI_WEIGHT = 0.3;
export function clampScore(value: number): number {
return Math.min(EVAL_SCORE_SCALE_MAX, Math.max(EVAL_SCORE_SCALE_MIN, value));
}
export function assertValidScore(value: number, fieldName = "score"): void {
if (!Number.isInteger(value) || value < EVAL_SCORE_SCALE_MIN || value > EVAL_SCORE_SCALE_MAX) {
throw new Error(`${fieldName} must be an integer between ${EVAL_SCORE_SCALE_MIN} and ${EVAL_SCORE_SCALE_MAX}`);
}
}
export function resolveScoreBand(score: number): EvalScoreBand {
assertValidScore(score);
for (const band of EVAL_SCORE_BANDS) {
if (score >= band.min && score <= band.max) {
return band.id;
}
}
throw new Error(`No score band for score ${score}`);
}
export function computeCategoryFinalScore(deterministicScore: number, aiScore: number): number {
assertValidScore(deterministicScore, "deterministicScore");
assertValidScore(aiScore, "aiScore");
return Math.round(clampScore((deterministicScore * DETERMINISTIC_WEIGHT) + (aiScore * AI_WEIGHT)));
}
export function normalizeCategoryScore(input: {
category: EvalScoreCategory;
deterministicScore: number;
aiScore: number;
rationale: string;
evidence: EvalCategoryScore["evidence"];
}): EvalCategoryScore {
const { category, deterministicScore, aiScore, rationale, evidence } = input;
if (!EVAL_SCORE_CATEGORIES.includes(category)) {
throw new Error(`Unknown score category: ${category}`);
}
if (!rationale.trim()) {
throw new Error(`rationale is required for ${category}`);
}
const finalScore = computeCategoryFinalScore(deterministicScore, aiScore);
return {
category,
deterministicScore,
aiScore,
finalScore,
weight: EVAL_CATEGORY_WEIGHTS[category],
band: resolveScoreBand(finalScore),
rationale,
evidence,
};
}
export function computeOverallScore(categoryScores: EvalCategoryScore[]): number {
if (categoryScores.length !== EVAL_SCORE_CATEGORIES.length) {
throw new Error(`Expected ${EVAL_SCORE_CATEGORIES.length} category scores`);
}
const byCategory = new Map<EvalScoreCategory, EvalCategoryScore>();
for (const categoryScore of categoryScores) {
if (!EVAL_SCORE_CATEGORIES.includes(categoryScore.category)) {
throw new Error(`Unknown score category: ${categoryScore.category}`);
}
byCategory.set(categoryScore.category, categoryScore);
}
let weightedSum = 0;
for (const category of EVAL_SCORE_CATEGORIES) {
const score = byCategory.get(category);
if (!score) {
throw new Error(`Missing category score: ${category}`);
}
assertValidScore(score.finalScore, `${category}.finalScore`);
weightedSum += score.finalScore * EVAL_CATEGORY_WEIGHTS[category];
}
return Math.round(clampScore(weightedSum));
}

36
packages/core/src/eval-types.ts
View File

@@ -19,16 +19,26 @@ export const EVAL_RUN_TRIGGERS = ["manual", "schedule", "api", "task_completion"
export type EvalRunTrigger = typeof EVAL_RUN_TRIGGERS[number];
export const EVAL_SCORE_CATEGORIES = [
"correctness",
"completeness",
"quality",
"reliability",
"tests",
"documentation",
"agentPerformance",
"taskOutcomeQuality",
"processCompliance",
] as const;
export type EvalScoreCategory = typeof EVAL_SCORE_CATEGORIES[number];
export const EVAL_SCORE_SCALE_MIN = 0;
export const EVAL_SCORE_SCALE_MAX = 100;
export const EVAL_SCORE_BANDS = [
{ id: "failing", min: 0, max: 39 },
{ id: "weak", min: 40, max: 59 },
{ id: "acceptable", min: 60, max: 74 },
{ id: "strong", min: 75, max: 89 },
{ id: "excellent", min: 90, max: 100 },
] as const;
export type EvalScoreBand = typeof EVAL_SCORE_BANDS[number]["id"];
export interface EvalTaskSnapshot {
taskId: string;
title?: string;
@@ -83,10 +93,14 @@ export interface EvalEvidenceReference {
}
export interface EvalCategoryScore {
category: EvalScoreCategory | string;
score: number;
maxScore?: number;
rationale?: string;
category: EvalScoreCategory;
deterministicScore: number;
aiScore: number;
finalScore: number;
weight: number;
band: EvalScoreBand;
rationale: string;
evidence: EvalEvidenceReference[];
}
export interface EvalFollowUpSuggestion {
@@ -264,7 +278,7 @@ export interface TaskEvaluation {
taskId: string;
deterministicSignals: DeterministicSignals;
overallScore: number;
categoryScores: Record<string, number>;
categoryScores: EvalCategoryScore[];
rationale: string;
evidence: EvaluationEvidenceRef[];
followUpDrafts: FollowUpDraft[];

20
packages/core/src/index.ts
View File

@@ -725,6 +725,8 @@ export type {
EvalTaskResultCreateInput,
EvalTaskResultUpdateInput,
EvalTaskResultListOptions,
EvalScoreBand,
EvalScoreCategory,
EvalCategoryScore,
EvalEvidenceReference,
EvalSignal,
@@ -736,7 +738,23 @@ export type {
FollowUpDraft,
TaskEvaluation,
} from "./eval-types.js";
export { EVAL_RUN_STATUSES, EVAL_RUN_TRIGGERS, EVAL_SCORE_CATEGORIES } from "./eval-types.js";
export {
EVAL_RUN_STATUSES,
EVAL_RUN_TRIGGERS,
EVAL_SCORE_CATEGORIES,
EVAL_SCORE_BANDS,
EVAL_SCORE_SCALE_MIN,
EVAL_SCORE_SCALE_MAX,
} from "./eval-types.js";
export {
EVAL_CATEGORY_WEIGHTS,
assertValidScore,
clampScore,
computeCategoryFinalScore,
computeOverallScore,
normalizeCategoryScore,
resolveScoreBand,
} from "./eval-scoring.js";
export {
TASK_EVALUATION_SCHEDULE_NAME,
DEFAULT_TASK_EVALUATION_SCHEDULE,

91
packages/engine/src/__tests__/evaluator.test.ts
View File

@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { createDatabase, EvalStore, runScheduledEvalBatch, type TaskDetail } from "@fusion/core";
import { computeOverallScore, createDatabase, EvalStore, runScheduledEvalBatch, type TaskDetail } from "@fusion/core";
import { HybridEvaluatorService, buildEvaluationPrompt, parseAiResponse, resolveEvaluatorModel } from "../evaluator.js";
function makeTask(overrides: Partial<TaskDetail> = {}): TaskDetail {
@@ -18,6 +18,31 @@ function makeTask(overrides: Partial<TaskDetail> = {}): TaskDetail {
} as TaskDetail;
}
function makeAiResponse(overrides: Partial<Record<string, unknown>> = {}): string {
return JSON.stringify({
categories: {
agentPerformance: {
score: 80,
rationale: "Strong execution decisions.",
evidence: [{ kind: "task", label: "status", value: "done", source: "task" }],
},
taskOutcomeQuality: {
score: 90,
rationale: "Shipped result is complete and correct.",
evidence: [{ kind: "workflow", label: "tests", value: "pass", source: "workflow" }],
},
processCompliance: {
score: 70,
rationale: "Workflow mostly followed.",
evidence: [{ kind: "review", label: "review", value: "approved", source: "review" }],
},
},
overallRationale: "Solid result with complete verification.",
followUpDrafts: [],
...overrides,
});
}
describe("evaluator", () => {
it("resolves explicit complete model override before validator lane", () => {
expect(resolveEvaluatorModel({ validatorProvider: "anthropic", validatorModelId: "claude" }, { provider: "openai", modelId: "gpt-4o" }))
@@ -29,16 +54,51 @@ describe("evaluator", () => {
.toEqual({ provider: "anthropic", modelId: "claude" });
});
it("parses strict AI JSON response", () => {
const parsed = parseAiResponse('{"overallScore":0.8,"categoryScores":{"quality":0.8},"rationale":"ok","evidence":[],"followUpDrafts":[]}');
expect(parsed.overallScore).toBe(0.8);
expect(parsed.rationale).toBe("ok");
it("parses strict AI JSON response with canonical categories", () => {
const parsed = parseAiResponse(makeAiResponse());
expect(parsed.overallRationale).toBe("Solid result with complete verification.");
expect(parsed.categories.agentPerformance.score).toBe(80);
expect(parsed.categories.taskOutcomeQuality.score).toBe(90);
expect(parsed.categories.processCompliance.score).toBe(70);
});
it("throws on malformed AI JSON response", () => {
expect(() => parseAiResponse("not-json")).toThrow(/not valid JSON/);
});
it("rejects invalid evaluator payloads (out of range, missing rationale/evidence, missing category)", () => {
expect(() => parseAiResponse(makeAiResponse({
categories: {
agentPerformance: { score: 101, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
taskOutcomeQuality: { score: 90, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
processCompliance: { score: 80, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
},
}))).toThrow(/agentPerformance score must be an integer in 0..100/);
expect(() => parseAiResponse(makeAiResponse({
categories: {
agentPerformance: { score: 80, rationale: "", evidence: [{ kind: "task", label: "l" }] },
taskOutcomeQuality: { score: 90, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
processCompliance: { score: 80, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
},
}))).toThrow(/agentPerformance rationale is required/);
expect(() => parseAiResponse(makeAiResponse({
categories: {
agentPerformance: { score: 80, rationale: "x", evidence: [] },
taskOutcomeQuality: { score: 90, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
processCompliance: { score: 80, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
},
}))).toThrow(/agentPerformance evidence is required/);
expect(() => parseAiResponse(makeAiResponse({
categories: {
taskOutcomeQuality: { score: 90, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
processCompliance: { score: 80, rationale: "x", evidence: [{ kind: "task", label: "l" }] },
},
}))).toThrow(/missing category agentPerformance/);
});
it("builds prompt with deterministic signal bundle", () => {
const task = makeTask();
const prompt = buildEvaluationPrompt(task, { runId: "ER-1", startedAt: "2026-05-02T00:00:00.000Z" }, {
@@ -56,13 +116,19 @@ describe("evaluator", () => {
it("returns merged evaluation payload shape for persistence", async () => {
const service = new HybridEvaluatorService({
cwd: process.cwd(),
runPrompt: async () => '{"overallScore":0.9,"categoryScores":{"quality":0.9},"rationale":"Great","evidence":[{"kind":"task","label":"done"}],"followUpDrafts":[{"title":"Add tests","description":"More tests","reason":"coverage","evidenceRefs":["task:done"]}]}'
runPrompt: async () => makeAiResponse(),
});
const result = await service.evaluateTask(makeTask(), { runId: "ER-1", startedAt: "2026-05-01T00:00:00.000Z" }, {});
expect(result.status).toBe("scored");
expect(result.overallScore).toBe(0.9);
expect(result.categoryScores?.[0]?.category).toBe("quality");
expect(result.followUps?.[0]?.title).toBe("Add tests");
expect(result.overallScore).toBeGreaterThanOrEqual(0);
expect(result.overallScore).toBeLessThanOrEqual(100);
expect(result.categoryScores).toHaveLength(3);
expect(result.overallScore).toBe(computeOverallScore(result.categoryScores ?? []));
expect(result.categoryScores?.map((score) => score.category)).toEqual([
"agentPerformance",
"taskOutcomeQuality",
"processCompliance",
]);
expect((result.metadata as any).hybridEvaluation).toBeDefined();
});
@@ -74,7 +140,7 @@ describe("evaluator", () => {
const service = new HybridEvaluatorService({
cwd: process.cwd(),
runPrompt: async () => '{"overallScore":0.7,"categoryScores":{"quality":0.7},"rationale":"Solid","evidence":[],"followUpDrafts":[]}'
runPrompt: async () => makeAiResponse(),
});
const mockStore = {
@@ -100,6 +166,9 @@ describe("evaluator", () => {
expect(run2.tasksSelected).toBe(0);
const all = evalStore.listTaskResults({ taskId: doneTask.id });
expect(all).toHaveLength(1);
expect(all[0]?.overallScore).toBe(0.7);
expect(all[0]?.overallScore).toBeGreaterThanOrEqual(0);
expect(all[0]?.overallScore).toBeLessThanOrEqual(100);
expect(all[0]?.categoryScores).toHaveLength(3);
expect(all[0]?.overallScore).toBe(computeOverallScore(all[0]?.categoryScores ?? []));
});
});

103
packages/engine/src/evaluator.ts
View File

@@ -1,7 +1,11 @@
import {
collectDeterministicSignals,
computeOverallScore,
normalizeCategoryScore,
resolveValidatorSettingsModel,
EVAL_SCORE_CATEGORIES,
type DeterministicSignals,
type EvalScoreCategory,
type EvalTaskResultCreateInput,
type EvaluationEvidenceRef,
type FollowUpDraft,
@@ -28,11 +32,15 @@ export interface EvaluatorDeps {
runPrompt?: (prompt: string, provider?: string, modelId?: string) => Promise<string>;
}
interface EvaluatorAiResponse {
overallScore: number;
categoryScores: Record<string, number>;
interface EvaluatorAiCategoryResponse {
score: number;
rationale: string;
evidence: EvaluationEvidenceRef[];
}
interface EvaluatorAiResponse {
categories: Record<EvalScoreCategory, EvaluatorAiCategoryResponse>;
overallRationale: string;
followUpDrafts: FollowUpDraft[];
}
@@ -62,13 +70,31 @@ export class HybridEvaluatorService {
const responseText = await this.runPrompt(prompt, model.provider, model.modelId);
const ai = parseAiResponse(responseText);
const categoryScores = EVAL_SCORE_CATEGORIES.map((category) => {
const aiCategory = ai.categories[category];
return normalizeCategoryScore({
category,
deterministicScore: deriveDeterministicCategoryScore(category, deterministicSignals),
aiScore: aiCategory.score,
rationale: aiCategory.rationale,
evidence: aiCategory.evidence.map((ev) => ({
type: "other",
ref: `${ev.kind}:${ev.label}`,
excerpt: ev.value,
metadata: { source: ev.source },
})),
});
});
const overallScore = computeOverallScore(categoryScores);
return {
status: "scored",
overallScore: ai.overallScore,
categoryScores: Object.entries(ai.categoryScores).map(([category, score]) => ({ category, score })),
rationale: ai.rationale,
summary: ai.rationale,
evidence: ai.evidence.map((ev) => ({ type: "other", ref: `${ev.kind}:${ev.label}`, excerpt: ev.value })),
overallScore,
categoryScores,
rationale: ai.overallRationale,
summary: ai.overallRationale,
evidence: categoryScores.flatMap((categoryScore) => categoryScore.evidence),
deterministicSignals: deterministicSignalsToEvalSignals(deterministicSignals),
followUps: ai.followUpDrafts.map((draft) => ({
title: draft.title,
@@ -78,7 +104,7 @@ export class HybridEvaluatorService {
metadata: {
runId: run.runId,
evaluatorModel: model,
evaluatorRationale: ai.rationale,
evaluatorRationale: ai.overallRationale,
hybridEvaluation: {
deterministicSignals,
ai,
@@ -117,6 +143,27 @@ export class HybridEvaluatorService {
}
}
function deriveDeterministicCategoryScore(category: EvalScoreCategory, signals: DeterministicSignals): number {
const workflowPassRate = signals.workflowSummary.total > 0
? (signals.workflowSummary.passed / signals.workflowSummary.total) * 100
: 50;
const errorPenalty = Math.min(signals.logSummary.errorCount * 20, 60);
const warningPenalty = Math.min(signals.logSummary.warningCount * 5, 25);
const commitScore = Math.min(signals.commitSummary.commitCount * 15, 100);
const reviewScore = signals.reviewStatus === "approved" ? 100 : signals.reviewStatus ? 70 : 50;
switch (category) {
case "agentPerformance":
return Math.round(Math.max(0, Math.min(100, (workflowPassRate * 0.5) + (reviewScore * 0.3) + (commitScore * 0.2) - warningPenalty)));
case "taskOutcomeQuality":
return Math.round(Math.max(0, Math.min(100, (workflowPassRate * 0.6) + (commitScore * 0.2) + (100 - errorPenalty) * 0.2)));
case "processCompliance":
return Math.round(Math.max(0, Math.min(100, (workflowPassRate * 0.5) + (reviewScore * 0.3) + ((signals.logSummary.timingEntries > 0 ? 100 : 50) * 0.2) - errorPenalty)));
default:
throw new Error(`Unsupported eval score category: ${String(category)}`);
}
}
function deterministicSignalsToEvalSignals(signals: DeterministicSignals): Array<{ signalId: string; kind: string; name: string; value?: string | number; passed?: boolean }> {
return [
{
@@ -144,13 +191,16 @@ function deterministicSignalsToEvalSignals(signals: DeterministicSignals): Array
export function buildEvaluationPrompt(task: TaskDetail, run: EvalRunContext, deterministicSignals: DeterministicSignals): string {
return [
"Evaluate the completed task and respond with strict JSON.",
"Scores must be integers between 0 and 100.",
`Run: ${run.runId}`,
"Schema:",
JSON.stringify({
overallScore: 0,
categoryScores: { quality: 0, reliability: 0, testing: 0 },
rationale: "",
evidence: [{ kind: "task", label: "", value: "", source: "" }],
categories: {
agentPerformance: { score: 0, rationale: "", evidence: [{ kind: "task", label: "", value: "", source: "" }] },
taskOutcomeQuality: { score: 0, rationale: "", evidence: [{ kind: "task", label: "", value: "", source: "" }] },
processCompliance: { score: 0, rationale: "", evidence: [{ kind: "task", label: "", value: "", source: "" }] },
},
overallRationale: "",
followUpDrafts: [{ title: "", description: "", reason: "", evidenceRefs: [] }],
}, null, 2),
"Task:",
@@ -176,15 +226,28 @@ export function parseAiResponse(raw: string): EvaluatorAiResponse {
}
const record = parsed as Partial<EvaluatorAiResponse>;
if (typeof record.overallScore !== "number") throw new Error("Evaluator response missing numeric overallScore");
if (!record.categoryScores || typeof record.categoryScores !== "object") throw new Error("Evaluator response missing categoryScores");
if (typeof record.rationale !== "string" || !record.rationale.trim()) throw new Error("Evaluator response missing rationale");
if (!record.categories || typeof record.categories !== "object") throw new Error("Evaluator response missing categories");
if (typeof record.overallRationale !== "string" || !record.overallRationale.trim()) throw new Error("Evaluator response missing overallRationale");
const categories = {} as Record<EvalScoreCategory, EvaluatorAiCategoryResponse>;
for (const category of EVAL_SCORE_CATEGORIES) {
const entry = (record.categories as Record<string, EvaluatorAiCategoryResponse>)[category];
if (!entry) throw new Error(`Evaluator response missing category ${category}`);
if (!Number.isInteger(entry.score) || entry.score < 0 || entry.score > 100) {
throw new Error(`Evaluator category ${category} score must be an integer in 0..100`);
}
if (typeof entry.rationale !== "string" || !entry.rationale.trim()) {
throw new Error(`Evaluator category ${category} rationale is required`);
}
if (!Array.isArray(entry.evidence) || entry.evidence.length === 0) {
throw new Error(`Evaluator category ${category} evidence is required`);
}
categories[category] = entry;
}
return {
overallScore: record.overallScore,
categoryScores: record.categoryScores as Record<string, number>,
rationale: record.rationale,
evidence: Array.isArray(record.evidence) ? record.evidence : [],
categories,
overallRationale: record.overallRationale,
followUpDrafts: Array.isArray(record.followUpDrafts) ? record.followUpDrafts : [],
};
}