root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-3390): export eval score category type and harden evaluator switch

Browse Source 
Exported the eval score category type from `@fusion/core` and added a defensive guard in the evaluator to prevent edge-case failures in the score evaluation switch.

Fusion-Task-Id: FN-3390
This commit is contained in:
Fusion
2026-05-06 11:09:05 -07:00
committed by gsxdsm
parent 828ab8a3bb
commit 8d3ceb9dd0
10 changed files with 484 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
packages/core/src/__tests__/eval-scoring.test.ts Normal file
View File

@@ -0,0 +1,61 @@
import { describe, expect, it } from "vitest";
import {
computeCategoryFinalScore,
computeOverallScore,
normalizeCategoryScore,
resolveScoreBand,
} from "../eval-scoring.js";
describe("eval-scoring", () => {
it("resolves score bands at boundaries", () => {
expect(resolveScoreBand(39)).toBe("failing");
expect(resolveScoreBand(40)).toBe("weak");
expect(resolveScoreBand(59)).toBe("weak");
expect(resolveScoreBand(60)).toBe("acceptable");
expect(resolveScoreBand(74)).toBe("acceptable");
expect(resolveScoreBand(75)).toBe("strong");
expect(resolveScoreBand(89)).toBe("strong");
expect(resolveScoreBand(90)).toBe("excellent");
expect(resolveScoreBand(100)).toBe("excellent");
});
it("computes deterministic/AI blend with canonical 70/30 rule", () => {
expect(computeCategoryFinalScore(0, 100)).toBe(30);
expect(computeCategoryFinalScore(100, 0)).toBe(70);
expect(computeCategoryFinalScore(73, 89)).toBe(78);
});
it("computes overall weighted score with canonical category weights", () => {
const categories = [
normalizeCategoryScore({
category: "agentPerformance",
deterministicScore: 80,
aiScore: 80,
rationale: "solid execution",
evidence: [],
}),
normalizeCategoryScore({
category: "taskOutcomeQuality",
deterministicScore: 90,
aiScore: 90,
rationale: "high quality",
evidence: [],
}),
normalizeCategoryScore({
category: "processCompliance",
deterministicScore: 70,
aiScore: 70,
rationale: "mostly compliant",
evidence: [],
}),
];
expect(computeOverallScore(categories)).toBe(82);
});
it("rejects invalid input scores", () => {
expect(() => computeCategoryFinalScore(-1, 50)).toThrow(/deterministicScore/);
expect(() => computeCategoryFinalScore(50, 101)).toThrow(/aiScore/);
expect(() => resolveScoreBand(101)).toThrow(/integer between/);
});
});

42
packages/core/src/__tests__/eval-store.test.ts
View File

@@ -40,8 +40,35 @@ describe("EvalStore", () => {
taskId: "FN-123",
taskSnapshot: { taskId: "FN-123", title: "Snapshot title", status: "done", summary: "task summary" },
status: "scored",
overallScore: 0.8,
categoryScores: [{ category: "quality", score: 0.8 }],
overallScore: 80,
categoryScores: [{
category: "agentPerformance",
deterministicScore: 78,
aiScore: 82,
finalScore: 79,
weight: 0.3,
band: "strong",
rationale: "handled execution well",
evidence: [{ type: "task_log", ref: "log:1" }],
}, {
category: "taskOutcomeQuality",
deterministicScore: 80,
aiScore: 80,
finalScore: 80,
weight: 0.45,
band: "strong",
rationale: "good",
evidence: [{ type: "test", ref: "test:all" }],
}, {
category: "processCompliance",
deterministicScore: 72,
aiScore: 76,
finalScore: 73,
weight: 0.25,
band: "acceptable",
rationale: "mostly compliant",
evidence: [{ type: "other", ref: "workflow:review" }],
}],
evidence: [{ type: "task_log", ref: "log:1" }],
deterministicSignals: [{ signalId: "s1", kind: "test", name: "tests-pass", passed: true }],
});
@@ -51,6 +78,11 @@ describe("EvalStore", () => {
const fetched = store.getTaskResult(result.id);
expect(fetched?.taskSnapshot.title).toBe("Snapshot title");
expect(fetched?.taskId).toBe("FN-123");
expect(fetched?.categoryScores).toHaveLength(3);
expect(fetched?.categoryScores[0]?.category).toBe("agentPerformance");
expect(fetched?.categoryScores[0]?.deterministicScore).toBe(78);
expect(fetched?.categoryScores[1]?.weight).toBe(0.45);
expect(fetched?.categoryScores[2]?.band).toBe("acceptable");
});
it("persists run window boundaries and evaluated task rollups", () => {
@@ -79,18 +111,18 @@ describe("EvalStore", () => {
taskId: "FN-dup",
taskSnapshot: { taskId: "FN-dup", title: "A" },
status: "scored",
overallScore: 0.2,
overallScore: 20,
});
const second = store.createTaskResult(run.id, {
taskId: "FN-dup",
taskSnapshot: { taskId: "FN-dup", title: "B" },
status: "scored",
overallScore: 0.9,
overallScore: 90,
});
const rows = store.listTaskResults({ runId: run.id, taskId: "FN-dup" });
expect(rows).toHaveLength(1);
expect(rows[0]?.overallScore).toBe(0.9);
expect(rows[0]?.overallScore).toBe(90);
expect(rows[0]?.taskSnapshot.title).toBe("B");
expect(second.id).toBe(first.id);
});

98
packages/core/src/eval-scoring.ts Normal file
View File

@@ -0,0 +1,98 @@
import {
EVAL_SCORE_BANDS,
EVAL_SCORE_CATEGORIES,
EVAL_SCORE_SCALE_MAX,
EVAL_SCORE_SCALE_MIN,
type EvalCategoryScore,
type EvalScoreBand,
type EvalScoreCategory,
} from "./eval-types.js";
export const EVAL_CATEGORY_WEIGHTS: Record<EvalScoreCategory, number> = {
agentPerformance: 0.30,
taskOutcomeQuality: 0.45,
processCompliance: 0.25,
};
const DETERMINISTIC_WEIGHT = 0.7;
const AI_WEIGHT = 0.3;
export function clampScore(value: number): number {
return Math.min(EVAL_SCORE_SCALE_MAX, Math.max(EVAL_SCORE_SCALE_MIN, value));
}
export function assertValidScore(value: number, fieldName = "score"): void {
if (!Number.isInteger(value) || value < EVAL_SCORE_SCALE_MIN || value > EVAL_SCORE_SCALE_MAX) {
throw new Error(`${fieldName} must be an integer between ${EVAL_SCORE_SCALE_MIN} and ${EVAL_SCORE_SCALE_MAX}`);
}
}
export function resolveScoreBand(score: number): EvalScoreBand {
assertValidScore(score);
for (const band of EVAL_SCORE_BANDS) {
if (score >= band.min && score <= band.max) {
return band.id;
}
}
throw new Error(`No score band for score ${score}`);
}
export function computeCategoryFinalScore(deterministicScore: number, aiScore: number): number {
assertValidScore(deterministicScore, "deterministicScore");
assertValidScore(aiScore, "aiScore");
return Math.round(clampScore((deterministicScore * DETERMINISTIC_WEIGHT) + (aiScore * AI_WEIGHT)));
}
export function normalizeCategoryScore(input: {
category: EvalScoreCategory;
deterministicScore: number;
aiScore: number;
rationale: string;
evidence: EvalCategoryScore["evidence"];
}): EvalCategoryScore {
const { category, deterministicScore, aiScore, rationale, evidence } = input;
if (!EVAL_SCORE_CATEGORIES.includes(category)) {
throw new Error(`Unknown score category: ${category}`);
}
if (!rationale.trim()) {
throw new Error(`rationale is required for ${category}`);
}
const finalScore = computeCategoryFinalScore(deterministicScore, aiScore);
return {
category,
deterministicScore,
aiScore,
finalScore,
weight: EVAL_CATEGORY_WEIGHTS[category],
band: resolveScoreBand(finalScore),
rationale,
evidence,
};
}
export function computeOverallScore(categoryScores: EvalCategoryScore[]): number {
if (categoryScores.length !== EVAL_SCORE_CATEGORIES.length) {
throw new Error(`Expected ${EVAL_SCORE_CATEGORIES.length} category scores`);
}
const byCategory = new Map<EvalScoreCategory, EvalCategoryScore>();
for (const categoryScore of categoryScores) {
if (!EVAL_SCORE_CATEGORIES.includes(categoryScore.category)) {
throw new Error(`Unknown score category: ${categoryScore.category}`);
}
byCategory.set(categoryScore.category, categoryScore);
}
let weightedSum = 0;
for (const category of EVAL_SCORE_CATEGORIES) {
const score = byCategory.get(category);
if (!score) {
throw new Error(`Missing category score: ${category}`);
}
assertValidScore(score.finalScore, `${category}.finalScore`);
weightedSum += score.finalScore * EVAL_CATEGORY_WEIGHTS[category];
}
return Math.round(clampScore(weightedSum));
}

36
packages/core/src/eval-types.ts
View File

@@ -19,16 +19,26 @@ export const EVAL_RUN_TRIGGERS = ["manual", "schedule", "api", "task_completion"
export type EvalRunTrigger = typeof EVAL_RUN_TRIGGERS[number];
export const EVAL_SCORE_CATEGORIES = [
"correctness",
"completeness",
"quality",
"reliability",
"tests",
"documentation",
"agentPerformance",
"taskOutcomeQuality",
"processCompliance",
] as const;
export type EvalScoreCategory = typeof EVAL_SCORE_CATEGORIES[number];
export const EVAL_SCORE_SCALE_MIN = 0;
export const EVAL_SCORE_SCALE_MAX = 100;
export const EVAL_SCORE_BANDS = [
{ id: "failing", min: 0, max: 39 },
{ id: "weak", min: 40, max: 59 },
{ id: "acceptable", min: 60, max: 74 },
{ id: "strong", min: 75, max: 89 },
{ id: "excellent", min: 90, max: 100 },
] as const;
export type EvalScoreBand = typeof EVAL_SCORE_BANDS[number]["id"];
export interface EvalTaskSnapshot {
taskId: string;
title?: string;
@@ -83,10 +93,14 @@ export interface EvalEvidenceReference {
}
export interface EvalCategoryScore {
category: EvalScoreCategory | string;
score: number;
maxScore?: number;
rationale?: string;
category: EvalScoreCategory;
deterministicScore: number;
aiScore: number;
finalScore: number;
weight: number;
band: EvalScoreBand;
rationale: string;
evidence: EvalEvidenceReference[];
}
export interface EvalFollowUpSuggestion {
@@ -264,7 +278,7 @@ export interface TaskEvaluation {
taskId: string;
deterministicSignals: DeterministicSignals;
overallScore: number;
categoryScores: Record<string, number>;
categoryScores: EvalCategoryScore[];
rationale: string;
evidence: EvaluationEvidenceRef[];
followUpDrafts: FollowUpDraft[];

20
packages/core/src/index.ts
View File

@@ -725,6 +725,8 @@ export type {
EvalTaskResultCreateInput,
EvalTaskResultUpdateInput,
EvalTaskResultListOptions,
EvalScoreBand,
EvalScoreCategory,
EvalCategoryScore,
EvalEvidenceReference,
EvalSignal,
@@ -736,7 +738,23 @@ export type {
FollowUpDraft,
TaskEvaluation,
} from "./eval-types.js";
export { EVAL_RUN_STATUSES, EVAL_RUN_TRIGGERS, EVAL_SCORE_CATEGORIES } from "./eval-types.js";
export {
EVAL_RUN_STATUSES,
EVAL_RUN_TRIGGERS,
EVAL_SCORE_CATEGORIES,
EVAL_SCORE_BANDS,
EVAL_SCORE_SCALE_MIN,
EVAL_SCORE_SCALE_MAX,
} from "./eval-types.js";
export {
EVAL_CATEGORY_WEIGHTS,
assertValidScore,
clampScore,
computeCategoryFinalScore,
computeOverallScore,
normalizeCategoryScore,
resolveScoreBand,
} from "./eval-scoring.js";
export {
TASK_EVALUATION_SCHEDULE_NAME,
DEFAULT_TASK_EVALUATION_SCHEDULE,