root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-3391): persist evaluator evidence with score categories

Browse Source 
- Add eval score category types and exports in core with store support and coverage
- Implement engine evaluator evidence extraction and persistence with dedicated tests
- Update evaluator flow and cron wiring to record evidence alongside eval runs
- Refresh architecture, storage, and eval docs for evidence and categorization behavior

Fusion-Task-Id: FN-3391
This commit is contained in:
Fusion
2026-05-06 13:33:58 -07:00
committed by gsxdsm
parent 91f187f9ff
commit 2449472a63
13 changed files with 926 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
packages/core/src/__tests__/eval-store.test.ts
View File

@@ -1,6 +1,7 @@
import { beforeEach, describe, expect, it } from "vitest";
import { createDatabase, type Database } from "../db.js";
import { EvalLifecycleError, EvalStore } from "../eval-store.js";
import { EVIDENCE_EXCERPT_TRUNCATION_MARKER, EVIDENCE_LIMITS, TASK_EVALUATION_EVIDENCE_SOURCE_ORDER } from "../eval-types.js";
let db: Database;
let store: EvalStore;
@@ -127,6 +128,145 @@ describe("EvalStore", () => {
expect(second.id).toBe(first.id);
});
it("persists evidence bundles via metadata and preserves stable source ordering", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
const created = store.createTaskResult(run.id, {
taskId: "FN-evidence",
taskSnapshot: { taskId: "FN-evidence", title: "Evidence task" },
status: "scored",
evidenceBundle: {
taskId: "FN-evidence",
runId: run.id,
sourceOrder: [...TASK_EVALUATION_EVIDENCE_SOURCE_ORDER],
taskMetadata: [{ id: "tm-1", source: "taskMetadata", label: "task snapshot", taskId: "FN-evidence", runId: run.id }],
commits: [{ id: "c-1", source: "commits", label: "commit", sha: "abc123", taskId: "FN-evidence", runId: run.id }],
workflow: [],
reviews: [],
documents: [],
taskActivity: [],
agentLogs: [],
runAudit: [],
},
});
const fetched = store.getTaskResult(created.id);
expect(fetched?.evidenceBundle?.sourceOrder).toEqual(TASK_EVALUATION_EVIDENCE_SOURCE_ORDER);
expect(fetched?.evidenceBundle?.taskMetadata[0]?.id).toBe("tm-1");
expect(fetched?.metadata?.__taskEvaluationEvidenceBundle).toBeDefined();
});
it("rejects evidence bundles that exceed per-source limits", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
expect(() => store.createTaskResult(run.id, {
taskId: "FN-over-limit",
taskSnapshot: { taskId: "FN-over-limit" },
status: "scored",
evidenceBundle: {
taskId: "FN-over-limit",
runId: run.id,
sourceOrder: [...TASK_EVALUATION_EVIDENCE_SOURCE_ORDER],
taskMetadata: [],
commits: Array.from({ length: EVIDENCE_LIMITS.commits + 1 }, (_, i) => ({
id: `c-${i}`,
source: "commits" as const,
label: `commit ${i}`,
sha: `${i}`,
taskId: "FN-over-limit",
runId: run.id,
})),
workflow: [],
reviews: [],
documents: [],
taskActivity: [],
agentLogs: [],
runAudit: [],
},
})).toThrow(/commits exceeds limit/);
});
it("truncates overlong evidence excerpts to bounded persisted size", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
const result = store.createTaskResult(run.id, {
taskId: "FN-truncate",
taskSnapshot: { taskId: "FN-truncate" },
status: "scored",
evidenceBundle: {
taskId: "FN-truncate",
runId: run.id,
sourceOrder: [...TASK_EVALUATION_EVIDENCE_SOURCE_ORDER],
taskMetadata: [{
id: "tm-1",
source: "taskMetadata",
label: "summary",
taskId: "FN-truncate",
runId: run.id,
excerpt: "x".repeat(800),
}],
commits: [],
workflow: [],
reviews: [],
documents: [],
taskActivity: [],
agentLogs: [],
runAudit: [],
},
});
const fetched = store.getTaskResult(result.id);
const excerpt = fetched?.evidenceBundle?.taskMetadata[0]?.excerpt ?? "";
expect(excerpt.length).toBeLessThanOrEqual(500);
expect(excerpt.endsWith(EVIDENCE_EXCERPT_TRUNCATION_MARKER)).toBe(true);
expect(fetched?.evidenceBundle?.taskMetadata[0]?.truncated).toBe(true);
});
it("rejects evidence bundles with incorrect sourceOrder", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
expect(() => store.createTaskResult(run.id, {
taskId: "FN-wrong-order",
taskSnapshot: { taskId: "FN-wrong-order" },
status: "scored",
evidenceBundle: {
taskId: "FN-wrong-order",
runId: run.id,
sourceOrder: ["commits", "taskMetadata", "workflow", "reviews", "documents", "taskActivity", "agentLogs", "runAudit"],
taskMetadata: [],
commits: [],
workflow: [],
reviews: [],
documents: [],
taskActivity: [],
agentLogs: [],
runAudit: [],
},
})).toThrow(/sourceOrder must match/);
});
it("round-trips optional empty evidence source groups", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
const result = store.createTaskResult(run.id, {
taskId: "FN-empty-sources",
taskSnapshot: { taskId: "FN-empty-sources" },
status: "scored",
evidenceBundle: {
taskId: "FN-empty-sources",
runId: run.id,
sourceOrder: [...TASK_EVALUATION_EVIDENCE_SOURCE_ORDER],
taskMetadata: [],
commits: [],
workflow: [],
reviews: [],
documents: [],
taskActivity: [],
agentLogs: [],
runAudit: [],
},
});
const fetched = store.getTaskResult(result.id);
expect(fetched?.evidenceBundle?.commits).toEqual([]);
expect(fetched?.evidenceBundle?.runAudit).toEqual([]);
});
it("appends run events with sequential ordering", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
const evt1 = store.appendRunEvent(run.id, { type: "info", message: "started" });

95
packages/core/src/eval-store.ts
View File

@@ -2,6 +2,12 @@ import { EventEmitter } from "node:events";
import { randomUUID } from "node:crypto";
import type { Database } from "./db.js";
import { fromJson, toJson, toJsonNullable } from "./db.js";
import {
EVIDENCE_EXCERPT_TRUNCATION_MARKER,
EVIDENCE_LIMITS,
MAX_EVIDENCE_EXCERPT_LENGTH,
TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
} from "./eval-types.js";
import type {
EvalRun,
EvalRunCreateInput,
@@ -14,10 +20,13 @@ import type {
EvalTaskResultCreateInput,
EvalTaskResultListOptions,
EvalTaskResultUpdateInput,
TaskEvaluationEvidenceBundle,
TaskEvidenceEntryBase,
} from "./eval-types.js";
const TERMINAL_STATUSES = new Set<EvalRunStatus>(["completed", "failed", "cancelled"]);
const ACTIVE_STATUSES = new Set<EvalRunStatus>(["pending", "running"]);
const EVIDENCE_BUNDLE_METADATA_KEY = "__taskEvaluationEvidenceBundle";
const VALID_TRANSITIONS: Record<EvalRunStatus, EvalRunStatus[]> = {
pending: ["running", "completed", "failed", "cancelled"],
running: ["completed", "failed", "cancelled"],
@@ -45,6 +54,77 @@ function generateEventId(): string {
return `ERE-${randomUUID()}`;
}
function withEvidenceBundleMetadata(
metadata: Record<string, unknown> | undefined,
evidenceBundle: EvalTaskResult["evidenceBundle"],
): Record<string, unknown> | undefined {
if (!evidenceBundle) return metadata;
return {
...(metadata ?? {}),
[EVIDENCE_BUNDLE_METADATA_KEY]: evidenceBundle,
};
}
function readEvidenceBundleFromMetadata(metadata: Record<string, unknown> | undefined): EvalTaskResult["evidenceBundle"] {
if (!metadata) return undefined;
return metadata[EVIDENCE_BUNDLE_METADATA_KEY] as EvalTaskResult["evidenceBundle"] | undefined;
}
function sortEvidenceEntries<T extends TaskEvidenceEntryBase>(entries: T[]): T[] {
return [...entries].sort((a, b) => {
const timeOrder = (a.timestamp ?? "").localeCompare(b.timestamp ?? "");
if (timeOrder !== 0) return timeOrder;
return a.id.localeCompare(b.id);
});
}
function truncateEvidenceExcerpt<T extends TaskEvidenceEntryBase>(entry: T): T {
if (!entry.excerpt || entry.excerpt.length <= MAX_EVIDENCE_EXCERPT_LENGTH) {
return entry;
}
const maxPrefix = Math.max(0, MAX_EVIDENCE_EXCERPT_LENGTH - EVIDENCE_EXCERPT_TRUNCATION_MARKER.length);
return {
...entry,
excerpt: `${entry.excerpt.slice(0, maxPrefix)}${EVIDENCE_EXCERPT_TRUNCATION_MARKER}`,
truncated: true,
};
}
function validateEvidenceBundle(bundle: TaskEvaluationEvidenceBundle | undefined): TaskEvaluationEvidenceBundle | undefined {
if (!bundle) return undefined;
if (bundle.sourceOrder.join("|") !== TASK_EVALUATION_EVIDENCE_SOURCE_ORDER.join("|")) {
throw new Error("evidenceBundle.sourceOrder must match TASK_EVALUATION_EVIDENCE_SOURCE_ORDER");
}
const groupLimits: Array<[keyof TaskEvaluationEvidenceBundle, number]> = [
["taskMetadata", EVIDENCE_LIMITS.taskMetadata],
["commits", EVIDENCE_LIMITS.commits],
["workflow", EVIDENCE_LIMITS.workflow],
["reviews", EVIDENCE_LIMITS.reviews],
["documents", EVIDENCE_LIMITS.documents],
["taskActivity", EVIDENCE_LIMITS.taskActivity],
["agentLogs", EVIDENCE_LIMITS.agentLogs],
["runAudit", EVIDENCE_LIMITS.runAudit],
];
for (const [group, limit] of groupLimits) {
const entries = bundle[group];
if (Array.isArray(entries) && entries.length > limit) {
throw new Error(`evidenceBundle.${group} exceeds limit ${limit}`);
}
}
return {
...bundle,
taskMetadata: sortEvidenceEntries(bundle.taskMetadata).map(truncateEvidenceExcerpt),
commits: sortEvidenceEntries(bundle.commits).map(truncateEvidenceExcerpt),
workflow: sortEvidenceEntries(bundle.workflow).map(truncateEvidenceExcerpt),
reviews: sortEvidenceEntries(bundle.reviews).map(truncateEvidenceExcerpt),
documents: sortEvidenceEntries(bundle.documents).map(truncateEvidenceExcerpt),
taskActivity: sortEvidenceEntries(bundle.taskActivity).map(truncateEvidenceExcerpt),
agentLogs: sortEvidenceEntries(bundle.agentLogs).map(truncateEvidenceExcerpt),
runAudit: sortEvidenceEntries(bundle.runAudit).map(truncateEvidenceExcerpt),
};
}
export class EvalStore extends EventEmitter<EvalStoreEvents> {
constructor(private readonly db: Database) {
super();
@@ -188,6 +268,8 @@ export class EvalStore extends EventEmitter<EvalStoreEvents> {
if (!run) throw new Error(`Eval run not found: ${runId}`);
const now = new Date().toISOString();
const evidenceBundle = validateEvidenceBundle(input.evidenceBundle);
const metadata = withEvidenceBundleMetadata(input.metadata, evidenceBundle);
const result: EvalTaskResult = {
id: generateResultId(),
runId,
@@ -200,11 +282,12 @@ export class EvalStore extends EventEmitter<EvalStoreEvents> {
rationale: input.rationale,
summary: input.summary,
evidence: input.evidence ?? [],
evidenceBundle,
deterministicSignals: input.deterministicSignals ?? [],
aiSignals: input.aiSignals,
followUps: input.followUps ?? [],
provenance: input.provenance,
metadata: input.metadata,
metadata,
createdAt: now,
updatedAt: now,
};
@@ -301,10 +384,13 @@ export class EvalStore extends EventEmitter<EvalStoreEvents> {
if (!existing) return undefined;
const now = new Date().toISOString();
const mergedMetadata = input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata;
const evidenceBundle = validateEvidenceBundle(input.evidenceBundle ?? existing.evidenceBundle);
const updated: EvalTaskResult = {
...existing,
...input,
metadata: input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata,
evidenceBundle,
metadata: withEvidenceBundleMetadata(mergedMetadata, evidenceBundle),
provenance: input.provenance ? { ...(existing.provenance ?? {}), ...input.provenance } : existing.provenance,
updatedAt: now,
};
@@ -437,6 +523,8 @@ export class EvalStore extends EventEmitter<EvalStoreEvents> {
}
private rowToResult(row: Record<string, unknown>): EvalTaskResult {
const metadata = fromJson<Record<string, unknown>>(row.metadata as string);
const evidenceBundle = readEvidenceBundleFromMetadata(metadata);
return {
id: String(row.id),
runId: String(row.runId),
@@ -449,11 +537,12 @@ export class EvalStore extends EventEmitter<EvalStoreEvents> {
rationale: (row.rationale as string | null) ?? undefined,
summary: (row.summary as string | null) ?? undefined,
evidence: fromJson(row.evidence as string) ?? [],
evidenceBundle,
deterministicSignals: fromJson(row.deterministicSignals as string) ?? [],
aiSignals: fromJson(row.aiSignals as string),
followUps: fromJson(row.followUps as string) ?? [],
provenance: fromJson(row.provenance as string),
metadata: fromJson(row.metadata as string),
metadata,
createdAt: String(row.createdAt),
updatedAt: String(row.updatedAt),
};

141
packages/core/src/eval-types.ts
View File

@@ -92,6 +92,144 @@ export interface EvalEvidenceReference {
metadata?: Record<string, unknown>;
}
export const TASK_EVALUATION_EVIDENCE_SOURCE_ORDER = [
"taskMetadata",
"commits",
"workflow",
"reviews",
"documents",
"taskActivity",
"agentLogs",
"runAudit",
] as const;
export const EVIDENCE_LIMITS = {
taskMetadata: 25,
commits: 20,
workflow: 25,
reviews: 25,
documents: 25,
taskActivity: 25,
agentLogs: 25,
runAudit: 25,
} as const;
export const MAX_EVIDENCE_EXCERPT_LENGTH = 500;
export const EVIDENCE_EXCERPT_TRUNCATION_MARKER = "… [truncated]";
export type TaskEvaluationEvidenceSource = typeof TASK_EVALUATION_EVIDENCE_SOURCE_ORDER[number];
export interface TaskEvidenceEntryBase {
id: string;
source: TaskEvaluationEvidenceSource;
label: string;
timestamp?: string;
excerpt?: string;
truncated?: boolean;
}
export interface TaskMetadataEvidence extends TaskEvidenceEntryBase {
source: "taskMetadata";
taskId: string;
runId: string;
summary?: string;
references?: {
prNumber?: number;
prUrl?: string;
mergeCommitSha?: string;
mergeCompletedAt?: string;
executionStartedAt?: string;
executionCompletedAt?: string;
};
retryMetrics?: {
mergeRetries: number;
workflowStepRetries: number;
stuckKillCount: number;
postReviewFixCount: number;
recoveryRetryCount: number;
taskDoneRetryCount: number;
verificationFailureCount: number;
mergeConflictBounceCount: number;
};
}
export interface CommitEvidence extends TaskEvidenceEntryBase {
source: "commits";
sha: string;
taskId: string;
runId: string;
authoredAt?: string;
authorName?: string;
subject?: string;
}
export interface WorkflowEvidence extends TaskEvidenceEntryBase {
source: "workflow";
taskId: string;
runId: string;
workflowStepId?: string;
stepName?: string;
status?: string;
command?: string;
}
export interface ReviewEvidence extends TaskEvidenceEntryBase {
source: "reviews";
taskId: string;
runId: string;
reviewStep?: number;
reviewType?: string;
verdict?: string;
}
export interface DocumentEvidence extends TaskEvidenceEntryBase {
source: "documents";
taskId: string;
runId: string;
documentKey: string;
revision?: number;
author?: string;
}
export interface TaskActivityEvidence extends TaskEvidenceEntryBase {
source: "taskActivity";
taskId: string;
runId: string;
activityType?: string;
}
export interface AgentLogEvidence extends TaskEvidenceEntryBase {
source: "agentLogs";
taskId: string;
runId: string;
logType?: string;
agentId?: string;
}
export interface RunAuditEvidence extends TaskEvidenceEntryBase {
source: "runAudit";
taskId: string;
runId: string;
eventId: string;
domain?: string;
mutationType?: string;
target?: string;
}
export interface TaskEvaluationEvidenceBundle {
taskId: string;
runId: string;
sourceOrder: readonly TaskEvaluationEvidenceSource[];
taskMetadata: TaskMetadataEvidence[];
commits: CommitEvidence[];
workflow: WorkflowEvidence[];
reviews: ReviewEvidence[];
documents: DocumentEvidence[];
taskActivity: TaskActivityEvidence[];
agentLogs: AgentLogEvidence[];
runAudit: RunAuditEvidence[];
}
export interface EvalCategoryScore {
category: EvalScoreCategory;
deterministicScore: number;
@@ -123,6 +261,7 @@ export interface EvalTaskResult {
rationale?: string;
summary?: string;
evidence: EvalEvidenceReference[];
evidenceBundle?: TaskEvaluationEvidenceBundle;
deterministicSignals: EvalSignal[];
aiSignals?: EvalSignal[];
followUps: EvalFollowUpSuggestion[];
@@ -215,6 +354,7 @@ export interface EvalTaskResultCreateInput {
rationale?: string;
summary?: string;
evidence?: EvalEvidenceReference[];
evidenceBundle?: TaskEvaluationEvidenceBundle;
deterministicSignals?: EvalSignal[];
aiSignals?: EvalSignal[];
followUps?: EvalFollowUpSuggestion[];
@@ -230,6 +370,7 @@ export interface EvalTaskResultUpdateInput {
rationale?: string;
summary?: string;
evidence?: EvalEvidenceReference[];
evidenceBundle?: TaskEvaluationEvidenceBundle;
deterministicSignals?: EvalSignal[];
aiSignals?: EvalSignal[];
followUps?: EvalFollowUpSuggestion[];

15
packages/core/src/index.ts
View File

@@ -729,6 +729,17 @@ export type {
EvalScoreCategory,
EvalCategoryScore,
EvalEvidenceReference,
TaskEvaluationEvidenceSource,
TaskEvidenceEntryBase,
TaskMetadataEvidence,
CommitEvidence,
WorkflowEvidence,
ReviewEvidence,
DocumentEvidence,
TaskActivityEvidence,
AgentLogEvidence,
RunAuditEvidence,
TaskEvaluationEvidenceBundle,
EvalSignal,
EvalFollowUpSuggestion,
EvalProvenance,
@@ -745,6 +756,10 @@ export {
EVAL_SCORE_BANDS,
EVAL_SCORE_SCALE_MIN,
EVAL_SCORE_SCALE_MAX,
TASK_EVALUATION_EVIDENCE_SOURCE_ORDER,
EVIDENCE_LIMITS,
MAX_EVIDENCE_EXCERPT_LENGTH,
EVIDENCE_EXCERPT_TRUNCATION_MARKER,
} from "./eval-types.js";
export {
EVAL_CATEGORY_WEIGHTS,