feat(FN-3387): add eval domain with store and persistence schema
Introduced a new evaluation persistence layer in `@fusion/core` with domain contracts, SQLite-backed storage APIs, and retention/window-rollup support, wired through the core store and documented in the architecture and storage docs. Fusion-Task-Id: FN-3387
This commit is contained in:
@@ -160,7 +160,7 @@ describe("Database", () => {
|
||||
});
|
||||
|
||||
it("seeds schema version", () => {
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
});
|
||||
|
||||
it("seeds lastModified", () => {
|
||||
@@ -183,7 +183,7 @@ describe("Database", () => {
|
||||
|
||||
it("is idempotent - calling init() twice does not fail", () => {
|
||||
expect(() => db.init()).not.toThrow();
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
});
|
||||
|
||||
it("does not overwrite existing config on re-init", () => {
|
||||
@@ -957,7 +957,7 @@ describe("schema migrations", () => {
|
||||
db.init();
|
||||
|
||||
// Verify version bumped to 29 (includes v1→v2 through v26→v29)
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
// Verify new columns exist and existing data is intact
|
||||
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
@@ -982,11 +982,11 @@ describe("schema migrations", () => {
|
||||
const db = new Database(fusionDir);
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
// Re-init should not fail
|
||||
db.init();
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
db.close();
|
||||
});
|
||||
@@ -1021,7 +1021,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
expect(cols.map((col) => col.name)).toContain("priority");
|
||||
@@ -1062,7 +1062,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
const colNames = cols.map((col) => col.name);
|
||||
@@ -1131,7 +1131,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
const colNames = cols.map((col) => col.name);
|
||||
@@ -1234,7 +1234,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
const cols = db.prepare("PRAGMA table_info(chat_messages)").all() as Array<{ name: string }>;
|
||||
expect(cols.map((col) => col.name)).toContain("attachments");
|
||||
@@ -1308,7 +1308,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>;
|
||||
expect(tables).toEqual([{ name: "agentRatings" }]);
|
||||
@@ -1332,7 +1332,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>;
|
||||
expect(tables).toEqual([{ name: "mission_events" }]);
|
||||
@@ -1436,7 +1436,7 @@ describe("schema migrations", () => {
|
||||
db.init();
|
||||
|
||||
// Verify version bumped to 29
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
// Verify new columns exist and existing data is intact
|
||||
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
@@ -1905,7 +1905,7 @@ describe("createDatabase factory", () => {
|
||||
const db = createDatabase(fusionDir);
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
expect(db.getLastModified()).toBeGreaterThan(0);
|
||||
|
||||
db.close();
|
||||
|
||||
82
packages/core/src/__tests__/eval-store.test.ts
Normal file
82
packages/core/src/__tests__/eval-store.test.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { createDatabase, type Database } from "../db.js";
|
||||
import { EvalLifecycleError, EvalStore } from "../eval-store.js";
|
||||
|
||||
let db: Database;
|
||||
let store: EvalStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = createDatabase("/tmp/fn-eval-store-test", { inMemory: true });
|
||||
db.init();
|
||||
store = new EvalStore(db);
|
||||
});
|
||||
|
||||
describe("EvalStore", () => {
|
||||
it("creates and lists runs with deterministic ordering", () => {
|
||||
const runA = store.createRun({ projectId: "p1", scope: "completed-since-last", requestedTaskIds: ["FN-1"] });
|
||||
const runB = store.createRun({ projectId: "p1", scope: "completed-since-last", requestedTaskIds: ["FN-2"] });
|
||||
|
||||
const runs = store.listRuns({ projectId: "p1" });
|
||||
expect(runs.map((run) => run.id)).toEqual([runA.id, runB.id].sort());
|
||||
});
|
||||
|
||||
it("enforces active run conflict for scheduled trigger", () => {
|
||||
store.createRun({ projectId: "p1", scope: "window", trigger: "schedule" });
|
||||
expect(() => store.createRun({ projectId: "p1", scope: "window", trigger: "schedule" })).toThrow(EvalLifecycleError);
|
||||
});
|
||||
|
||||
it("enforces terminal immutability", () => {
|
||||
const run = store.createRun({ projectId: "p1", scope: "window" });
|
||||
store.updateRun(run.id, { status: "completed" });
|
||||
expect(() => store.updateRun(run.id, { summary: "late change" })).toThrow(EvalLifecycleError);
|
||||
});
|
||||
|
||||
it("creates results and preserves task snapshot after tasks row deletion", () => {
|
||||
const run = store.createRun({ projectId: "p1", scope: "window" });
|
||||
const result = store.createTaskResult(run.id, {
|
||||
taskId: "FN-123",
|
||||
taskSnapshot: { taskId: "FN-123", title: "Snapshot title", status: "done", summary: "task summary" },
|
||||
status: "scored",
|
||||
overallScore: 0.8,
|
||||
categoryScores: [{ category: "quality", score: 0.8 }],
|
||||
evidence: [{ type: "task_log", ref: "log:1" }],
|
||||
deterministicSignals: [{ signalId: "s1", kind: "test", name: "tests-pass", passed: true }],
|
||||
});
|
||||
|
||||
db.prepare("DELETE FROM tasks WHERE id = ?").run("FN-123");
|
||||
|
||||
const fetched = store.getTaskResult(result.id);
|
||||
expect(fetched?.taskSnapshot.title).toBe("Snapshot title");
|
||||
expect(fetched?.taskId).toBe("FN-123");
|
||||
});
|
||||
|
||||
it("persists run window boundaries and evaluated task rollups", () => {
|
||||
const run = store.createRun({
|
||||
projectId: "p1",
|
||||
trigger: "schedule",
|
||||
scope: "completed-since-last",
|
||||
window: { since: "2026-05-01T00:00:00.000Z", until: "2026-05-02T00:00:00.000Z", baselineRunId: "ER-BASE" },
|
||||
requestedTaskIds: ["FN-1", "FN-2"],
|
||||
});
|
||||
|
||||
const updated = store.updateRun(run.id, {
|
||||
status: "running",
|
||||
evaluatedTaskIds: ["FN-1", "FN-2"],
|
||||
counts: { totalTasks: 2, scoredTasks: 1, skippedTasks: 1, erroredTasks: 0 },
|
||||
});
|
||||
|
||||
expect(updated?.window.since).toBe("2026-05-01T00:00:00.000Z");
|
||||
expect(updated?.evaluatedTaskIds).toEqual(["FN-1", "FN-2"]);
|
||||
expect(updated?.counts.scoredTasks).toBe(1);
|
||||
});
|
||||
|
||||
it("appends run events with sequential ordering", () => {
|
||||
const run = store.createRun({ projectId: "p1", scope: "window" });
|
||||
const evt1 = store.appendRunEvent(run.id, { type: "info", message: "started" });
|
||||
const evt2 = store.appendRunEvent(run.id, { type: "task_evaluated", message: "scored", taskId: "FN-1" });
|
||||
|
||||
const events = store.listRunEvents(run.id);
|
||||
expect(events.map((event) => event.id)).toEqual([evt1.id, evt2.id]);
|
||||
expect(events.map((event) => event.seq)).toEqual([1, 2]);
|
||||
});
|
||||
});
|
||||
@@ -886,7 +886,7 @@ describe("Migration: pre-33 DB upgrade", () => {
|
||||
// Step 1: Create a fresh database at v33 (runs all migrations up to 33)
|
||||
const db1 = createDatabase(legacyDir);
|
||||
db1.init();
|
||||
expect(db1.getSchemaVersion()).toBe(61);
|
||||
expect(db1.getSchemaVersion()).toBe(62);
|
||||
db1.close();
|
||||
|
||||
// Step 2: Manually downgrade to version 32 and drop insight tables
|
||||
@@ -921,7 +921,7 @@ describe("Migration: pre-33 DB upgrade", () => {
|
||||
expect(tableNamesBefore).not.toContain("project_insight_runs");
|
||||
// Now run init — this triggers the v32→v33 migration
|
||||
db3.init();
|
||||
expect(db3.getSchemaVersion()).toBe(61);
|
||||
expect(db3.getSchemaVersion()).toBe(62);
|
||||
|
||||
// Step 4: Verify insight tables exist after migration
|
||||
const tablesAfter = db3.prepare(
|
||||
@@ -952,12 +952,12 @@ describe("Migration: pre-33 DB upgrade", () => {
|
||||
try {
|
||||
const db1 = createDatabase(testDir);
|
||||
db1.init();
|
||||
expect(db1.getSchemaVersion()).toBe(61);
|
||||
expect(db1.getSchemaVersion()).toBe(62);
|
||||
db1.close();
|
||||
|
||||
const db2 = createDatabase(testDir);
|
||||
expect(() => db2.init()).not.toThrow();
|
||||
expect(db2.getSchemaVersion()).toBe(61);
|
||||
expect(db2.getSchemaVersion()).toBe(62);
|
||||
db2.close();
|
||||
} finally {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
|
||||
@@ -2629,7 +2629,7 @@ describe("MissionStore", () => {
|
||||
|
||||
describe("Loop State & Validator Run Schema (v31)", () => {
|
||||
it("schema version is 40 after migration", () => {
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
});
|
||||
|
||||
it("mission_features table has loop state columns", () => {
|
||||
|
||||
@@ -742,7 +742,7 @@ describe("RoadmapStore", () => {
|
||||
|
||||
describe("schema version", () => {
|
||||
it("schema version is 40 after init", () => {
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -465,7 +465,7 @@ describe("Run Audit", () => {
|
||||
});
|
||||
|
||||
it("schema version is bumped to 40", () => {
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -51,7 +51,7 @@ describe("TaskStore task documents", () => {
|
||||
|
||||
expect(tableNames.has("task_documents")).toBe(true);
|
||||
expect(tableNames.has("task_document_revisions")).toBe(true);
|
||||
expect(db.getSchemaVersion()).toBe(61);
|
||||
expect(db.getSchemaVersion()).toBe(62);
|
||||
|
||||
const index = db
|
||||
.prepare(
|
||||
|
||||
@@ -88,7 +88,7 @@ export function probeFts5(db: DatabaseSync): boolean {
|
||||
|
||||
// ── Schema Definition ────────────────────────────────────────────────
|
||||
|
||||
const SCHEMA_VERSION = 61;
|
||||
const SCHEMA_VERSION = 62;
|
||||
|
||||
function normalizeTaskComments(
|
||||
steeringComments: SteeringComment[] | undefined,
|
||||
@@ -466,6 +466,71 @@ CREATE TABLE IF NOT EXISTS research_run_events (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idxResearchRunEventsRunIdSeq ON research_run_events(runId, seq);
|
||||
|
||||
-- Eval run persistence (FN-3387)
|
||||
CREATE TABLE IF NOT EXISTS eval_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
projectId TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
trigger TEXT NOT NULL,
|
||||
scope TEXT NOT NULL,
|
||||
window TEXT NOT NULL DEFAULT '{}',
|
||||
requestedTaskIds TEXT NOT NULL DEFAULT '[]',
|
||||
evaluatedTaskIds TEXT NOT NULL DEFAULT '[]',
|
||||
counts TEXT NOT NULL DEFAULT '{"totalTasks":0,"scoredTasks":0,"skippedTasks":0,"erroredTasks":0}',
|
||||
aggregateScores TEXT,
|
||||
summary TEXT,
|
||||
error TEXT,
|
||||
provenance TEXT,
|
||||
metadata TEXT,
|
||||
createdAt TEXT NOT NULL,
|
||||
updatedAt TEXT NOT NULL,
|
||||
startedAt TEXT,
|
||||
completedAt TEXT,
|
||||
cancelledAt TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idxEvalRunsProjectIdCreatedAt ON eval_runs(projectId, createdAt);
|
||||
CREATE INDEX IF NOT EXISTS idxEvalRunsProjectTriggerStatus ON eval_runs(projectId, trigger, status);
|
||||
CREATE INDEX IF NOT EXISTS idxEvalRunsStatusCreatedAt ON eval_runs(status, createdAt);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eval_task_results (
|
||||
id TEXT PRIMARY KEY,
|
||||
runId TEXT NOT NULL,
|
||||
taskId TEXT NOT NULL,
|
||||
taskSnapshot TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
overallScore REAL,
|
||||
maxScore REAL,
|
||||
categoryScores TEXT NOT NULL DEFAULT '[]',
|
||||
rationale TEXT,
|
||||
summary TEXT,
|
||||
evidence TEXT NOT NULL DEFAULT '[]',
|
||||
deterministicSignals TEXT NOT NULL DEFAULT '[]',
|
||||
aiSignals TEXT,
|
||||
followUps TEXT NOT NULL DEFAULT '[]',
|
||||
provenance TEXT,
|
||||
metadata TEXT,
|
||||
createdAt TEXT NOT NULL,
|
||||
updatedAt TEXT NOT NULL,
|
||||
FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idxEvalTaskResultsRunIdCreatedAt ON eval_task_results(runId, createdAt);
|
||||
CREATE INDEX IF NOT EXISTS idxEvalTaskResultsTaskIdCreatedAt ON eval_task_results(taskId, createdAt);
|
||||
CREATE INDEX IF NOT EXISTS idxEvalTaskResultsStatusRunId ON eval_task_results(status, runId);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eval_run_events (
|
||||
id TEXT PRIMARY KEY,
|
||||
runId TEXT NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
status TEXT,
|
||||
taskId TEXT,
|
||||
metadata TEXT,
|
||||
createdAt TEXT NOT NULL,
|
||||
FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idxEvalRunEventsRunIdSeq ON eval_run_events(runId, seq);
|
||||
|
||||
-- Schema version tracking
|
||||
CREATE TABLE IF NOT EXISTS __meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
@@ -2413,6 +2478,80 @@ export class Database {
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 62) {
|
||||
this.applyMigration(62, () => {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS eval_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
projectId TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
trigger TEXT NOT NULL,
|
||||
scope TEXT NOT NULL,
|
||||
window TEXT NOT NULL DEFAULT '{}',
|
||||
requestedTaskIds TEXT NOT NULL DEFAULT '[]',
|
||||
evaluatedTaskIds TEXT NOT NULL DEFAULT '[]',
|
||||
counts TEXT NOT NULL DEFAULT '{"totalTasks":0,"scoredTasks":0,"skippedTasks":0,"erroredTasks":0}',
|
||||
aggregateScores TEXT,
|
||||
summary TEXT,
|
||||
error TEXT,
|
||||
provenance TEXT,
|
||||
metadata TEXT,
|
||||
createdAt TEXT NOT NULL,
|
||||
updatedAt TEXT NOT NULL,
|
||||
startedAt TEXT,
|
||||
completedAt TEXT,
|
||||
cancelledAt TEXT
|
||||
)
|
||||
`);
|
||||
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsProjectIdCreatedAt ON eval_runs(projectId, createdAt)`);
|
||||
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsProjectTriggerStatus ON eval_runs(projectId, trigger, status)`);
|
||||
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsStatusCreatedAt ON eval_runs(status, createdAt)`);
|
||||
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS eval_task_results (
|
||||
id TEXT PRIMARY KEY,
|
||||
runId TEXT NOT NULL,
|
||||
taskId TEXT NOT NULL,
|
||||
taskSnapshot TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
overallScore REAL,
|
||||
maxScore REAL,
|
||||
categoryScores TEXT NOT NULL DEFAULT '[]',
|
||||
rationale TEXT,
|
||||
summary TEXT,
|
||||
evidence TEXT NOT NULL DEFAULT '[]',
|
||||
deterministicSignals TEXT NOT NULL DEFAULT '[]',
|
||||
aiSignals TEXT,
|
||||
followUps TEXT NOT NULL DEFAULT '[]',
|
||||
provenance TEXT,
|
||||
metadata TEXT,
|
||||
createdAt TEXT NOT NULL,
|
||||
updatedAt TEXT NOT NULL,
|
||||
FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE
|
||||
)
|
||||
`);
|
||||
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsRunIdCreatedAt ON eval_task_results(runId, createdAt)`);
|
||||
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsTaskIdCreatedAt ON eval_task_results(taskId, createdAt)`);
|
||||
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsStatusRunId ON eval_task_results(status, runId)`);
|
||||
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS eval_run_events (
|
||||
id TEXT PRIMARY KEY,
|
||||
runId TEXT NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
status TEXT,
|
||||
taskId TEXT,
|
||||
metadata TEXT,
|
||||
createdAt TEXT NOT NULL,
|
||||
FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE
|
||||
)
|
||||
`);
|
||||
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunEventsRunIdSeq ON eval_run_events(runId, seq)`);
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
454
packages/core/src/eval-store.ts
Normal file
454
packages/core/src/eval-store.ts
Normal file
@@ -0,0 +1,454 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { Database } from "./db.js";
|
||||
import { fromJson, toJson, toJsonNullable } from "./db.js";
|
||||
import type {
|
||||
EvalRun,
|
||||
EvalRunCreateInput,
|
||||
EvalRunEvent,
|
||||
EvalRunListOptions,
|
||||
EvalRunStatus,
|
||||
EvalRunUpdateInput,
|
||||
EvalStoreEvents,
|
||||
EvalTaskResult,
|
||||
EvalTaskResultCreateInput,
|
||||
EvalTaskResultListOptions,
|
||||
EvalTaskResultUpdateInput,
|
||||
} from "./eval-types.js";
|
||||
|
||||
const TERMINAL_STATUSES = new Set<EvalRunStatus>(["completed", "failed", "cancelled"]);
|
||||
const ACTIVE_STATUSES = new Set<EvalRunStatus>(["pending", "running"]);
|
||||
const VALID_TRANSITIONS: Record<EvalRunStatus, EvalRunStatus[]> = {
|
||||
pending: ["running", "completed", "failed", "cancelled"],
|
||||
running: ["completed", "failed", "cancelled"],
|
||||
completed: [],
|
||||
failed: [],
|
||||
cancelled: [],
|
||||
};
|
||||
|
||||
export class EvalLifecycleError extends Error {
|
||||
constructor(message: string, readonly code: "invalid_transition" | "terminal_immutable" | "active_run_conflict") {
|
||||
super(message);
|
||||
this.name = "EvalLifecycleError";
|
||||
}
|
||||
}
|
||||
|
||||
function generateRunId(): string {
|
||||
return `ER-${Date.now().toString(36).toUpperCase()}-${Math.random().toString(36).slice(2, 7).toUpperCase()}`;
|
||||
}
|
||||
|
||||
function generateResultId(): string {
|
||||
return `ETR-${randomUUID()}`;
|
||||
}
|
||||
|
||||
function generateEventId(): string {
|
||||
return `ERE-${randomUUID()}`;
|
||||
}
|
||||
|
||||
export class EvalStore extends EventEmitter<EvalStoreEvents> {
|
||||
constructor(private readonly db: Database) {
|
||||
super();
|
||||
this.setMaxListeners(50);
|
||||
}
|
||||
|
||||
createRun(input: EvalRunCreateInput): EvalRun {
|
||||
const now = new Date().toISOString();
|
||||
if ((input.trigger === "schedule" || input.trigger === "task_completion") && this.hasActiveRun(input.projectId, input.trigger)) {
|
||||
throw new EvalLifecycleError(`Active eval run already exists for project ${input.projectId} trigger ${input.trigger}`, "active_run_conflict");
|
||||
}
|
||||
|
||||
const run: EvalRun = {
|
||||
id: generateRunId(),
|
||||
projectId: input.projectId,
|
||||
status: "pending",
|
||||
trigger: input.trigger ?? "manual",
|
||||
scope: input.scope,
|
||||
window: input.window ?? {},
|
||||
requestedTaskIds: input.requestedTaskIds ?? [],
|
||||
evaluatedTaskIds: [],
|
||||
counts: { totalTasks: input.requestedTaskIds?.length ?? 0, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 },
|
||||
provenance: input.provenance,
|
||||
metadata: input.metadata,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
|
||||
this.db.prepare(`
|
||||
INSERT INTO eval_runs (
|
||||
id, projectId, status, trigger, scope, window, requestedTaskIds, evaluatedTaskIds,
|
||||
counts, aggregateScores, summary, error, provenance, metadata,
|
||||
createdAt, updatedAt, startedAt, completedAt, cancelledAt
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
run.id,
|
||||
run.projectId,
|
||||
run.status,
|
||||
run.trigger,
|
||||
run.scope,
|
||||
toJson(run.window),
|
||||
toJson(run.requestedTaskIds),
|
||||
toJson(run.evaluatedTaskIds),
|
||||
toJson(run.counts),
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
toJsonNullable(run.provenance),
|
||||
toJsonNullable(run.metadata),
|
||||
run.createdAt,
|
||||
run.updatedAt,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
);
|
||||
|
||||
this.db.bumpLastModified();
|
||||
this.emit("run:created", run);
|
||||
return run;
|
||||
}
|
||||
|
||||
getRun(id: string): EvalRun | undefined {
|
||||
const row = this.db.prepare("SELECT * FROM eval_runs WHERE id = ?").get(id) as Record<string, unknown> | undefined;
|
||||
return row ? this.rowToRun(row) : undefined;
|
||||
}
|
||||
|
||||
listRuns(options: EvalRunListOptions = {}): EvalRun[] {
|
||||
const clauses: string[] = [];
|
||||
const params: Array<string | number> = [];
|
||||
if (options.projectId) {
|
||||
clauses.push("projectId = ?");
|
||||
params.push(options.projectId);
|
||||
}
|
||||
if (options.status) {
|
||||
clauses.push("status = ?");
|
||||
params.push(options.status);
|
||||
}
|
||||
if (options.trigger) {
|
||||
clauses.push("trigger = ?");
|
||||
params.push(options.trigger);
|
||||
}
|
||||
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
||||
const limit = options.limit !== undefined ? `LIMIT ${options.limit}` : "";
|
||||
const offset = options.offset !== undefined ? `OFFSET ${options.offset}` : "";
|
||||
|
||||
const rows = this.db.prepare(`
|
||||
SELECT * FROM eval_runs
|
||||
${where}
|
||||
ORDER BY createdAt ASC, id ASC
|
||||
${limit}
|
||||
${offset}
|
||||
`).all(...params) as Record<string, unknown>[];
|
||||
|
||||
return rows.map((row) => this.rowToRun(row));
|
||||
}
|
||||
|
||||
updateRun(id: string, input: EvalRunUpdateInput): EvalRun | undefined {
|
||||
const existing = this.getRun(id);
|
||||
if (!existing) return undefined;
|
||||
|
||||
if (TERMINAL_STATUSES.has(existing.status) && Object.keys(input).some((k) => k !== "status")) {
|
||||
throw new EvalLifecycleError(`Eval run ${id} is terminal and immutable`, "terminal_immutable");
|
||||
}
|
||||
|
||||
if (input.status && input.status !== existing.status) {
|
||||
if (!VALID_TRANSITIONS[existing.status].includes(input.status)) {
|
||||
throw new EvalLifecycleError(`Invalid eval run status transition: ${existing.status} -> ${input.status}`, "invalid_transition");
|
||||
}
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const updated: EvalRun = {
|
||||
...existing,
|
||||
...input,
|
||||
error: input.error === null ? undefined : (input.error ?? existing.error),
|
||||
metadata: input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata,
|
||||
provenance: input.provenance ? { ...(existing.provenance ?? {}), ...input.provenance } : existing.provenance,
|
||||
updatedAt: now,
|
||||
startedAt: input.startedAt === null ? undefined : (input.startedAt ?? existing.startedAt),
|
||||
completedAt: input.completedAt === null ? undefined : (input.completedAt ?? existing.completedAt),
|
||||
cancelledAt: input.cancelledAt === null ? undefined : (input.cancelledAt ?? existing.cancelledAt),
|
||||
};
|
||||
|
||||
this.persistRun(updated);
|
||||
this.emit("run:updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
deleteRun(id: string): boolean {
|
||||
const result = this.db.prepare("DELETE FROM eval_runs WHERE id = ?").run(id) as { changes?: number };
|
||||
const deleted = (result.changes ?? 0) > 0;
|
||||
if (deleted) {
|
||||
this.db.bumpLastModified();
|
||||
this.emit("run:deleted", id);
|
||||
}
|
||||
return deleted;
|
||||
}
|
||||
|
||||
createTaskResult(runId: string, input: EvalTaskResultCreateInput): EvalTaskResult {
|
||||
const run = this.getRun(runId);
|
||||
if (!run) throw new Error(`Eval run not found: ${runId}`);
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const result: EvalTaskResult = {
|
||||
id: generateResultId(),
|
||||
runId,
|
||||
taskId: input.taskId,
|
||||
taskSnapshot: input.taskSnapshot,
|
||||
status: input.status,
|
||||
overallScore: input.overallScore,
|
||||
maxScore: input.maxScore,
|
||||
categoryScores: input.categoryScores ?? [],
|
||||
rationale: input.rationale,
|
||||
summary: input.summary,
|
||||
evidence: input.evidence ?? [],
|
||||
deterministicSignals: input.deterministicSignals ?? [],
|
||||
aiSignals: input.aiSignals,
|
||||
followUps: input.followUps ?? [],
|
||||
provenance: input.provenance,
|
||||
metadata: input.metadata,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
|
||||
this.db.prepare(`
|
||||
INSERT INTO eval_task_results (
|
||||
id, runId, taskId, taskSnapshot, status, overallScore, maxScore,
|
||||
categoryScores, rationale, summary, evidence, deterministicSignals, aiSignals,
|
||||
followUps, provenance, metadata, createdAt, updatedAt
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
result.id,
|
||||
result.runId,
|
||||
result.taskId,
|
||||
toJson(result.taskSnapshot),
|
||||
result.status,
|
||||
result.overallScore ?? null,
|
||||
result.maxScore ?? null,
|
||||
toJson(result.categoryScores),
|
||||
result.rationale ?? null,
|
||||
result.summary ?? null,
|
||||
toJson(result.evidence),
|
||||
toJson(result.deterministicSignals),
|
||||
toJsonNullable(result.aiSignals),
|
||||
toJson(result.followUps),
|
||||
toJsonNullable(result.provenance),
|
||||
toJsonNullable(result.metadata),
|
||||
result.createdAt,
|
||||
result.updatedAt,
|
||||
);
|
||||
|
||||
this.db.bumpLastModified();
|
||||
this.emit("result:created", result);
|
||||
return result;
|
||||
}
|
||||
|
||||
getTaskResult(id: string): EvalTaskResult | undefined {
|
||||
const row = this.db.prepare("SELECT * FROM eval_task_results WHERE id = ?").get(id) as Record<string, unknown> | undefined;
|
||||
return row ? this.rowToResult(row) : undefined;
|
||||
}
|
||||
|
||||
listTaskResults(options: EvalTaskResultListOptions = {}): EvalTaskResult[] {
|
||||
const clauses: string[] = [];
|
||||
const params: Array<string | number> = [];
|
||||
if (options.runId) {
|
||||
clauses.push("runId = ?");
|
||||
params.push(options.runId);
|
||||
}
|
||||
if (options.taskId) {
|
||||
clauses.push("taskId = ?");
|
||||
params.push(options.taskId);
|
||||
}
|
||||
if (options.status) {
|
||||
clauses.push("status = ?");
|
||||
params.push(options.status);
|
||||
}
|
||||
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
||||
const limit = options.limit !== undefined ? `LIMIT ${options.limit}` : "";
|
||||
const offset = options.offset !== undefined ? `OFFSET ${options.offset}` : "";
|
||||
|
||||
const rows = this.db.prepare(`
|
||||
SELECT * FROM eval_task_results
|
||||
${where}
|
||||
ORDER BY createdAt ASC, id ASC
|
||||
${limit}
|
||||
${offset}
|
||||
`).all(...params) as Record<string, unknown>[];
|
||||
return rows.map((row) => this.rowToResult(row));
|
||||
}
|
||||
|
||||
updateTaskResult(id: string, input: EvalTaskResultUpdateInput): EvalTaskResult | undefined {
|
||||
const existing = this.getTaskResult(id);
|
||||
if (!existing) return undefined;
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const updated: EvalTaskResult = {
|
||||
...existing,
|
||||
...input,
|
||||
metadata: input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata,
|
||||
provenance: input.provenance ? { ...(existing.provenance ?? {}), ...input.provenance } : existing.provenance,
|
||||
updatedAt: now,
|
||||
};
|
||||
|
||||
this.db.prepare(`
|
||||
UPDATE eval_task_results SET
|
||||
status = ?, overallScore = ?, maxScore = ?, categoryScores = ?, rationale = ?, summary = ?,
|
||||
evidence = ?, deterministicSignals = ?, aiSignals = ?, followUps = ?, provenance = ?, metadata = ?, updatedAt = ?
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
updated.status,
|
||||
updated.overallScore ?? null,
|
||||
updated.maxScore ?? null,
|
||||
toJson(updated.categoryScores),
|
||||
updated.rationale ?? null,
|
||||
updated.summary ?? null,
|
||||
toJson(updated.evidence),
|
||||
toJson(updated.deterministicSignals),
|
||||
toJsonNullable(updated.aiSignals),
|
||||
toJson(updated.followUps),
|
||||
toJsonNullable(updated.provenance),
|
||||
toJsonNullable(updated.metadata),
|
||||
updated.updatedAt,
|
||||
id,
|
||||
);
|
||||
|
||||
this.db.bumpLastModified();
|
||||
this.emit("result:updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
appendRunEvent(runId: string, event: Omit<EvalRunEvent, "id" | "runId" | "seq" | "createdAt">): EvalRunEvent {
|
||||
const run = this.getRun(runId);
|
||||
if (!run) throw new Error(`Eval run not found: ${runId}`);
|
||||
|
||||
const maxSeq = this.db.prepare("SELECT COALESCE(MAX(seq), 0) as maxSeq FROM eval_run_events WHERE runId = ?").get(runId) as { maxSeq: number };
|
||||
const created: EvalRunEvent = {
|
||||
id: generateEventId(),
|
||||
runId,
|
||||
seq: (maxSeq?.maxSeq ?? 0) + 1,
|
||||
type: event.type,
|
||||
message: event.message,
|
||||
status: event.status,
|
||||
taskId: event.taskId,
|
||||
metadata: event.metadata,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
this.db.prepare(`
|
||||
INSERT INTO eval_run_events (id, runId, seq, type, message, status, taskId, metadata, createdAt)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
created.id,
|
||||
created.runId,
|
||||
created.seq,
|
||||
created.type,
|
||||
created.message,
|
||||
created.status ?? null,
|
||||
created.taskId ?? null,
|
||||
toJsonNullable(created.metadata),
|
||||
created.createdAt,
|
||||
);
|
||||
|
||||
this.db.bumpLastModified();
|
||||
this.emit("run:event", { runId, event: created });
|
||||
return created;
|
||||
}
|
||||
|
||||
listRunEvents(runId: string): EvalRunEvent[] {
|
||||
const rows = this.db.prepare("SELECT * FROM eval_run_events WHERE runId = ? ORDER BY seq ASC, id ASC").all(runId) as Record<string, unknown>[];
|
||||
return rows.map((row) => this.rowToEvent(row));
|
||||
}
|
||||
|
||||
private hasActiveRun(projectId: string, trigger: string): boolean {
|
||||
const placeholders = Array.from(ACTIVE_STATUSES).map(() => "?").join(", ");
|
||||
const row = this.db.prepare(`SELECT id FROM eval_runs WHERE projectId = ? AND trigger = ? AND status IN (${placeholders}) LIMIT 1`)
|
||||
.get(projectId, trigger, ...Array.from(ACTIVE_STATUSES)) as { id?: string } | undefined;
|
||||
return Boolean(row?.id);
|
||||
}
|
||||
|
||||
private persistRun(run: EvalRun): void {
|
||||
this.db.prepare(`
|
||||
UPDATE eval_runs SET
|
||||
status = ?, scope = ?, window = ?, requestedTaskIds = ?, evaluatedTaskIds = ?, counts = ?, aggregateScores = ?,
|
||||
summary = ?, error = ?, provenance = ?, metadata = ?, updatedAt = ?, startedAt = ?, completedAt = ?, cancelledAt = ?
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
run.status,
|
||||
run.scope,
|
||||
toJson(run.window),
|
||||
toJson(run.requestedTaskIds),
|
||||
toJson(run.evaluatedTaskIds),
|
||||
toJson(run.counts),
|
||||
toJsonNullable(run.aggregateScores),
|
||||
run.summary ?? null,
|
||||
run.error ?? null,
|
||||
toJsonNullable(run.provenance),
|
||||
toJsonNullable(run.metadata),
|
||||
run.updatedAt,
|
||||
run.startedAt ?? null,
|
||||
run.completedAt ?? null,
|
||||
run.cancelledAt ?? null,
|
||||
run.id,
|
||||
);
|
||||
this.db.bumpLastModified();
|
||||
}
|
||||
|
||||
private rowToRun(row: Record<string, unknown>): EvalRun {
|
||||
return {
|
||||
id: String(row.id),
|
||||
projectId: String(row.projectId),
|
||||
status: row.status as EvalRunStatus,
|
||||
trigger: row.trigger as EvalRun["trigger"],
|
||||
scope: String(row.scope),
|
||||
window: fromJson(row.window as string) ?? {},
|
||||
requestedTaskIds: fromJson<string[]>(row.requestedTaskIds as string) ?? [],
|
||||
evaluatedTaskIds: fromJson<string[]>(row.evaluatedTaskIds as string) ?? [],
|
||||
counts: fromJson(row.counts as string) ?? { totalTasks: 0, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 },
|
||||
aggregateScores: fromJson<Record<string, number>>(row.aggregateScores as string),
|
||||
summary: (row.summary as string | null) ?? undefined,
|
||||
error: (row.error as string | null) ?? undefined,
|
||||
provenance: fromJson(row.provenance as string),
|
||||
metadata: fromJson(row.metadata as string),
|
||||
createdAt: String(row.createdAt),
|
||||
updatedAt: String(row.updatedAt),
|
||||
startedAt: (row.startedAt as string | null) ?? undefined,
|
||||
completedAt: (row.completedAt as string | null) ?? undefined,
|
||||
cancelledAt: (row.cancelledAt as string | null) ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
private rowToResult(row: Record<string, unknown>): EvalTaskResult {
|
||||
return {
|
||||
id: String(row.id),
|
||||
runId: String(row.runId),
|
||||
taskId: String(row.taskId),
|
||||
taskSnapshot: fromJson(row.taskSnapshot as string) ?? { taskId: String(row.taskId) },
|
||||
status: row.status as EvalTaskResult["status"],
|
||||
overallScore: row.overallScore == null ? undefined : Number(row.overallScore),
|
||||
maxScore: row.maxScore == null ? undefined : Number(row.maxScore),
|
||||
categoryScores: fromJson(row.categoryScores as string) ?? [],
|
||||
rationale: (row.rationale as string | null) ?? undefined,
|
||||
summary: (row.summary as string | null) ?? undefined,
|
||||
evidence: fromJson(row.evidence as string) ?? [],
|
||||
deterministicSignals: fromJson(row.deterministicSignals as string) ?? [],
|
||||
aiSignals: fromJson(row.aiSignals as string),
|
||||
followUps: fromJson(row.followUps as string) ?? [],
|
||||
provenance: fromJson(row.provenance as string),
|
||||
metadata: fromJson(row.metadata as string),
|
||||
createdAt: String(row.createdAt),
|
||||
updatedAt: String(row.updatedAt),
|
||||
};
|
||||
}
|
||||
|
||||
private rowToEvent(row: Record<string, unknown>): EvalRunEvent {
|
||||
return {
|
||||
id: String(row.id),
|
||||
runId: String(row.runId),
|
||||
seq: Number(row.seq),
|
||||
type: row.type as EvalRunEvent["type"],
|
||||
message: String(row.message),
|
||||
status: (row.status as EvalRunStatus | null) ?? undefined,
|
||||
taskId: (row.taskId as string | null) ?? undefined,
|
||||
metadata: fromJson(row.metadata as string),
|
||||
createdAt: String(row.createdAt),
|
||||
};
|
||||
}
|
||||
}
|
||||
239
packages/core/src/eval-types.ts
Normal file
239
packages/core/src/eval-types.ts
Normal file
@@ -0,0 +1,239 @@
|
||||
/**
|
||||
* Eval Domain Types
|
||||
*
|
||||
* Contracts for eval run persistence and per-task evaluation results.
|
||||
*/
|
||||
|
||||
export const EVAL_RUN_STATUSES = [
|
||||
"pending",
|
||||
"running",
|
||||
"completed",
|
||||
"failed",
|
||||
"cancelled",
|
||||
] as const;
|
||||
|
||||
export type EvalRunStatus = typeof EVAL_RUN_STATUSES[number];
|
||||
|
||||
export const EVAL_RUN_TRIGGERS = ["manual", "schedule", "api", "task_completion"] as const;
|
||||
|
||||
export type EvalRunTrigger = typeof EVAL_RUN_TRIGGERS[number];
|
||||
|
||||
export const EVAL_SCORE_CATEGORIES = [
|
||||
"correctness",
|
||||
"completeness",
|
||||
"quality",
|
||||
"reliability",
|
||||
"tests",
|
||||
"documentation",
|
||||
] as const;
|
||||
|
||||
export type EvalScoreCategory = typeof EVAL_SCORE_CATEGORIES[number];
|
||||
|
||||
export interface EvalTaskSnapshot {
|
||||
taskId: string;
|
||||
title?: string;
|
||||
column?: string;
|
||||
status?: string;
|
||||
priority?: string;
|
||||
size?: string;
|
||||
reviewLevel?: number;
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
executionCompletedAt?: string;
|
||||
summary?: string;
|
||||
labels?: string[];
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface EvalRunWindow {
|
||||
since?: string;
|
||||
until?: string;
|
||||
baselineRunId?: string;
|
||||
}
|
||||
|
||||
export interface EvalProvenance {
|
||||
evaluatorProvider?: string;
|
||||
evaluatorModelId?: string;
|
||||
evaluatorVersion?: string;
|
||||
promptVersion?: string;
|
||||
runConfig?: Record<string, unknown>;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface EvalSignal {
|
||||
signalId: string;
|
||||
kind: string;
|
||||
name: string;
|
||||
passed?: boolean;
|
||||
score?: number;
|
||||
value?: number | string | boolean | null;
|
||||
threshold?: number;
|
||||
unit?: string;
|
||||
summary?: string;
|
||||
details?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface EvalEvidenceReference {
|
||||
type: "task_log" | "task_document" | "file" | "command" | "test" | "other";
|
||||
ref: string;
|
||||
excerpt?: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface EvalCategoryScore {
|
||||
category: EvalScoreCategory | string;
|
||||
score: number;
|
||||
maxScore?: number;
|
||||
rationale?: string;
|
||||
}
|
||||
|
||||
export interface EvalFollowUpSuggestion {
|
||||
title: string;
|
||||
description: string;
|
||||
priority?: "low" | "normal" | "high" | "urgent";
|
||||
tags?: string[];
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface EvalTaskResult {
|
||||
id: string;
|
||||
runId: string;
|
||||
taskId: string;
|
||||
taskSnapshot: EvalTaskSnapshot;
|
||||
status: "scored" | "skipped" | "error";
|
||||
overallScore?: number;
|
||||
maxScore?: number;
|
||||
categoryScores: EvalCategoryScore[];
|
||||
rationale?: string;
|
||||
summary?: string;
|
||||
evidence: EvalEvidenceReference[];
|
||||
deterministicSignals: EvalSignal[];
|
||||
aiSignals?: EvalSignal[];
|
||||
followUps: EvalFollowUpSuggestion[];
|
||||
provenance?: EvalProvenance;
|
||||
metadata?: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface EvalRunCounts {
|
||||
totalTasks: number;
|
||||
scoredTasks: number;
|
||||
skippedTasks: number;
|
||||
erroredTasks: number;
|
||||
}
|
||||
|
||||
export interface EvalRun {
|
||||
id: string;
|
||||
projectId: string;
|
||||
status: EvalRunStatus;
|
||||
trigger: EvalRunTrigger;
|
||||
scope: string;
|
||||
window: EvalRunWindow;
|
||||
requestedTaskIds: string[];
|
||||
evaluatedTaskIds: string[];
|
||||
counts: EvalRunCounts;
|
||||
aggregateScores?: Record<string, number>;
|
||||
summary?: string;
|
||||
error?: string;
|
||||
provenance?: EvalProvenance;
|
||||
metadata?: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
startedAt?: string;
|
||||
completedAt?: string;
|
||||
cancelledAt?: string;
|
||||
}
|
||||
|
||||
export interface EvalRunEvent {
|
||||
id: string;
|
||||
runId: string;
|
||||
seq: number;
|
||||
type: "status_changed" | "task_evaluated" | "info" | "warning" | "error";
|
||||
message: string;
|
||||
status?: EvalRunStatus;
|
||||
taskId?: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface EvalRunCreateInput {
|
||||
projectId: string;
|
||||
trigger?: EvalRunTrigger;
|
||||
scope: string;
|
||||
window?: EvalRunWindow;
|
||||
requestedTaskIds?: string[];
|
||||
provenance?: EvalProvenance;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface EvalRunUpdateInput {
|
||||
status?: EvalRunStatus;
|
||||
evaluatedTaskIds?: string[];
|
||||
counts?: EvalRunCounts;
|
||||
aggregateScores?: Record<string, number>;
|
||||
summary?: string;
|
||||
error?: string | null;
|
||||
provenance?: EvalProvenance;
|
||||
metadata?: Record<string, unknown>;
|
||||
startedAt?: string | null;
|
||||
completedAt?: string | null;
|
||||
cancelledAt?: string | null;
|
||||
}
|
||||
|
||||
export interface EvalRunListOptions {
|
||||
projectId?: string;
|
||||
status?: EvalRunStatus;
|
||||
trigger?: EvalRunTrigger;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
export interface EvalTaskResultCreateInput {
|
||||
taskId: string;
|
||||
taskSnapshot: EvalTaskSnapshot;
|
||||
status: "scored" | "skipped" | "error";
|
||||
overallScore?: number;
|
||||
maxScore?: number;
|
||||
categoryScores?: EvalCategoryScore[];
|
||||
rationale?: string;
|
||||
summary?: string;
|
||||
evidence?: EvalEvidenceReference[];
|
||||
deterministicSignals?: EvalSignal[];
|
||||
aiSignals?: EvalSignal[];
|
||||
followUps?: EvalFollowUpSuggestion[];
|
||||
provenance?: EvalProvenance;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface EvalTaskResultUpdateInput {
|
||||
status?: "scored" | "skipped" | "error";
|
||||
overallScore?: number;
|
||||
maxScore?: number;
|
||||
categoryScores?: EvalCategoryScore[];
|
||||
rationale?: string;
|
||||
summary?: string;
|
||||
evidence?: EvalEvidenceReference[];
|
||||
deterministicSignals?: EvalSignal[];
|
||||
aiSignals?: EvalSignal[];
|
||||
followUps?: EvalFollowUpSuggestion[];
|
||||
provenance?: EvalProvenance;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface EvalTaskResultListOptions {
|
||||
runId?: string;
|
||||
taskId?: string;
|
||||
status?: "scored" | "skipped" | "error";
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
export interface EvalStoreEvents {
|
||||
"run:created": [EvalRun];
|
||||
"run:updated": [EvalRun];
|
||||
"run:deleted": [string];
|
||||
"run:event": [{ runId: string; event: EvalRunEvent }];
|
||||
"result:created": [EvalTaskResult];
|
||||
"result:updated": [EvalTaskResult];
|
||||
}
|
||||
@@ -695,6 +695,30 @@ export type { ResolvedResearchSettings } from "./research-settings.js";
|
||||
|
||||
export { TodoStore } from "./todo-store.js";
|
||||
export type { TodoStoreEvents } from "./todo-store.js";
|
||||
export { EvalLifecycleError, EvalStore } from "./eval-store.js";
|
||||
export type {
|
||||
EvalRun,
|
||||
EvalRunStatus,
|
||||
EvalRunTrigger,
|
||||
EvalRunWindow,
|
||||
EvalRunCounts,
|
||||
EvalRunEvent,
|
||||
EvalRunCreateInput,
|
||||
EvalRunUpdateInput,
|
||||
EvalRunListOptions,
|
||||
EvalTaskSnapshot,
|
||||
EvalTaskResult,
|
||||
EvalTaskResultCreateInput,
|
||||
EvalTaskResultUpdateInput,
|
||||
EvalTaskResultListOptions,
|
||||
EvalCategoryScore,
|
||||
EvalEvidenceReference,
|
||||
EvalSignal,
|
||||
EvalFollowUpSuggestion,
|
||||
EvalProvenance,
|
||||
EvalStoreEvents,
|
||||
} from "./eval-types.js";
|
||||
export { EVAL_RUN_STATUSES, EVAL_RUN_TRIGGERS, EVAL_SCORE_CATEGORIES } from "./eval-types.js";
|
||||
|
||||
// ── Agent Companies Types ──────────────────────────────────
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import { RoadmapStore } from "./roadmap-store.js";
|
||||
import { InsightStore } from "./insight-store.js";
|
||||
import { ResearchStore } from "./research-store.js";
|
||||
import { TodoStore } from "./todo-store.js";
|
||||
import { EvalStore } from "./eval-store.js";
|
||||
import { BackwardCompat, ProjectRequiredError } from "./migration.js";
|
||||
import { CentralCore } from "./central-core.js";
|
||||
import { getTaskMergeBlocker } from "./task-merge.js";
|
||||
@@ -512,6 +513,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
private researchStore: ResearchStore | null = null;
|
||||
/** Cached TodoStore instance */
|
||||
private todoStore: TodoStore | null = null;
|
||||
/** Cached EvalStore instance */
|
||||
private evalStore: EvalStore | null = null;
|
||||
|
||||
/** Buffer for batching agent log writes to reduce WAL pressure. */
|
||||
private agentLogBuffer: Array<{
|
||||
@@ -6592,6 +6595,17 @@ ${notificationsSection}`;
|
||||
return this.todoStore;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the EvalStore instance for eval run and task result operations.
|
||||
* Lazily initializes the EvalStore on first access.
|
||||
*/
|
||||
getEvalStore(): EvalStore {
|
||||
if (!this.evalStore) {
|
||||
this.evalStore = new EvalStore(this.db);
|
||||
}
|
||||
return this.evalStore;
|
||||
}
|
||||
|
||||
// ── Verification Cache ────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
@@ -1231,7 +1231,8 @@ export function __setCreateFnAgent(mock: typeof createFnAgent): void {
|
||||
// hit the real engine. Mirror the same fake into the resolved-session slot
|
||||
// so existing test setups that only call `__setCreateFnAgent` continue to
|
||||
// work.
|
||||
createResolvedAgentSession = (async (options: unknown) => mock(options)) as typeof createResolvedAgentSession;
|
||||
createResolvedAgentSession = (async (options: Parameters<typeof createResolvedAgentSession>[0]) =>
|
||||
mock(options)) as typeof createResolvedAgentSession;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user