feat(FN-3387): add eval domain with store and persistence schema

Introduced a new evaluation persistence layer in `@fusion/core` with domain contracts, SQLite-backed storage APIs, and retention/window-rollup support, wired through the core store and documented in the architecture and storage docs.

Fusion-Task-Id: FN-3387
This commit is contained in:
Fusion
2026-05-05 11:56:11 -07:00
committed by gsxdsm
parent b4bc105726
commit c87c9e2e1f
15 changed files with 989 additions and 25 deletions

View File

@@ -160,7 +160,7 @@ describe("Database", () => {
});
it("seeds schema version", () => {
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
});
it("seeds lastModified", () => {
@@ -183,7 +183,7 @@ describe("Database", () => {
it("is idempotent - calling init() twice does not fail", () => {
expect(() => db.init()).not.toThrow();
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
});
it("does not overwrite existing config on re-init", () => {
@@ -957,7 +957,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 29 (includes v1→v2 through v26→v29)
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -982,11 +982,11 @@ describe("schema migrations", () => {
const db = new Database(fusionDir);
db.init();
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
// Re-init should not fail
db.init();
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
db.close();
});
@@ -1021,7 +1021,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
expect(cols.map((col) => col.name)).toContain("priority");
@@ -1062,7 +1062,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
const colNames = cols.map((col) => col.name);
@@ -1131,7 +1131,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
const colNames = cols.map((col) => col.name);
@@ -1234,7 +1234,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
const cols = db.prepare("PRAGMA table_info(chat_messages)").all() as Array<{ name: string }>;
expect(cols.map((col) => col.name)).toContain("attachments");
@@ -1308,7 +1308,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>;
expect(tables).toEqual([{ name: "agentRatings" }]);
@@ -1332,7 +1332,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>;
expect(tables).toEqual([{ name: "mission_events" }]);
@@ -1436,7 +1436,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 29
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -1905,7 +1905,7 @@ describe("createDatabase factory", () => {
const db = createDatabase(fusionDir);
db.init();
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
expect(db.getLastModified()).toBeGreaterThan(0);
db.close();

View File

@@ -0,0 +1,82 @@
import { beforeEach, describe, expect, it } from "vitest";
import { createDatabase, type Database } from "../db.js";
import { EvalLifecycleError, EvalStore } from "../eval-store.js";
let db: Database;
let store: EvalStore;
beforeEach(() => {
db = createDatabase("/tmp/fn-eval-store-test", { inMemory: true });
db.init();
store = new EvalStore(db);
});
describe("EvalStore", () => {
it("creates and lists runs with deterministic ordering", () => {
const runA = store.createRun({ projectId: "p1", scope: "completed-since-last", requestedTaskIds: ["FN-1"] });
const runB = store.createRun({ projectId: "p1", scope: "completed-since-last", requestedTaskIds: ["FN-2"] });
const runs = store.listRuns({ projectId: "p1" });
expect(runs.map((run) => run.id)).toEqual([runA.id, runB.id].sort());
});
it("enforces active run conflict for scheduled trigger", () => {
store.createRun({ projectId: "p1", scope: "window", trigger: "schedule" });
expect(() => store.createRun({ projectId: "p1", scope: "window", trigger: "schedule" })).toThrow(EvalLifecycleError);
});
it("enforces terminal immutability", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
store.updateRun(run.id, { status: "completed" });
expect(() => store.updateRun(run.id, { summary: "late change" })).toThrow(EvalLifecycleError);
});
it("creates results and preserves task snapshot after tasks row deletion", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
const result = store.createTaskResult(run.id, {
taskId: "FN-123",
taskSnapshot: { taskId: "FN-123", title: "Snapshot title", status: "done", summary: "task summary" },
status: "scored",
overallScore: 0.8,
categoryScores: [{ category: "quality", score: 0.8 }],
evidence: [{ type: "task_log", ref: "log:1" }],
deterministicSignals: [{ signalId: "s1", kind: "test", name: "tests-pass", passed: true }],
});
db.prepare("DELETE FROM tasks WHERE id = ?").run("FN-123");
const fetched = store.getTaskResult(result.id);
expect(fetched?.taskSnapshot.title).toBe("Snapshot title");
expect(fetched?.taskId).toBe("FN-123");
});
it("persists run window boundaries and evaluated task rollups", () => {
const run = store.createRun({
projectId: "p1",
trigger: "schedule",
scope: "completed-since-last",
window: { since: "2026-05-01T00:00:00.000Z", until: "2026-05-02T00:00:00.000Z", baselineRunId: "ER-BASE" },
requestedTaskIds: ["FN-1", "FN-2"],
});
const updated = store.updateRun(run.id, {
status: "running",
evaluatedTaskIds: ["FN-1", "FN-2"],
counts: { totalTasks: 2, scoredTasks: 1, skippedTasks: 1, erroredTasks: 0 },
});
expect(updated?.window.since).toBe("2026-05-01T00:00:00.000Z");
expect(updated?.evaluatedTaskIds).toEqual(["FN-1", "FN-2"]);
expect(updated?.counts.scoredTasks).toBe(1);
});
it("appends run events with sequential ordering", () => {
const run = store.createRun({ projectId: "p1", scope: "window" });
const evt1 = store.appendRunEvent(run.id, { type: "info", message: "started" });
const evt2 = store.appendRunEvent(run.id, { type: "task_evaluated", message: "scored", taskId: "FN-1" });
const events = store.listRunEvents(run.id);
expect(events.map((event) => event.id)).toEqual([evt1.id, evt2.id]);
expect(events.map((event) => event.seq)).toEqual([1, 2]);
});
});

View File

@@ -886,7 +886,7 @@ describe("Migration: pre-33 DB upgrade", () => {
// Step 1: Create a fresh database at v33 (runs all migrations up to 33)
const db1 = createDatabase(legacyDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(61);
expect(db1.getSchemaVersion()).toBe(62);
db1.close();
// Step 2: Manually downgrade to version 32 and drop insight tables
@@ -921,7 +921,7 @@ describe("Migration: pre-33 DB upgrade", () => {
expect(tableNamesBefore).not.toContain("project_insight_runs");
// Now run init — this triggers the v32→v33 migration
db3.init();
expect(db3.getSchemaVersion()).toBe(61);
expect(db3.getSchemaVersion()).toBe(62);
// Step 4: Verify insight tables exist after migration
const tablesAfter = db3.prepare(
@@ -952,12 +952,12 @@ describe("Migration: pre-33 DB upgrade", () => {
try {
const db1 = createDatabase(testDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(61);
expect(db1.getSchemaVersion()).toBe(62);
db1.close();
const db2 = createDatabase(testDir);
expect(() => db2.init()).not.toThrow();
expect(db2.getSchemaVersion()).toBe(61);
expect(db2.getSchemaVersion()).toBe(62);
db2.close();
} finally {
rmSync(testDir, { recursive: true, force: true });

View File

@@ -2629,7 +2629,7 @@ describe("MissionStore", () => {
describe("Loop State & Validator Run Schema (v31)", () => {
it("schema version is 40 after migration", () => {
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
});
it("mission_features table has loop state columns", () => {

View File

@@ -742,7 +742,7 @@ describe("RoadmapStore", () => {
describe("schema version", () => {
it("schema version is 40 after init", () => {
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
});
});

View File

@@ -465,7 +465,7 @@ describe("Run Audit", () => {
});
it("schema version is bumped to 40", () => {
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
});
});
});

View File

@@ -51,7 +51,7 @@ describe("TaskStore task documents", () => {
expect(tableNames.has("task_documents")).toBe(true);
expect(tableNames.has("task_document_revisions")).toBe(true);
expect(db.getSchemaVersion()).toBe(61);
expect(db.getSchemaVersion()).toBe(62);
const index = db
.prepare(

View File

@@ -88,7 +88,7 @@ export function probeFts5(db: DatabaseSync): boolean {
// ── Schema Definition ────────────────────────────────────────────────
const SCHEMA_VERSION = 61;
const SCHEMA_VERSION = 62;
function normalizeTaskComments(
steeringComments: SteeringComment[] | undefined,
@@ -466,6 +466,71 @@ CREATE TABLE IF NOT EXISTS research_run_events (
);
CREATE INDEX IF NOT EXISTS idxResearchRunEventsRunIdSeq ON research_run_events(runId, seq);
-- Eval run persistence (FN-3387)
CREATE TABLE IF NOT EXISTS eval_runs (
id TEXT PRIMARY KEY,
projectId TEXT NOT NULL,
status TEXT NOT NULL,
trigger TEXT NOT NULL,
scope TEXT NOT NULL,
window TEXT NOT NULL DEFAULT '{}',
requestedTaskIds TEXT NOT NULL DEFAULT '[]',
evaluatedTaskIds TEXT NOT NULL DEFAULT '[]',
counts TEXT NOT NULL DEFAULT '{"totalTasks":0,"scoredTasks":0,"skippedTasks":0,"erroredTasks":0}',
aggregateScores TEXT,
summary TEXT,
error TEXT,
provenance TEXT,
metadata TEXT,
createdAt TEXT NOT NULL,
updatedAt TEXT NOT NULL,
startedAt TEXT,
completedAt TEXT,
cancelledAt TEXT
);
CREATE INDEX IF NOT EXISTS idxEvalRunsProjectIdCreatedAt ON eval_runs(projectId, createdAt);
CREATE INDEX IF NOT EXISTS idxEvalRunsProjectTriggerStatus ON eval_runs(projectId, trigger, status);
CREATE INDEX IF NOT EXISTS idxEvalRunsStatusCreatedAt ON eval_runs(status, createdAt);
CREATE TABLE IF NOT EXISTS eval_task_results (
id TEXT PRIMARY KEY,
runId TEXT NOT NULL,
taskId TEXT NOT NULL,
taskSnapshot TEXT NOT NULL,
status TEXT NOT NULL,
overallScore REAL,
maxScore REAL,
categoryScores TEXT NOT NULL DEFAULT '[]',
rationale TEXT,
summary TEXT,
evidence TEXT NOT NULL DEFAULT '[]',
deterministicSignals TEXT NOT NULL DEFAULT '[]',
aiSignals TEXT,
followUps TEXT NOT NULL DEFAULT '[]',
provenance TEXT,
metadata TEXT,
createdAt TEXT NOT NULL,
updatedAt TEXT NOT NULL,
FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idxEvalTaskResultsRunIdCreatedAt ON eval_task_results(runId, createdAt);
CREATE INDEX IF NOT EXISTS idxEvalTaskResultsTaskIdCreatedAt ON eval_task_results(taskId, createdAt);
CREATE INDEX IF NOT EXISTS idxEvalTaskResultsStatusRunId ON eval_task_results(status, runId);
CREATE TABLE IF NOT EXISTS eval_run_events (
id TEXT PRIMARY KEY,
runId TEXT NOT NULL,
seq INTEGER NOT NULL,
type TEXT NOT NULL,
message TEXT NOT NULL,
status TEXT,
taskId TEXT,
metadata TEXT,
createdAt TEXT NOT NULL,
FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idxEvalRunEventsRunIdSeq ON eval_run_events(runId, seq);
-- Schema version tracking
CREATE TABLE IF NOT EXISTS __meta (
key TEXT PRIMARY KEY,
@@ -2413,6 +2478,80 @@ export class Database {
});
}
if (version < 62) {
this.applyMigration(62, () => {
this.db.exec(`
CREATE TABLE IF NOT EXISTS eval_runs (
id TEXT PRIMARY KEY,
projectId TEXT NOT NULL,
status TEXT NOT NULL,
trigger TEXT NOT NULL,
scope TEXT NOT NULL,
window TEXT NOT NULL DEFAULT '{}',
requestedTaskIds TEXT NOT NULL DEFAULT '[]',
evaluatedTaskIds TEXT NOT NULL DEFAULT '[]',
counts TEXT NOT NULL DEFAULT '{"totalTasks":0,"scoredTasks":0,"skippedTasks":0,"erroredTasks":0}',
aggregateScores TEXT,
summary TEXT,
error TEXT,
provenance TEXT,
metadata TEXT,
createdAt TEXT NOT NULL,
updatedAt TEXT NOT NULL,
startedAt TEXT,
completedAt TEXT,
cancelledAt TEXT
)
`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsProjectIdCreatedAt ON eval_runs(projectId, createdAt)`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsProjectTriggerStatus ON eval_runs(projectId, trigger, status)`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunsStatusCreatedAt ON eval_runs(status, createdAt)`);
this.db.exec(`
CREATE TABLE IF NOT EXISTS eval_task_results (
id TEXT PRIMARY KEY,
runId TEXT NOT NULL,
taskId TEXT NOT NULL,
taskSnapshot TEXT NOT NULL,
status TEXT NOT NULL,
overallScore REAL,
maxScore REAL,
categoryScores TEXT NOT NULL DEFAULT '[]',
rationale TEXT,
summary TEXT,
evidence TEXT NOT NULL DEFAULT '[]',
deterministicSignals TEXT NOT NULL DEFAULT '[]',
aiSignals TEXT,
followUps TEXT NOT NULL DEFAULT '[]',
provenance TEXT,
metadata TEXT,
createdAt TEXT NOT NULL,
updatedAt TEXT NOT NULL,
FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE
)
`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsRunIdCreatedAt ON eval_task_results(runId, createdAt)`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsTaskIdCreatedAt ON eval_task_results(taskId, createdAt)`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalTaskResultsStatusRunId ON eval_task_results(status, runId)`);
this.db.exec(`
CREATE TABLE IF NOT EXISTS eval_run_events (
id TEXT PRIMARY KEY,
runId TEXT NOT NULL,
seq INTEGER NOT NULL,
type TEXT NOT NULL,
message TEXT NOT NULL,
status TEXT,
taskId TEXT,
metadata TEXT,
createdAt TEXT NOT NULL,
FOREIGN KEY (runId) REFERENCES eval_runs(id) ON DELETE CASCADE
)
`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idxEvalRunEventsRunIdSeq ON eval_run_events(runId, seq)`);
});
}
}
/**

View File

@@ -0,0 +1,454 @@
import { EventEmitter } from "node:events";
import { randomUUID } from "node:crypto";
import type { Database } from "./db.js";
import { fromJson, toJson, toJsonNullable } from "./db.js";
import type {
EvalRun,
EvalRunCreateInput,
EvalRunEvent,
EvalRunListOptions,
EvalRunStatus,
EvalRunUpdateInput,
EvalStoreEvents,
EvalTaskResult,
EvalTaskResultCreateInput,
EvalTaskResultListOptions,
EvalTaskResultUpdateInput,
} from "./eval-types.js";
const TERMINAL_STATUSES = new Set<EvalRunStatus>(["completed", "failed", "cancelled"]);
const ACTIVE_STATUSES = new Set<EvalRunStatus>(["pending", "running"]);
const VALID_TRANSITIONS: Record<EvalRunStatus, EvalRunStatus[]> = {
pending: ["running", "completed", "failed", "cancelled"],
running: ["completed", "failed", "cancelled"],
completed: [],
failed: [],
cancelled: [],
};
export class EvalLifecycleError extends Error {
constructor(message: string, readonly code: "invalid_transition" | "terminal_immutable" | "active_run_conflict") {
super(message);
this.name = "EvalLifecycleError";
}
}
function generateRunId(): string {
return `ER-${Date.now().toString(36).toUpperCase()}-${Math.random().toString(36).slice(2, 7).toUpperCase()}`;
}
function generateResultId(): string {
return `ETR-${randomUUID()}`;
}
function generateEventId(): string {
return `ERE-${randomUUID()}`;
}
export class EvalStore extends EventEmitter<EvalStoreEvents> {
constructor(private readonly db: Database) {
super();
this.setMaxListeners(50);
}
createRun(input: EvalRunCreateInput): EvalRun {
const now = new Date().toISOString();
if ((input.trigger === "schedule" || input.trigger === "task_completion") && this.hasActiveRun(input.projectId, input.trigger)) {
throw new EvalLifecycleError(`Active eval run already exists for project ${input.projectId} trigger ${input.trigger}`, "active_run_conflict");
}
const run: EvalRun = {
id: generateRunId(),
projectId: input.projectId,
status: "pending",
trigger: input.trigger ?? "manual",
scope: input.scope,
window: input.window ?? {},
requestedTaskIds: input.requestedTaskIds ?? [],
evaluatedTaskIds: [],
counts: { totalTasks: input.requestedTaskIds?.length ?? 0, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 },
provenance: input.provenance,
metadata: input.metadata,
createdAt: now,
updatedAt: now,
};
this.db.prepare(`
INSERT INTO eval_runs (
id, projectId, status, trigger, scope, window, requestedTaskIds, evaluatedTaskIds,
counts, aggregateScores, summary, error, provenance, metadata,
createdAt, updatedAt, startedAt, completedAt, cancelledAt
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
run.id,
run.projectId,
run.status,
run.trigger,
run.scope,
toJson(run.window),
toJson(run.requestedTaskIds),
toJson(run.evaluatedTaskIds),
toJson(run.counts),
null,
null,
null,
toJsonNullable(run.provenance),
toJsonNullable(run.metadata),
run.createdAt,
run.updatedAt,
null,
null,
null,
);
this.db.bumpLastModified();
this.emit("run:created", run);
return run;
}
getRun(id: string): EvalRun | undefined {
const row = this.db.prepare("SELECT * FROM eval_runs WHERE id = ?").get(id) as Record<string, unknown> | undefined;
return row ? this.rowToRun(row) : undefined;
}
listRuns(options: EvalRunListOptions = {}): EvalRun[] {
const clauses: string[] = [];
const params: Array<string | number> = [];
if (options.projectId) {
clauses.push("projectId = ?");
params.push(options.projectId);
}
if (options.status) {
clauses.push("status = ?");
params.push(options.status);
}
if (options.trigger) {
clauses.push("trigger = ?");
params.push(options.trigger);
}
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
const limit = options.limit !== undefined ? `LIMIT ${options.limit}` : "";
const offset = options.offset !== undefined ? `OFFSET ${options.offset}` : "";
const rows = this.db.prepare(`
SELECT * FROM eval_runs
${where}
ORDER BY createdAt ASC, id ASC
${limit}
${offset}
`).all(...params) as Record<string, unknown>[];
return rows.map((row) => this.rowToRun(row));
}
updateRun(id: string, input: EvalRunUpdateInput): EvalRun | undefined {
const existing = this.getRun(id);
if (!existing) return undefined;
if (TERMINAL_STATUSES.has(existing.status) && Object.keys(input).some((k) => k !== "status")) {
throw new EvalLifecycleError(`Eval run ${id} is terminal and immutable`, "terminal_immutable");
}
if (input.status && input.status !== existing.status) {
if (!VALID_TRANSITIONS[existing.status].includes(input.status)) {
throw new EvalLifecycleError(`Invalid eval run status transition: ${existing.status} -> ${input.status}`, "invalid_transition");
}
}
const now = new Date().toISOString();
const updated: EvalRun = {
...existing,
...input,
error: input.error === null ? undefined : (input.error ?? existing.error),
metadata: input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata,
provenance: input.provenance ? { ...(existing.provenance ?? {}), ...input.provenance } : existing.provenance,
updatedAt: now,
startedAt: input.startedAt === null ? undefined : (input.startedAt ?? existing.startedAt),
completedAt: input.completedAt === null ? undefined : (input.completedAt ?? existing.completedAt),
cancelledAt: input.cancelledAt === null ? undefined : (input.cancelledAt ?? existing.cancelledAt),
};
this.persistRun(updated);
this.emit("run:updated", updated);
return updated;
}
deleteRun(id: string): boolean {
const result = this.db.prepare("DELETE FROM eval_runs WHERE id = ?").run(id) as { changes?: number };
const deleted = (result.changes ?? 0) > 0;
if (deleted) {
this.db.bumpLastModified();
this.emit("run:deleted", id);
}
return deleted;
}
createTaskResult(runId: string, input: EvalTaskResultCreateInput): EvalTaskResult {
const run = this.getRun(runId);
if (!run) throw new Error(`Eval run not found: ${runId}`);
const now = new Date().toISOString();
const result: EvalTaskResult = {
id: generateResultId(),
runId,
taskId: input.taskId,
taskSnapshot: input.taskSnapshot,
status: input.status,
overallScore: input.overallScore,
maxScore: input.maxScore,
categoryScores: input.categoryScores ?? [],
rationale: input.rationale,
summary: input.summary,
evidence: input.evidence ?? [],
deterministicSignals: input.deterministicSignals ?? [],
aiSignals: input.aiSignals,
followUps: input.followUps ?? [],
provenance: input.provenance,
metadata: input.metadata,
createdAt: now,
updatedAt: now,
};
this.db.prepare(`
INSERT INTO eval_task_results (
id, runId, taskId, taskSnapshot, status, overallScore, maxScore,
categoryScores, rationale, summary, evidence, deterministicSignals, aiSignals,
followUps, provenance, metadata, createdAt, updatedAt
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
result.id,
result.runId,
result.taskId,
toJson(result.taskSnapshot),
result.status,
result.overallScore ?? null,
result.maxScore ?? null,
toJson(result.categoryScores),
result.rationale ?? null,
result.summary ?? null,
toJson(result.evidence),
toJson(result.deterministicSignals),
toJsonNullable(result.aiSignals),
toJson(result.followUps),
toJsonNullable(result.provenance),
toJsonNullable(result.metadata),
result.createdAt,
result.updatedAt,
);
this.db.bumpLastModified();
this.emit("result:created", result);
return result;
}
getTaskResult(id: string): EvalTaskResult | undefined {
const row = this.db.prepare("SELECT * FROM eval_task_results WHERE id = ?").get(id) as Record<string, unknown> | undefined;
return row ? this.rowToResult(row) : undefined;
}
listTaskResults(options: EvalTaskResultListOptions = {}): EvalTaskResult[] {
const clauses: string[] = [];
const params: Array<string | number> = [];
if (options.runId) {
clauses.push("runId = ?");
params.push(options.runId);
}
if (options.taskId) {
clauses.push("taskId = ?");
params.push(options.taskId);
}
if (options.status) {
clauses.push("status = ?");
params.push(options.status);
}
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
const limit = options.limit !== undefined ? `LIMIT ${options.limit}` : "";
const offset = options.offset !== undefined ? `OFFSET ${options.offset}` : "";
const rows = this.db.prepare(`
SELECT * FROM eval_task_results
${where}
ORDER BY createdAt ASC, id ASC
${limit}
${offset}
`).all(...params) as Record<string, unknown>[];
return rows.map((row) => this.rowToResult(row));
}
updateTaskResult(id: string, input: EvalTaskResultUpdateInput): EvalTaskResult | undefined {
const existing = this.getTaskResult(id);
if (!existing) return undefined;
const now = new Date().toISOString();
const updated: EvalTaskResult = {
...existing,
...input,
metadata: input.metadata ? { ...(existing.metadata ?? {}), ...input.metadata } : existing.metadata,
provenance: input.provenance ? { ...(existing.provenance ?? {}), ...input.provenance } : existing.provenance,
updatedAt: now,
};
this.db.prepare(`
UPDATE eval_task_results SET
status = ?, overallScore = ?, maxScore = ?, categoryScores = ?, rationale = ?, summary = ?,
evidence = ?, deterministicSignals = ?, aiSignals = ?, followUps = ?, provenance = ?, metadata = ?, updatedAt = ?
WHERE id = ?
`).run(
updated.status,
updated.overallScore ?? null,
updated.maxScore ?? null,
toJson(updated.categoryScores),
updated.rationale ?? null,
updated.summary ?? null,
toJson(updated.evidence),
toJson(updated.deterministicSignals),
toJsonNullable(updated.aiSignals),
toJson(updated.followUps),
toJsonNullable(updated.provenance),
toJsonNullable(updated.metadata),
updated.updatedAt,
id,
);
this.db.bumpLastModified();
this.emit("result:updated", updated);
return updated;
}
appendRunEvent(runId: string, event: Omit<EvalRunEvent, "id" | "runId" | "seq" | "createdAt">): EvalRunEvent {
const run = this.getRun(runId);
if (!run) throw new Error(`Eval run not found: ${runId}`);
const maxSeq = this.db.prepare("SELECT COALESCE(MAX(seq), 0) as maxSeq FROM eval_run_events WHERE runId = ?").get(runId) as { maxSeq: number };
const created: EvalRunEvent = {
id: generateEventId(),
runId,
seq: (maxSeq?.maxSeq ?? 0) + 1,
type: event.type,
message: event.message,
status: event.status,
taskId: event.taskId,
metadata: event.metadata,
createdAt: new Date().toISOString(),
};
this.db.prepare(`
INSERT INTO eval_run_events (id, runId, seq, type, message, status, taskId, metadata, createdAt)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
created.id,
created.runId,
created.seq,
created.type,
created.message,
created.status ?? null,
created.taskId ?? null,
toJsonNullable(created.metadata),
created.createdAt,
);
this.db.bumpLastModified();
this.emit("run:event", { runId, event: created });
return created;
}
listRunEvents(runId: string): EvalRunEvent[] {
const rows = this.db.prepare("SELECT * FROM eval_run_events WHERE runId = ? ORDER BY seq ASC, id ASC").all(runId) as Record<string, unknown>[];
return rows.map((row) => this.rowToEvent(row));
}
private hasActiveRun(projectId: string, trigger: string): boolean {
const placeholders = Array.from(ACTIVE_STATUSES).map(() => "?").join(", ");
const row = this.db.prepare(`SELECT id FROM eval_runs WHERE projectId = ? AND trigger = ? AND status IN (${placeholders}) LIMIT 1`)
.get(projectId, trigger, ...Array.from(ACTIVE_STATUSES)) as { id?: string } | undefined;
return Boolean(row?.id);
}
private persistRun(run: EvalRun): void {
this.db.prepare(`
UPDATE eval_runs SET
status = ?, scope = ?, window = ?, requestedTaskIds = ?, evaluatedTaskIds = ?, counts = ?, aggregateScores = ?,
summary = ?, error = ?, provenance = ?, metadata = ?, updatedAt = ?, startedAt = ?, completedAt = ?, cancelledAt = ?
WHERE id = ?
`).run(
run.status,
run.scope,
toJson(run.window),
toJson(run.requestedTaskIds),
toJson(run.evaluatedTaskIds),
toJson(run.counts),
toJsonNullable(run.aggregateScores),
run.summary ?? null,
run.error ?? null,
toJsonNullable(run.provenance),
toJsonNullable(run.metadata),
run.updatedAt,
run.startedAt ?? null,
run.completedAt ?? null,
run.cancelledAt ?? null,
run.id,
);
this.db.bumpLastModified();
}
private rowToRun(row: Record<string, unknown>): EvalRun {
return {
id: String(row.id),
projectId: String(row.projectId),
status: row.status as EvalRunStatus,
trigger: row.trigger as EvalRun["trigger"],
scope: String(row.scope),
window: fromJson(row.window as string) ?? {},
requestedTaskIds: fromJson<string[]>(row.requestedTaskIds as string) ?? [],
evaluatedTaskIds: fromJson<string[]>(row.evaluatedTaskIds as string) ?? [],
counts: fromJson(row.counts as string) ?? { totalTasks: 0, scoredTasks: 0, skippedTasks: 0, erroredTasks: 0 },
aggregateScores: fromJson<Record<string, number>>(row.aggregateScores as string),
summary: (row.summary as string | null) ?? undefined,
error: (row.error as string | null) ?? undefined,
provenance: fromJson(row.provenance as string),
metadata: fromJson(row.metadata as string),
createdAt: String(row.createdAt),
updatedAt: String(row.updatedAt),
startedAt: (row.startedAt as string | null) ?? undefined,
completedAt: (row.completedAt as string | null) ?? undefined,
cancelledAt: (row.cancelledAt as string | null) ?? undefined,
};
}
private rowToResult(row: Record<string, unknown>): EvalTaskResult {
return {
id: String(row.id),
runId: String(row.runId),
taskId: String(row.taskId),
taskSnapshot: fromJson(row.taskSnapshot as string) ?? { taskId: String(row.taskId) },
status: row.status as EvalTaskResult["status"],
overallScore: row.overallScore == null ? undefined : Number(row.overallScore),
maxScore: row.maxScore == null ? undefined : Number(row.maxScore),
categoryScores: fromJson(row.categoryScores as string) ?? [],
rationale: (row.rationale as string | null) ?? undefined,
summary: (row.summary as string | null) ?? undefined,
evidence: fromJson(row.evidence as string) ?? [],
deterministicSignals: fromJson(row.deterministicSignals as string) ?? [],
aiSignals: fromJson(row.aiSignals as string),
followUps: fromJson(row.followUps as string) ?? [],
provenance: fromJson(row.provenance as string),
metadata: fromJson(row.metadata as string),
createdAt: String(row.createdAt),
updatedAt: String(row.updatedAt),
};
}
private rowToEvent(row: Record<string, unknown>): EvalRunEvent {
return {
id: String(row.id),
runId: String(row.runId),
seq: Number(row.seq),
type: row.type as EvalRunEvent["type"],
message: String(row.message),
status: (row.status as EvalRunStatus | null) ?? undefined,
taskId: (row.taskId as string | null) ?? undefined,
metadata: fromJson(row.metadata as string),
createdAt: String(row.createdAt),
};
}
}

View File

@@ -0,0 +1,239 @@
/**
* Eval Domain Types
*
* Contracts for eval run persistence and per-task evaluation results.
*/
export const EVAL_RUN_STATUSES = [
"pending",
"running",
"completed",
"failed",
"cancelled",
] as const;
export type EvalRunStatus = typeof EVAL_RUN_STATUSES[number];
export const EVAL_RUN_TRIGGERS = ["manual", "schedule", "api", "task_completion"] as const;
export type EvalRunTrigger = typeof EVAL_RUN_TRIGGERS[number];
export const EVAL_SCORE_CATEGORIES = [
"correctness",
"completeness",
"quality",
"reliability",
"tests",
"documentation",
] as const;
export type EvalScoreCategory = typeof EVAL_SCORE_CATEGORIES[number];
export interface EvalTaskSnapshot {
taskId: string;
title?: string;
column?: string;
status?: string;
priority?: string;
size?: string;
reviewLevel?: number;
createdAt?: string;
updatedAt?: string;
executionCompletedAt?: string;
summary?: string;
labels?: string[];
metadata?: Record<string, unknown>;
}
export interface EvalRunWindow {
since?: string;
until?: string;
baselineRunId?: string;
}
export interface EvalProvenance {
evaluatorProvider?: string;
evaluatorModelId?: string;
evaluatorVersion?: string;
promptVersion?: string;
runConfig?: Record<string, unknown>;
metadata?: Record<string, unknown>;
}
export interface EvalSignal {
signalId: string;
kind: string;
name: string;
passed?: boolean;
score?: number;
value?: number | string | boolean | null;
threshold?: number;
unit?: string;
summary?: string;
details?: Record<string, unknown>;
}
export interface EvalEvidenceReference {
type: "task_log" | "task_document" | "file" | "command" | "test" | "other";
ref: string;
excerpt?: string;
metadata?: Record<string, unknown>;
}
export interface EvalCategoryScore {
category: EvalScoreCategory | string;
score: number;
maxScore?: number;
rationale?: string;
}
export interface EvalFollowUpSuggestion {
title: string;
description: string;
priority?: "low" | "normal" | "high" | "urgent";
tags?: string[];
metadata?: Record<string, unknown>;
}
export interface EvalTaskResult {
id: string;
runId: string;
taskId: string;
taskSnapshot: EvalTaskSnapshot;
status: "scored" | "skipped" | "error";
overallScore?: number;
maxScore?: number;
categoryScores: EvalCategoryScore[];
rationale?: string;
summary?: string;
evidence: EvalEvidenceReference[];
deterministicSignals: EvalSignal[];
aiSignals?: EvalSignal[];
followUps: EvalFollowUpSuggestion[];
provenance?: EvalProvenance;
metadata?: Record<string, unknown>;
createdAt: string;
updatedAt: string;
}
export interface EvalRunCounts {
totalTasks: number;
scoredTasks: number;
skippedTasks: number;
erroredTasks: number;
}
export interface EvalRun {
id: string;
projectId: string;
status: EvalRunStatus;
trigger: EvalRunTrigger;
scope: string;
window: EvalRunWindow;
requestedTaskIds: string[];
evaluatedTaskIds: string[];
counts: EvalRunCounts;
aggregateScores?: Record<string, number>;
summary?: string;
error?: string;
provenance?: EvalProvenance;
metadata?: Record<string, unknown>;
createdAt: string;
updatedAt: string;
startedAt?: string;
completedAt?: string;
cancelledAt?: string;
}
export interface EvalRunEvent {
id: string;
runId: string;
seq: number;
type: "status_changed" | "task_evaluated" | "info" | "warning" | "error";
message: string;
status?: EvalRunStatus;
taskId?: string;
metadata?: Record<string, unknown>;
createdAt: string;
}
export interface EvalRunCreateInput {
projectId: string;
trigger?: EvalRunTrigger;
scope: string;
window?: EvalRunWindow;
requestedTaskIds?: string[];
provenance?: EvalProvenance;
metadata?: Record<string, unknown>;
}
export interface EvalRunUpdateInput {
status?: EvalRunStatus;
evaluatedTaskIds?: string[];
counts?: EvalRunCounts;
aggregateScores?: Record<string, number>;
summary?: string;
error?: string | null;
provenance?: EvalProvenance;
metadata?: Record<string, unknown>;
startedAt?: string | null;
completedAt?: string | null;
cancelledAt?: string | null;
}
export interface EvalRunListOptions {
projectId?: string;
status?: EvalRunStatus;
trigger?: EvalRunTrigger;
limit?: number;
offset?: number;
}
export interface EvalTaskResultCreateInput {
taskId: string;
taskSnapshot: EvalTaskSnapshot;
status: "scored" | "skipped" | "error";
overallScore?: number;
maxScore?: number;
categoryScores?: EvalCategoryScore[];
rationale?: string;
summary?: string;
evidence?: EvalEvidenceReference[];
deterministicSignals?: EvalSignal[];
aiSignals?: EvalSignal[];
followUps?: EvalFollowUpSuggestion[];
provenance?: EvalProvenance;
metadata?: Record<string, unknown>;
}
export interface EvalTaskResultUpdateInput {
status?: "scored" | "skipped" | "error";
overallScore?: number;
maxScore?: number;
categoryScores?: EvalCategoryScore[];
rationale?: string;
summary?: string;
evidence?: EvalEvidenceReference[];
deterministicSignals?: EvalSignal[];
aiSignals?: EvalSignal[];
followUps?: EvalFollowUpSuggestion[];
provenance?: EvalProvenance;
metadata?: Record<string, unknown>;
}
export interface EvalTaskResultListOptions {
runId?: string;
taskId?: string;
status?: "scored" | "skipped" | "error";
limit?: number;
offset?: number;
}
export interface EvalStoreEvents {
"run:created": [EvalRun];
"run:updated": [EvalRun];
"run:deleted": [string];
"run:event": [{ runId: string; event: EvalRunEvent }];
"result:created": [EvalTaskResult];
"result:updated": [EvalTaskResult];
}

View File

@@ -695,6 +695,30 @@ export type { ResolvedResearchSettings } from "./research-settings.js";
export { TodoStore } from "./todo-store.js";
export type { TodoStoreEvents } from "./todo-store.js";
export { EvalLifecycleError, EvalStore } from "./eval-store.js";
export type {
EvalRun,
EvalRunStatus,
EvalRunTrigger,
EvalRunWindow,
EvalRunCounts,
EvalRunEvent,
EvalRunCreateInput,
EvalRunUpdateInput,
EvalRunListOptions,
EvalTaskSnapshot,
EvalTaskResult,
EvalTaskResultCreateInput,
EvalTaskResultUpdateInput,
EvalTaskResultListOptions,
EvalCategoryScore,
EvalEvidenceReference,
EvalSignal,
EvalFollowUpSuggestion,
EvalProvenance,
EvalStoreEvents,
} from "./eval-types.js";
export { EVAL_RUN_STATUSES, EVAL_RUN_TRIGGERS, EVAL_SCORE_CATEGORIES } from "./eval-types.js";
// ── Agent Companies Types ──────────────────────────────────

View File

@@ -16,6 +16,7 @@ import { RoadmapStore } from "./roadmap-store.js";
import { InsightStore } from "./insight-store.js";
import { ResearchStore } from "./research-store.js";
import { TodoStore } from "./todo-store.js";
import { EvalStore } from "./eval-store.js";
import { BackwardCompat, ProjectRequiredError } from "./migration.js";
import { CentralCore } from "./central-core.js";
import { getTaskMergeBlocker } from "./task-merge.js";
@@ -512,6 +513,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
private researchStore: ResearchStore | null = null;
/** Cached TodoStore instance */
private todoStore: TodoStore | null = null;
/** Cached EvalStore instance */
private evalStore: EvalStore | null = null;
/** Buffer for batching agent log writes to reduce WAL pressure. */
private agentLogBuffer: Array<{
@@ -6592,6 +6595,17 @@ ${notificationsSection}`;
return this.todoStore;
}
/**
* Get the EvalStore instance for eval run and task result operations.
* Lazily initializes the EvalStore on first access.
*/
getEvalStore(): EvalStore {
if (!this.evalStore) {
this.evalStore = new EvalStore(this.db);
}
return this.evalStore;
}
// ── Verification Cache ────────────────────────────────────────────────────
/**

View File

@@ -1231,7 +1231,8 @@ export function __setCreateFnAgent(mock: typeof createFnAgent): void {
// hit the real engine. Mirror the same fake into the resolved-session slot
// so existing test setups that only call `__setCreateFnAgent` continue to
// work.
createResolvedAgentSession = (async (options: unknown) => mock(options)) as typeof createResolvedAgentSession;
createResolvedAgentSession = (async (options: Parameters<typeof createResolvedAgentSession>[0]) =>
mock(options)) as typeof createResolvedAgentSession;
}
/**