feat(FN-4219): complete Step 4 — add experiment executor runtime

Fusion-Task-Id: FN-4219
Fusion-Task-Lineage: 8d9a9ba6-6729-4376-b935-549e28a7fa35
This commit is contained in:
Fusion
2026-05-14 00:42:33 -07:00
committed by gsxdsm
parent a73cfc710b
commit 24bf18a233
5 changed files with 469 additions and 0 deletions

View File

@@ -158,6 +158,23 @@ describe("ExperimentSessionStore", () => {
expect(onRecord).not.toHaveBeenCalled();
});
it("updates run payload patch additively", () => {
const session = store.createSession({ name: "p", metric: { name: "x", direction: "maximize" } });
const run = store.appendRecord(session.id, {
type: "run",
payload: { primaryMetric: 9, secondaryMetrics: [], status: "keep" },
});
const updated = store.updateRecordPayload(run.id, { commit: "abc123" });
expect(updated.payload).toEqual({
primaryMetric: 9,
secondaryMetrics: [],
status: "keep",
commit: "abc123",
});
expect(store.getRecord(run.id)?.payload).toEqual(updated.payload);
});
it("recordKept is idempotent", () => {
const session = store.createSession({ name: "k", metric: { name: "x", direction: "maximize" } });
const run = store.appendRecord(session.id, {

View File

@@ -270,6 +270,28 @@ export class ExperimentSessionStore extends EventEmitter<ExperimentSessionStoreE
return this.updateSession(session.id, { bestRunId: runRecordId });
}
updateRecordPayload(recordId: string, patch: Partial<ExperimentSessionRecord["payload"]>): ExperimentSessionRecord {
const record = this.getRecord(recordId);
if (!record) throw new Error(`Experiment record not found: ${recordId}`);
const updated = {
...record,
payload: {
...record.payload,
...patch,
},
} as ExperimentSessionRecord;
this.db.prepare(`
UPDATE experiment_session_records
SET payload = ?
WHERE id = ?
`).run(toJson(updated.payload), recordId);
this.db.bumpLastModified();
return updated;
}
recordKept(sessionId: string, runRecordId: string): ExperimentSession {
const session = this.assertRunRecordOwnership(sessionId, runRecordId);
const keptRunIds = session.keptRunIds.includes(runRecordId)

View File

@@ -0,0 +1,146 @@
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createDatabase, ExperimentSessionStore, type Database } from "@fusion/core";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
ExperimentExecutor,
ExperimentGitNotConfiguredError,
ExperimentMaxIterationsError,
} from "../experiment-executor.js";
import type { GitOps } from "../experiment/git-ops.js";
function createGitMock(): GitOps {
return {
head: vi.fn(),
add: vi.fn(),
commit: vi.fn(),
resetHard: vi.fn(),
stashPush: vi.fn(),
stashPop: vi.fn(),
statusPorcelain: vi.fn(),
};
}
describe("ExperimentExecutor", () => {
let db: Database;
let store: ExperimentSessionStore;
beforeEach(() => {
const dir = mkdtempSync(join(tmpdir(), "fn-exec-"));
db = createDatabase(dir, { inMemory: true });
db.init();
store = new ExperimentSessionStore(db);
});
it("initExperiment creates session and config record", async () => {
const executor = new ExperimentExecutor({ store, runBenchmark: vi.fn() as never });
const { session, configRecord } = await executor.initExperiment({
name: "exp",
metric: { name: "accuracy", direction: "maximize" },
});
expect(session.status).toBe("active");
expect(configRecord.type).toBe("config");
});
it("initExperiment duplicate active starts new segment", async () => {
const executor = new ExperimentExecutor({ store, runBenchmark: vi.fn() as never });
const first = await executor.initExperiment({ name: "dup", metric: { name: "m", direction: "maximize" }, projectId: "p" });
const second = await executor.initExperiment({ name: "dup", metric: { name: "m", direction: "maximize" }, projectId: "p" });
expect(second.session.id).toBe(first.session.id);
expect(second.session.currentSegment).toBe(2);
expect(second.configRecord.segment).toBe(2);
});
it("runExperiment parses metric and returns pending", async () => {
const runBenchmark = vi.fn().mockResolvedValue({ exitCode: 0, stdout: "METRIC accuracy=0.9", stderr: "", durationMs: 12, truncated: false, timedOut: false });
const executor = new ExperimentExecutor({ store, runBenchmark });
const { session } = await executor.initExperiment({ name: "run", metric: { name: "accuracy", direction: "maximize" } });
const result = await executor.runExperiment({ sessionId: session.id, command: "x", cwd: process.cwd() });
expect(result.status).toBe("pending");
expect(result.primaryMetric?.value).toBe(0.9);
});
it("runExperiment enforces maxIterations", async () => {
const runBenchmark = vi.fn().mockResolvedValue({ exitCode: 0, stdout: "METRIC accuracy=0.9", stderr: "", durationMs: 12, truncated: false, timedOut: false });
const executor = new ExperimentExecutor({ store, runBenchmark });
const { session } = await executor.initExperiment({ name: "max", metric: { name: "accuracy", direction: "maximize" }, maxIterations: 1 });
await executor.logExperiment({ sessionId: session.id, runResult: { runHandle: "h", exitCode: 0, stdout: "", stderr: "", durationMs: 1, primaryMetric: { name: "accuracy", value: 1 }, secondaryMetrics: [], parseWarnings: [], status: "pending" }, outcome: "errored" });
await expect(executor.runExperiment({ sessionId: session.id, command: "x", cwd: process.cwd() })).rejects.toBeInstanceOf(ExperimentMaxIterationsError);
});
it("runExperiment non-zero exit is errored", async () => {
const runBenchmark = vi.fn().mockResolvedValue({ exitCode: 1, stdout: "", stderr: "bad", durationMs: 12, truncated: false, timedOut: false });
const executor = new ExperimentExecutor({ store, runBenchmark });
const { session } = await executor.initExperiment({ name: "err", metric: { name: "accuracy", direction: "maximize" } });
const result = await executor.runExperiment({ sessionId: session.id, command: "x", cwd: process.cwd() });
expect(result.status).toBe("errored");
});
it("logExperiment keep commits and marks best/kept", async () => {
const git = createGitMock();
vi.mocked(git.commit).mockResolvedValue("sha123");
const executor = new ExperimentExecutor({ store, git, runBenchmark: vi.fn() as never });
const { session } = await executor.initExperiment({ name: "keep", metric: { name: "accuracy", direction: "maximize" } });
const runResult = { runHandle: "h", exitCode: 0, stdout: "", stderr: "", durationMs: 1, primaryMetric: { name: "accuracy", value: 1 }, secondaryMetrics: [], parseWarnings: [], status: "pending" as const };
const logged = await executor.logExperiment({ sessionId: session.id, runResult, outcome: "keep" });
expect(logged.commit).toBe("sha123");
expect(store.getSession(session.id)?.bestRunId).toBe(logged.runRecord.id);
expect(store.getSession(session.id)?.keptRunIds).toContain(logged.runRecord.id);
});
it("logExperiment discard calls revert", async () => {
const git = createGitMock();
vi.mocked(git.statusPorcelain).mockResolvedValue("");
const executor = new ExperimentExecutor({ store, git, runBenchmark: vi.fn() as never });
const { session } = await executor.initExperiment({ name: "discard", metric: { name: "accuracy", direction: "maximize" } });
const runResult = { runHandle: "h", exitCode: 0, stdout: "", stderr: "", durationMs: 1, primaryMetric: { name: "accuracy", value: 1 }, secondaryMetrics: [], parseWarnings: [], status: "pending" as const };
await executor.logExperiment({ sessionId: session.id, runResult, outcome: "discard", baselineCommit: "base" });
expect(git.resetHard).toHaveBeenCalledWith("base");
});
it("logExperiment keep without git throws and does not append", async () => {
const executor = new ExperimentExecutor({ store, runBenchmark: vi.fn() as never });
const { session } = await executor.initExperiment({ name: "nogit", metric: { name: "accuracy", direction: "maximize" } });
const before = store.listRecords(session.id, { type: "run" }).length;
await expect(executor.logExperiment({ sessionId: session.id, runResult: { runHandle: "h", exitCode: 0, stdout: "", stderr: "", durationMs: 1, primaryMetric: { name: "accuracy", value: 1 }, secondaryMetrics: [], parseWarnings: [], status: "pending" }, outcome: "keep" })).rejects.toBeInstanceOf(ExperimentGitNotConfiguredError);
expect(store.listRecords(session.id, { type: "run" })).toHaveLength(before);
});
it("serializes runs when maxConcurrentExperiments is 1", async () => {
const starts: number[] = [];
const runBenchmark = vi.fn(async () => {
starts.push(Date.now());
await new Promise((resolve) => setTimeout(resolve, 60));
return { exitCode: 0, stdout: "METRIC accuracy=1", stderr: "", durationMs: 60, truncated: false, timedOut: false };
});
const executor = new ExperimentExecutor({ store, runBenchmark, maxConcurrentExperiments: 1 });
const { session } = await executor.initExperiment({ name: "serial", metric: { name: "accuracy", direction: "maximize" } });
await Promise.all([
executor.runExperiment({ sessionId: session.id, command: "x", cwd: process.cwd() }),
executor.runExperiment({ sessionId: session.id, command: "y", cwd: process.cwd() }),
]);
expect(starts).toHaveLength(2);
expect(starts[1] - starts[0]).toBeGreaterThanOrEqual(40);
});
it("cancel aborts in-flight run", async () => {
let capturedSignal: AbortSignal | undefined;
const runBenchmark = vi.fn(async (opts: { abortSignal?: AbortSignal }) => {
capturedSignal = opts.abortSignal;
await new Promise((resolve) => setTimeout(resolve, 100));
return { exitCode: capturedSignal?.aborted ? 1 : 0, stdout: "", stderr: "", durationMs: 100, truncated: false, timedOut: false };
});
const executor = new ExperimentExecutor({ store, runBenchmark });
const { session } = await executor.initExperiment({ name: "cancel", metric: { name: "accuracy", direction: "maximize" } });
const runPromise = executor.runExperiment({ sessionId: session.id, command: "x", cwd: process.cwd() });
await new Promise((resolve) => setTimeout(resolve, 10));
const handle = executor.getStatus(session.id).activeHandles[0];
expect(executor.cancel(handle)).toBe(true);
const result = await runPromise;
expect(result.status).toBe("errored");
});
});

View File

@@ -0,0 +1,271 @@
import { randomUUID } from "node:crypto";
import {
EXPERIMENT_RUN_OUTCOMES,
type ExperimentMetricDefinition,
type ExperimentRunOutcome,
type ExperimentRunRecordPayload,
type ExperimentSecondaryMetric,
type ExperimentSession,
type ExperimentSessionRecord,
type ExperimentSessionStore,
} from "@fusion/core";
import { AgentSemaphore } from "./concurrency.js";
import { runBenchmark as defaultRunBenchmark, type BenchmarkRunOptions } from "./experiment/benchmark-runner.js";
import { defaultGitOps, type GitOps } from "./experiment/git-ops.js";
import { commitKept, ExperimentRevertConflictError, revertDiscarded } from "./experiment/git-policy.js";
import { parseMetricLines } from "./experiment/metric-parser.js";
import { createLogger, formatError } from "./logger.js";
export class ExperimentMaxIterationsError extends Error {}
export class ExperimentGitNotConfiguredError extends Error {}
export interface ExperimentExecutorOptions {
store: ExperimentSessionStore;
git?: GitOps;
runBenchmark?: typeof defaultRunBenchmark;
maxConcurrentExperiments?: number;
logger?: ReturnType<typeof createLogger>;
}
export interface InitExperimentInput {
name: string;
metric: ExperimentMetricDefinition;
maxIterations?: number;
workingDir?: string;
rules?: string;
ideas?: string;
projectId?: string;
tags?: string[];
}
export interface RunExperimentInput {
sessionId: string;
command: string;
cwd: string;
timeoutMs?: number;
env?: NodeJS.ProcessEnv;
onProgress?: BenchmarkRunOptions["onProgress"];
}
export interface RunExperimentResult {
runHandle: string;
exitCode: number;
stdout: string;
stderr: string;
durationMs: number;
primaryMetric?: { name: string; value: number; unit?: string };
secondaryMetrics: ExperimentSecondaryMetric[];
parseWarnings: string[];
status: "pending" | "errored";
truncatedTempFile?: string;
}
export interface LogExperimentInput {
sessionId: string;
runResult: RunExperimentResult;
outcome: ExperimentRunOutcome;
description?: string;
asi?: Record<string, unknown>;
confidence?: number;
commitMessage?: string;
baselineCommit?: string;
}
export interface ExperimentExecutorStatus {
sessionId: string;
status: ExperimentSession["status"];
currentSegment: number;
runsInSegment: number;
activeHandles: string[];
maxIterations?: number;
}
export class ExperimentExecutor {
private readonly semaphore: AgentSemaphore;
private readonly activeRuns = new Map<string, { controller: AbortController; sessionId: string; startedAt: number }>();
private readonly runBenchmark;
private readonly logger;
constructor(private readonly options: ExperimentExecutorOptions) {
this.semaphore = new AgentSemaphore(options.maxConcurrentExperiments ?? 2);
this.runBenchmark = options.runBenchmark ?? defaultRunBenchmark;
this.logger = options.logger ?? createLogger("experiment-executor");
}
async initExperiment(input: InitExperimentInput): Promise<{ session: ExperimentSession; configRecord: ExperimentSessionRecord }> {
const configPayload = {
metric: input.metric,
maxIterations: input.maxIterations,
workingDir: input.workingDir,
rules: input.rules,
ideas: input.ideas,
};
const existing = this.options.store
.listSessions({ projectId: input.projectId })
.find((session) => session.name === input.name && ["active", "finalizing"].includes(session.status));
if (existing) {
const result = this.options.store.startNewSegment(existing.id, configPayload);
this.logger.log(`initExperiment: ${existing.id} mode=new-segment`);
return { session: result.session, configRecord: result.record };
}
const session = this.options.store.createSession({
name: input.name,
projectId: input.projectId,
metric: input.metric,
maxIterations: input.maxIterations,
workingDir: input.workingDir,
tags: input.tags,
status: "active",
currentSegment: 1,
});
const configRecord = this.options.store.appendRecord(session.id, {
type: "config",
payload: configPayload,
segment: session.currentSegment,
});
this.logger.log(`initExperiment: ${session.id} mode=created`);
return { session, configRecord };
}
async runExperiment(input: RunExperimentInput, opts?: { abortSignal?: AbortSignal }): Promise<RunExperimentResult> {
const session = this.options.store.getSession(input.sessionId);
if (!session || session.status !== "active") throw new Error("Session not active");
const runsInSegment = this.options.store
.listRecords(input.sessionId, { segment: session.currentSegment, type: "run" })
.length;
if (session.maxIterations !== undefined && runsInSegment >= session.maxIterations) {
throw new ExperimentMaxIterationsError(`Session ${input.sessionId} reached max iterations`);
}
await this.semaphore.acquire();
const controller = new AbortController();
const runHandle = randomUUID();
if (opts?.abortSignal) {
opts.abortSignal.addEventListener("abort", () => controller.abort(), { once: true });
}
this.activeRuns.set(runHandle, { controller, sessionId: input.sessionId, startedAt: Date.now() });
try {
const benchmark = await this.runBenchmark({
command: input.command,
cwd: input.cwd,
timeoutMs: input.timeoutMs,
env: input.env,
abortSignal: controller.signal,
onProgress: input.onProgress,
sessionId: input.sessionId,
});
const parsed = parseMetricLines(benchmark.stdout);
const status = benchmark.exitCode !== 0 || benchmark.timedOut || !parsed.primary ? "errored" : "pending";
return {
runHandle,
exitCode: benchmark.exitCode,
stdout: benchmark.stdout,
stderr: benchmark.stderr,
durationMs: benchmark.durationMs,
primaryMetric: parsed.primary,
secondaryMetrics: parsed.secondary,
parseWarnings: parsed.warnings,
status,
truncatedTempFile: benchmark.truncatedTempFile,
};
} catch (error) {
this.logger.error(`runExperiment failed: ${formatError(error)}`);
throw error;
} finally {
this.activeRuns.delete(runHandle);
this.semaphore.release();
}
}
async logExperiment(input: LogExperimentInput): Promise<{ runRecord: ExperimentSessionRecord; commit?: string; revertedTo?: string }> {
const session = this.options.store.getSession(input.sessionId);
if (!session) throw new Error(`Experiment session not found: ${input.sessionId}`);
if (!EXPERIMENT_RUN_OUTCOMES.includes(input.outcome)) throw new Error(`Invalid outcome: ${input.outcome}`);
if (input.outcome === "keep" && !input.runResult.primaryMetric) throw new Error("keep outcome requires primary metric");
if (["discard", "checks_failed"].includes(input.outcome) && !input.baselineCommit) {
throw new Error("baselineCommit is required for discard/checks_failed");
}
if (input.outcome === "keep" && !this.options.git) {
throw new ExperimentGitNotConfiguredError("Git ops not configured");
}
const payload: ExperimentRunRecordPayload = {
commit: undefined,
primaryMetric: input.runResult.primaryMetric?.value ?? Number.NaN,
secondaryMetrics: input.runResult.secondaryMetrics,
status: input.outcome,
description: input.description,
confidence: input.confidence,
asi: input.asi,
durationMs: input.runResult.durationMs,
};
const runRecord = this.options.store.appendRecord(input.sessionId, {
type: "run",
payload,
segment: session.currentSegment,
});
let commit: string | undefined;
let revertedTo: string | undefined;
if (input.outcome === "keep" && this.options.git) {
const result = await commitKept({
session,
runRecord,
runPayload: payload,
git: this.options.git,
commitMessage: input.commitMessage,
});
commit = result.commit;
this.options.store.updateRecordPayload(runRecord.id, { commit });
this.options.store.setBestRun(input.sessionId, runRecord.id);
this.options.store.recordKept(input.sessionId, runRecord.id);
}
if (["discard", "checks_failed", "errored"].includes(input.outcome) && input.baselineCommit && this.options.git) {
const result = await revertDiscarded({ session, git: this.options.git, baselineCommit: input.baselineCommit });
revertedTo = result.revertedTo;
}
return { runRecord, commit, revertedTo };
}
getStatus(sessionId: string): ExperimentExecutorStatus {
const session = this.options.store.getSession(sessionId);
if (!session) throw new Error(`Experiment session not found: ${sessionId}`);
const runsInSegment = this.options.store
.listRecords(sessionId, { segment: session.currentSegment, type: "run" })
.length;
const activeHandles = [...this.activeRuns.entries()]
.filter(([, value]) => value.sessionId === sessionId)
.map(([handle]) => handle);
return {
sessionId,
status: session.status,
currentSegment: session.currentSegment,
runsInSegment,
activeHandles,
maxIterations: session.maxIterations,
};
}
cancel(runHandle: string): boolean {
const active = this.activeRuns.get(runHandle);
if (!active) return false;
active.controller.abort();
return true;
}
}
export { ExperimentRevertConflictError, defaultGitOps };

View File

@@ -114,6 +114,19 @@ export { classifyTaskError, type ErrorClass, type TaskErrorClassification } from
export { isUsageLimitError, UsageLimitPauser } from "./usage-limit-detector.js";
export { withRateLimitRetry } from "./rate-limit-retry.js";
export { ResearchOrchestrator, type ResearchOrchestratorOptions, type ResearchOrchestratorStatus, type ResearchOrchestratorStartOptions } from "./research-orchestrator.js";
export {
ExperimentExecutor,
ExperimentMaxIterationsError,
ExperimentGitNotConfiguredError,
ExperimentRevertConflictError,
defaultGitOps,
type ExperimentExecutorOptions,
type ExperimentExecutorStatus,
type InitExperimentInput,
type RunExperimentInput,
type RunExperimentResult,
type LogExperimentInput,
} from "./experiment-executor.js";
export {
ResearchStepRunner,
ResearchStepTimeoutError,