feat(FN-4219): complete Step 2 — add benchmark runner

Fusion-Task-Id: FN-4219
Fusion-Task-Lineage: 8d9a9ba6-6729-4376-b935-549e28a7fa35

packages/engine/src/__tests__/experiment-benchmark-runner.test.ts

@@ -0,0 +1,103 @@
+import { rm, stat } from "node:fs/promises";
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { runBenchmark } from "../experiment/benchmark-runner.js";
+
+describe("runBenchmark", () => {
+  const tempFiles: string[] = [];
+
+  afterEach(async () => {
+    await Promise.all(
+      tempFiles.map(async (file) => {
+        try {
+          await rm(file, { force: true, recursive: true });
+        } catch {
+          // ignore cleanup failures
+        }
+      }),
+    );
+    tempFiles.length = 0;
+  });
+
+  it("runs command successfully", async () => {
+    const result = await runBenchmark({
+      command: `${process.execPath} -e \"console.log('METRIC accuracy=0.91')\"`,
+      cwd: process.cwd(),
+    });
+
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout).toContain("METRIC accuracy=0.91");
+    expect(result.durationMs).toBeGreaterThanOrEqual(0);
+    expect(result.timedOut).toBe(false);
+  });
+
+  it("times out long-running process", async () => {
+    const result = await runBenchmark({
+      command: `${process.execPath} -e \"setTimeout(() => console.log('done'), 2000)\"`,
+      cwd: process.cwd(),
+      timeoutMs: 100,
+    });
+
+    expect(result.timedOut).toBe(true);
+    expect(result.exitCode).not.toBe(0);
+  });
+
+  it("truncates oversized stdout and writes full output to temp file", async () => {
+    const result = await runBenchmark({
+      command: `${process.execPath} -e \"process.stdout.write('x'.repeat(2048))\"`,
+      cwd: process.cwd(),
+      maxBufferBytes: 256,
+      sessionId: "EXP-1",
+    });
+
+    expect(result.truncated).toBe(true);
+    expect(result.truncatedTempFile).toBeTruthy();
+    if (result.truncatedTempFile) {
+      tempFiles.push(result.truncatedTempFile);
+      const stats = await stat(result.truncatedTempFile);
+      expect(stats.size).toBeGreaterThan(256);
+    }
+    expect(result.stdout.length).toBeLessThanOrEqual(64 * 1024);
+  });
+
+  it("supports abort signal", async () => {
+    const controller = new AbortController();
+    const runPromise = runBenchmark({
+      command: `${process.execPath} -e \"setInterval(() => process.stdout.write('tick\\n'), 50)\"`,
+      cwd: process.cwd(),
+      abortSignal: controller.signal,
+    });
+
+    setTimeout(() => controller.abort(), 120);
+
+    const result = await runPromise;
+    expect(result.exitCode).not.toBe(0);
+    expect(result.timedOut).toBe(false);
+  });
+
+  it("throttles progress callbacks and stops after completion", async () => {
+    vi.useFakeTimers();
+    const onProgress = vi.fn();
+
+    const promise = runBenchmark({
+      command: `${process.execPath} -e \"let i=0; const t=setInterval(()=>{console.log(i++); if(i===5){clearInterval(t); process.exit(0);} }, 50)\"`,
+      cwd: process.cwd(),
+      onProgress,
+    });
+
+    await vi.advanceTimersByTimeAsync(1500);
+    const result = await promise;
+    const callsAtFinish = onProgress.mock.calls.length;
+    await vi.advanceTimersByTimeAsync(1000);
+
+    expect(result.exitCode).toBe(0);
+    expect(callsAtFinish).toBeGreaterThan(0);
+    expect(onProgress.mock.calls.length).toBe(callsAtFinish);
+    for (const call of onProgress.mock.calls) {
+      expect(call[0].elapsedMs).toBeGreaterThanOrEqual(0);
+    }
+
+    vi.useRealTimers();
+  });
+});

packages/engine/src/experiment/benchmark-runner.ts

@@ -0,0 +1,172 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+const DEFAULT_TIMEOUT_MS = 600_000;
+const DEFAULT_MAX_BUFFER_BYTES = 10 * 1024 * 1024;
+const STDOUT_TAIL_BYTES = 64 * 1024;
+
+export interface BenchmarkRunOptions {
+  command: string;
+  cwd: string;
+  timeoutMs?: number;
+  maxBufferBytes?: number;
+  env?: NodeJS.ProcessEnv;
+  abortSignal?: AbortSignal;
+  onProgress?: (partial: {
+    stdoutChunk?: string;
+    stderrChunk?: string;
+    elapsedMs: number;
+  }) => void;
+  sessionId?: string;
+}
+
+export interface BenchmarkRunResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+  durationMs: number;
+  truncated: boolean;
+  truncatedTempFile?: string;
+  timedOut: boolean;
+}
+
+export async function runBenchmark(
+  opts: BenchmarkRunOptions,
+): Promise<BenchmarkRunResult> {
+  const startedAt = Date.now();
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxBufferBytes = opts.maxBufferBytes ?? DEFAULT_MAX_BUFFER_BYTES;
+  const env = { ...process.env, ...opts.env };
+
+  return await new Promise<BenchmarkRunResult>((resolve, reject) => {
+    const child = spawn(opts.command, {
+      cwd: opts.cwd,
+      env,
+      shell: true,
+      signal: opts.abortSignal,
+    });
+
+    let stdoutFull = "";
+    let stdoutTail = "";
+    let stdoutAll = "";
+    let stderr = "";
+    let stdoutBytes = 0;
+    let truncated = false;
+    let timedOut = false;
+    let truncatedTempFile: string | undefined;
+    let progressStdoutChunk = "";
+    let progressStderrChunk = "";
+    let finished = false;
+
+    const emitProgress = () => {
+      if (!opts.onProgress) {
+        progressStdoutChunk = "";
+        progressStderrChunk = "";
+        return;
+      }
+      if (!progressStdoutChunk && !progressStderrChunk) {
+        return;
+      }
+      opts.onProgress({
+        stdoutChunk: progressStdoutChunk || undefined,
+        stderrChunk: progressStderrChunk || undefined,
+        elapsedMs: Date.now() - startedAt,
+      });
+      progressStdoutChunk = "";
+      progressStderrChunk = "";
+    };
+
+    const progressTimer = setInterval(emitProgress, 500);
+
+    const timeoutTimer = setTimeout(() => {
+      timedOut = true;
+      child.kill("SIGTERM");
+      setTimeout(() => child.kill("SIGKILL"), 2_000).unref();
+    }, timeoutMs);
+
+    const cleanup = () => {
+      clearInterval(progressTimer);
+      clearTimeout(timeoutTimer);
+    };
+
+    child.stdout?.on("data", (chunk: Buffer | string) => {
+      const text = String(chunk);
+      progressStdoutChunk += text;
+      stdoutBytes += Buffer.byteLength(text);
+
+      stdoutAll += text;
+
+      if (!truncated || stdoutBytes <= maxBufferBytes) {
+        stdoutFull += text;
+      } else {
+        stdoutTail = (stdoutTail + text).slice(-STDOUT_TAIL_BYTES);
+      }
+
+      if (stdoutBytes > maxBufferBytes) {
+        truncated = true;
+        if (!stdoutTail) {
+          stdoutTail = stdoutFull.slice(-STDOUT_TAIL_BYTES);
+        }
+      }
+    });
+
+    child.stderr?.on("data", (chunk: Buffer | string) => {
+      const text = String(chunk);
+      progressStderrChunk += text;
+      stderr += text;
+    });
+
+    child.on("error", (error) => {
+      cleanup();
+      if ((error as NodeJS.ErrnoException).name === "AbortError") {
+        finished = true;
+        resolve({
+          exitCode: 1,
+          stdout: truncated ? (stdoutFull + stdoutTail).slice(-STDOUT_TAIL_BYTES) : stdoutFull,
+          stderr,
+          durationMs: Date.now() - startedAt,
+          truncated,
+          truncatedTempFile,
+          timedOut: false,
+        });
+        return;
+      }
+      reject(error);
+    });
+
+    child.on("close", async (code, signal) => {
+      cleanup();
+      emitProgress();
+
+      if (truncated) {
+        try {
+          const tempDir = await mkdtemp(path.join(os.tmpdir(), "fn-experiment-"));
+          truncatedTempFile = path.join(
+            tempDir,
+            `fn-experiment-${opts.sessionId ?? "session"}-${Date.now()}.log`,
+          );
+          await writeFile(truncatedTempFile, stdoutAll, "utf8");
+        } catch {
+          truncatedTempFile = undefined;
+        }
+      }
+
+      const durationMs = Date.now() - startedAt;
+      const effectiveStdout = truncated
+        ? (stdoutFull + stdoutTail).slice(-STDOUT_TAIL_BYTES)
+        : stdoutFull;
+      finished = true;
+      resolve({
+        exitCode: code ?? (signal ? 1 : 0),
+        stdout: effectiveStdout,
+        stderr,
+        durationMs,
+        truncated,
+        truncatedTempFile,
+        timedOut,
+      });
+    });
+  });
+}