FN-9020: parallelize boot smoke preflights
Reduce durable boot-smoke latency while preserving its CLI, initialization, health, and shutdown assertions. - Run independent help and init preflights concurrently with bounded async child processes. - Add phase timing diagnostics and deterministic init failure classification. - Cover phase scheduling and isolated environment behavior, and document the diagnostics flag. Files changed: docs/testing.md | 4 +- scripts/__tests__/boot-smoke.test.mjs | 65 ++++++++++- scripts/boot-smoke.mjs | 201 +++++++++++++++++++++++++--------- 3 files changed, 214 insertions(+), 56 deletions(-) Fusion-Task-Id: FN-9020 Fusion-Task-Lineage: 1f5d3d7f-dc0c-4d16-a4ab-fd5e3952dbdf Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
gsxdsm
@@ -6,7 +6,9 @@ This guide consolidates the detailed testing guidance moved from `AGENTS.md`.
|
||||
|
||||
## The merge gate
|
||||
|
||||
CI blocks PRs on exactly four checks (`.github/workflows/pr-checks.yml`): **Lint, Typecheck, Build, Gate**. The Gate job runs the boot smoke (`scripts/boot-smoke.mjs`: CLI `--help`, real `fn init` with a durable `.fusion/project.json` marker, then a real `fn serve` answering `GET /api/health`, all against one isolated home) and `pnpm test:gate`: 11 static policy validators, 22 curated `engine-core` files, two PostgreSQL canaries, four core unit files, then the CI-shape test. Everything else — the 4-way shards, the engine slow tier, the dashboard inventory guard — runs NON-BLOCKING in `.github/workflows/full-suite.yml` on push to main.
|
||||
CI blocks PRs on exactly four checks (`.github/workflows/pr-checks.yml`): **Lint, Typecheck, Build, Gate**. The Gate job runs the boot smoke (`scripts/boot-smoke.mjs`: independent CLI `--help` and real `fn init` preflights run concurrently, the latter proving a durable `.fusion/project.json` marker, then a real `fn serve` answers `GET /api/health`, all against one isolated home) and `pnpm test:gate`: 11 static policy validators, 22 curated `engine-core` files, two PostgreSQL canaries, four core unit files, then the CI-shape test.
|
||||
|
||||
Set `BOOT_SMOKE_TIMINGS=1` when invoking `pnpm smoke:boot` to print per-attempt help, init, health, and SIGTERM phase timings for diagnosis; the flag is off by default so normal gate output stays concise. Everything else — the 4-way shards, the engine slow tier, the dashboard inventory guard — runs NON-BLOCKING in `.github/workflows/full-suite.yml` on push to main.
|
||||
|
||||
Gate membership is the explicit allow-list in `packages/engine/vitest.config.ts` (`engine-core` project). Admission requires evidence of value (the test catches real regressions); tests never graduate in by default. A flaky gate test is evicted by deleting its allow-list line — the eviction PR does not need the flaky test to pass. The whole `engine-core` project must stay under ~60s wall-clock.
|
||||
|
||||
|
||||
@@ -4,7 +4,13 @@
|
||||
// persistently-throwing remover) so a post-PASS cleanup can never fail the gate.
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { removeTempDir } from "../boot-smoke.mjs";
|
||||
import {
|
||||
classifyInitFailure,
|
||||
createBootSmokePhasePlan,
|
||||
createChildEnv,
|
||||
runPreflightPhasePlan,
|
||||
removeTempDir,
|
||||
} from "../boot-smoke.mjs";
|
||||
|
||||
function enotempty() {
|
||||
const err = new Error("ENOTEMPTY: directory not empty");
|
||||
@@ -68,6 +74,63 @@ test("removeTempDir uses the provided maxRetries/retryDelayMs overrides", () =>
|
||||
assert.equal(seenOpts.retryDelay, 250);
|
||||
});
|
||||
|
||||
test("boot plan overlaps independent preflight checks while retaining all smoke assertions", () => {
|
||||
for (const dataDir of ["cold", "warm"]) {
|
||||
const plan = createBootSmokePhasePlan({ attempt: 2, dataDir });
|
||||
assert.deepEqual(plan.map((phase) => phase.name), ["help", "init", "serve", "shutdown"]);
|
||||
assert.deepEqual(plan.map((phase) => phase.assertion), [
|
||||
"help-exits-0-and-mentions-serve",
|
||||
"init-settles-and-writes-project-marker",
|
||||
"health-200",
|
||||
"sigterm-delivered-and-clean-exit",
|
||||
]);
|
||||
assert.equal(plan.filter((phase) => phase.concurrency === "preflight").length, 2);
|
||||
assert.equal(plan.find((phase) => phase.name === "serve").attempt, 2);
|
||||
assert.deepEqual(plan.find((phase) => phase.name === "serve").dependsOn, ["init"]);
|
||||
assert.equal(plan.find((phase) => phase.name === "init").dataDir, dataDir);
|
||||
}
|
||||
});
|
||||
|
||||
test("production preflight scheduler launches independent phases before either settles", async () => {
|
||||
const plan = createBootSmokePhasePlan();
|
||||
const started = [];
|
||||
const results = await runPreflightPhasePlan(plan, async (phase) => {
|
||||
started.push(phase.name);
|
||||
await new Promise((resolve, reject) => {
|
||||
Promise.resolve().then(() => {
|
||||
if (started.length !== 2) {
|
||||
reject(new Error("preflight phase was awaited before every independent phase launched"));
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
return `${phase.name}-result`;
|
||||
});
|
||||
|
||||
assert.deepEqual(started, ["help", "init"]);
|
||||
assert.deepEqual(results, { help: "help-result", init: "init-result" });
|
||||
});
|
||||
|
||||
test("init liveness classification preserves every failing state", () => {
|
||||
const base = { error: null, status: 0, stdout: "", stderr: "" };
|
||||
assert.equal(classifyInitFailure(base, true), null);
|
||||
assert.equal(classifyInitFailure({ ...base, status: 13 }, true), "non-zero-exit");
|
||||
assert.equal(classifyInitFailure({ ...base, stderr: "Detected unsettled top-level await" }, true), "unsettled-top-level-await");
|
||||
assert.equal(classifyInitFailure(base, false), "missing-project-marker");
|
||||
assert.equal(classifyInitFailure({ ...base, error: new Error("spawn failed") }, true), "process-error");
|
||||
});
|
||||
|
||||
test("isolated child env strips inherited database and port controls", () => {
|
||||
const env = createChildEnv({ DATABASE_URL: "postgres://ambient", FUSION_NO_EMBEDDED_PG: "1", PORT: "4040", KEEP: "yes" }, "/tmp/fusion-home");
|
||||
assert.equal(env.HOME, "/tmp/fusion-home");
|
||||
assert.equal(env.FUSION_SKIP_ONBOARDING, "1");
|
||||
assert.equal(env.DATABASE_URL, undefined);
|
||||
assert.equal(env.FUSION_NO_EMBEDDED_PG, undefined);
|
||||
assert.equal(env.PORT, undefined);
|
||||
assert.equal(env.KEEP, "yes");
|
||||
});
|
||||
|
||||
test("importing boot-smoke.mjs does not boot a server (main() guard holds)", async () => {
|
||||
// The module was already imported at the top of this file for `removeTempDir`.
|
||||
// If the main()/bootAndVerify() top-level invocation ran unguarded, this test
|
||||
|
||||
@@ -28,10 +28,11 @@
|
||||
* captured child stderr on stdout for CI logs.
|
||||
*/
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { spawn } from "node:child_process";
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { createServer } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
@@ -57,6 +58,61 @@ const SHUTDOWN_TIMEOUT_MS = 15_000;
|
||||
// Ephemeral-port TOCTOU: retry the whole boot with a fresh port when the
|
||||
// child loses the bind race (EADDRINUSE).
|
||||
const BOOT_ATTEMPTS = 3;
|
||||
const TIMINGS_ENABLED = process.env.BOOT_SMOKE_TIMINGS === "1";
|
||||
|
||||
/*
|
||||
* FNXC:BootSmoke 2026-08-13-02:23:
|
||||
* FN-9020 measured the post-W32 boot-smoke regression phase by phase. `fn init`
|
||||
* and `fn --help` are independent CLI processes, so serializing them charged both
|
||||
* imports to every gate run without adding proof. Start them together, wait for
|
||||
* both original assertions, then boot serve only after init has written its marker.
|
||||
* The optional timings flag keeps normal merge-gate output unchanged while leaving
|
||||
* future regressions attributable without a wall-clock test threshold.
|
||||
*/
|
||||
export function createBootSmokePhasePlan({ attempt = 1, dataDir = "cold" } = {}) {
|
||||
return [
|
||||
{ name: "help", assertion: "help-exits-0-and-mentions-serve", concurrency: "preflight" },
|
||||
{ name: "init", assertion: "init-settles-and-writes-project-marker", concurrency: "preflight", dataDir },
|
||||
{ name: "serve", assertion: "health-200", dependsOn: ["init"], attempt },
|
||||
{ name: "shutdown", assertion: "sigterm-delivered-and-clean-exit", dependsOn: ["serve"] },
|
||||
];
|
||||
}
|
||||
|
||||
/*
|
||||
* FNXC:BootSmoke 2026-08-13-02:32:
|
||||
* FN-9020 requires the phase plan to control production scheduling, not merely
|
||||
* describe it for tests. Dispatch every independent preflight before awaiting
|
||||
* either result; a serialized await would restore the avoidable CLI-import cost
|
||||
* while retaining all four smoke assertions.
|
||||
*/
|
||||
export async function runPreflightPhasePlan(plan, runPhase) {
|
||||
const preflight = plan.filter((phase) => phase.concurrency === "preflight");
|
||||
return Object.fromEntries(await Promise.all(
|
||||
preflight.map(async (phase) => [phase.name, await runPhase(phase)]),
|
||||
));
|
||||
}
|
||||
|
||||
/** Construct the isolated child environment without retaining ambient database or port state. */
|
||||
export function createChildEnv(baseEnv, isolatedHome) {
|
||||
return {
|
||||
...baseEnv,
|
||||
HOME: isolatedHome,
|
||||
FUSION_SKIP_ONBOARDING: "1",
|
||||
DATABASE_URL: undefined,
|
||||
FUSION_NO_EMBEDDED_PG: undefined,
|
||||
PORT: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** Return the deterministic init assertion failure, or null when its liveness proof holds. */
|
||||
export function classifyInitFailure(init, projectMarkerExists) {
|
||||
const output = `${init.stdout ?? ""}${init.stderr ?? ""}`;
|
||||
if (init.error) return "process-error";
|
||||
if (init.status !== 0) return "non-zero-exit";
|
||||
if (/Detected unsettled top-level await/.test(output)) return "unsettled-top-level-await";
|
||||
if (!projectMarkerExists) return "missing-project-marker";
|
||||
return null;
|
||||
}
|
||||
|
||||
function parsePortList(raw) {
|
||||
return String(raw ?? "")
|
||||
@@ -110,6 +166,23 @@ async function getEphemeralPort() {
|
||||
throw new Error("could not obtain a non-reserved ephemeral port");
|
||||
}
|
||||
|
||||
function createPhaseTimer() {
|
||||
const timings = {};
|
||||
return {
|
||||
async measure(name, run) {
|
||||
const startedAt = performance.now();
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
timings[name] = Math.round(performance.now() - startedAt);
|
||||
}
|
||||
},
|
||||
report(attempt) {
|
||||
if (TIMINGS_ENABLED) console.log(`boot-smoke: timings attempt ${attempt} ${JSON.stringify(timings)}`);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fail(message, stderr = "") {
|
||||
console.error(`boot-smoke: FAIL — ${message}`);
|
||||
if (stderr.trim()) {
|
||||
@@ -140,20 +213,7 @@ async function pollHealth(port, deadline) {
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// 1. CLI answers --help.
|
||||
const help = spawnSync(process.execPath, [cliBin, "--help"], {
|
||||
encoding: "utf8",
|
||||
timeout: 30_000,
|
||||
});
|
||||
if (help.status !== 0) {
|
||||
fail(`\`fn --help\` exited ${help.status ?? `signal ${help.signal}`}`, help.stderr ?? "");
|
||||
}
|
||||
if (!/serve/i.test(help.stdout ?? "")) {
|
||||
fail("`fn --help` output does not mention the serve command", help.stderr ?? "");
|
||||
}
|
||||
console.log("boot-smoke: `fn --help` OK");
|
||||
|
||||
// 2. Real server boot on an ephemeral port with isolated HOME/project state.
|
||||
// Real server boot on an ephemeral port with isolated HOME/project state.
|
||||
// The ephemeral-port probe is inherently TOCTOU (probe closes before the
|
||||
// server binds), so an EADDRINUSE loss on a busy machine retries with a
|
||||
// fresh port instead of failing the gate.
|
||||
@@ -183,53 +243,84 @@ async function main() {
|
||||
* Returns "retry-port" when the child lost the ephemeral-port race
|
||||
* (EADDRINUSE); calls fail() (which exits) on any real failure.
|
||||
*/
|
||||
/** Spawn a bounded CLI preflight command while collecting the output its assertion needs. */
|
||||
async function runCli(command, args, options) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(process.execPath, [cliBin, ...args], {
|
||||
...options,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let timeout;
|
||||
let settled = false;
|
||||
const settle = (result) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
resolve(result);
|
||||
};
|
||||
child.stdout.on("data", (data) => (stdout += data));
|
||||
child.stderr.on("data", (data) => (stderr += data));
|
||||
child.once("error", (error) => settle({ error, status: null, signal: null, stdout, stderr }));
|
||||
child.once("exit", (status, signal) => settle({ error: null, status, signal, stdout, stderr }));
|
||||
timeout = setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
settle({ error: new Error(`${command} timed out`), status: null, signal: "SIGKILL", stdout, stderr });
|
||||
}, options.timeout);
|
||||
});
|
||||
}
|
||||
|
||||
async function bootAndVerify(attempt, registerCleanup) {
|
||||
const timer = createPhaseTimer();
|
||||
const port = await getEphemeralPort();
|
||||
const isolatedHome = mkdtempSync(path.join(tmpdir(), "fusion-boot-smoke-home-"));
|
||||
const isolatedProject = mkdtempSync(path.join(tmpdir(), "fusion-boot-smoke-project-"));
|
||||
let stderrBuf = "";
|
||||
const childEnv = {
|
||||
...process.env,
|
||||
HOME: isolatedHome,
|
||||
FUSION_SKIP_ONBOARDING: "1",
|
||||
// FNXC:BackendFlip 2026-06-26-14:55:
|
||||
// Force the smoke to exercise the embedded PostgreSQL backend. Unset
|
||||
// DATABASE_URL so a developer's external DB connection never leaks in
|
||||
// (the smoke must prove the zero-config embedded path boots). Unset
|
||||
// FUSION_NO_EMBEDDED_PG so the smoke cannot be opted out by an
|
||||
// inherited env var — the embedded default is what the gate must prove.
|
||||
DATABASE_URL: undefined,
|
||||
FUSION_NO_EMBEDDED_PG: undefined,
|
||||
// Make sure nothing inherits a PORT that fights the explicit flag.
|
||||
PORT: undefined,
|
||||
};
|
||||
// FNXC:BackendFlip 2026-06-26-14:55:
|
||||
// Force the smoke to exercise the embedded PostgreSQL backend. Unset
|
||||
// DATABASE_URL so a developer's external DB connection never leaks in
|
||||
// (the smoke must prove the zero-config embedded path boots). Unset
|
||||
// FUSION_NO_EMBEDDED_PG so the smoke cannot be opted out by an
|
||||
// inherited env var — the embedded default is what the gate must prove.
|
||||
const childEnv = createChildEnv(process.env, isolatedHome);
|
||||
|
||||
registerCleanup(() => {
|
||||
removeTempDir(isolatedHome);
|
||||
removeTempDir(isolatedProject);
|
||||
});
|
||||
|
||||
/*
|
||||
* FNXC:CliAwaitLiveness 2026-08-11-09:30:
|
||||
* CI already uses Node 24; it missed FN-8954 because help-only smoke never
|
||||
* ran `fn init`. Share this attempt's isolated HOME with serve so cold initdb
|
||||
* is paid once while the check detects exit 13 before project registration.
|
||||
*/
|
||||
const init = spawnSync(process.execPath, [cliBin, "init", "--name", "boot-smoke", "--path", isolatedProject], {
|
||||
cwd: isolatedProject,
|
||||
env: childEnv,
|
||||
encoding: "utf8",
|
||||
timeout: HEALTH_TIMEOUT_MS,
|
||||
});
|
||||
const initOutput = `${init.stdout ?? ""}${init.stderr ?? ""}`;
|
||||
if (init.error || init.status !== 0 || /Detected unsettled top-level await/.test(initOutput)) {
|
||||
fail(
|
||||
`\`fn init\` exited ${init.status ?? `signal ${init.signal ?? "timeout"}`}`,
|
||||
`${init.error?.message ? `${init.error.message}\n` : ""}${initOutput}`,
|
||||
);
|
||||
// Both preflight commands must settle, but neither consumes the other's state.
|
||||
// Keep init's isolated HOME for serve; only their process startup overlaps.
|
||||
const preflightRuns = await runPreflightPhasePlan(
|
||||
createBootSmokePhasePlan({ attempt }),
|
||||
(phase) => {
|
||||
if (phase.name === "help") {
|
||||
return timer.measure(phase.name, () => runCli("fn --help", ["--help"], { timeout: 30_000 }));
|
||||
}
|
||||
return timer.measure(phase.name, () => runCli("fn init", ["init", "--name", "boot-smoke", "--path", isolatedProject], {
|
||||
cwd: isolatedProject,
|
||||
env: childEnv,
|
||||
timeout: HEALTH_TIMEOUT_MS,
|
||||
}));
|
||||
},
|
||||
);
|
||||
const { help, init } = preflightRuns;
|
||||
if (help.error || help.status !== 0) {
|
||||
fail(`\`fn --help\` exited ${help.status ?? `signal ${help.signal ?? "timeout"}`}`, `${help.error?.message ? `${help.error.message}\n` : ""}${help.stderr ?? ""}`);
|
||||
}
|
||||
if (!existsSync(path.join(isolatedProject, ".fusion", "project.json"))) {
|
||||
fail("`fn init` did not write .fusion/project.json", initOutput);
|
||||
if (!/serve/i.test(help.stdout ?? "")) {
|
||||
fail("`fn --help` output does not mention the serve command", help.stderr ?? "");
|
||||
}
|
||||
console.log("boot-smoke: `fn --help` OK");
|
||||
|
||||
const initOutput = `${init.stdout ?? ""}${init.stderr ?? ""}`;
|
||||
const initFailure = classifyInitFailure(init, existsSync(path.join(isolatedProject, ".fusion", "project.json")));
|
||||
if (initFailure) {
|
||||
const message = initFailure === "missing-project-marker"
|
||||
? "`fn init` did not write .fusion/project.json"
|
||||
: `\`fn init\` exited ${init.status ?? `signal ${init.signal ?? "timeout"}`}`;
|
||||
fail(message, `${init.error?.message ? `${init.error.message}\n` : ""}${initOutput}`);
|
||||
}
|
||||
console.log("boot-smoke: `fn init` OK");
|
||||
|
||||
@@ -272,18 +363,19 @@ async function bootAndVerify(attempt, registerCleanup) {
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.race([
|
||||
await timer.measure("serve-to-health-200", () => Promise.race([
|
||||
pollHealth(port, Date.now() + HEALTH_TIMEOUT_MS),
|
||||
exitedEarly.then(({ code, signal }) => {
|
||||
throw new Error(`server exited before becoming healthy (${code ?? `signal ${signal}`})`);
|
||||
}),
|
||||
]);
|
||||
]));
|
||||
} catch (err) {
|
||||
if (/EADDRINUSE/.test(stderrBuf) && attempt < BOOT_ATTEMPTS) {
|
||||
console.log(`boot-smoke: port :${port} lost to another process (EADDRINUSE), retrying with a fresh port (attempt ${attempt}/${BOOT_ATTEMPTS})`);
|
||||
await exitedEarly; // child is already dead or dying; wait so cleanup is race-free
|
||||
removeTempDir(isolatedHome);
|
||||
removeTempDir(isolatedProject);
|
||||
timer.report(attempt);
|
||||
return "retry-port";
|
||||
}
|
||||
fail(err.message, stderrBuf);
|
||||
@@ -300,12 +392,12 @@ async function bootAndVerify(attempt, registerCleanup) {
|
||||
} catch {
|
||||
// ESRCH: server already exited — sigtermSent stays false and fails below.
|
||||
}
|
||||
const { code, signal } = await Promise.race([
|
||||
const { code, signal } = await timer.measure("sigterm-to-exit", () => Promise.race([
|
||||
exitedEarly,
|
||||
new Promise((resolve) =>
|
||||
setTimeout(() => resolve({ code: null, signal: "timeout" }), SHUTDOWN_TIMEOUT_MS),
|
||||
),
|
||||
]);
|
||||
]));
|
||||
if (!sigtermSent) {
|
||||
fail(`server exited on its own after the health check (${code ?? `signal ${signal}`}) — SIGTERM shutdown could not be verified`, stderrBuf);
|
||||
}
|
||||
@@ -317,6 +409,7 @@ async function bootAndVerify(attempt, registerCleanup) {
|
||||
fail(`server exited uncleanly on SIGTERM (${code ?? `signal ${signal}`})`, stderrBuf);
|
||||
}
|
||||
console.log(`boot-smoke: clean shutdown (${code ?? signal})`);
|
||||
timer.report(attempt);
|
||||
return "ok";
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user