feat(FN-3593): add test isolation CI enforcement, fix stuck-requeue race, a
This merge lands five FN-3593 commits establishing a test isolation contract with a new `scripts/check-test-isolation.mjs` guard that scans for accidental `beforeEach`/`afterEach`/`beforeAll`/`afterAll` in setup helpers, plus per-package `setup-test-isolation.ts` bootstraps that canonicalize the pat Fusion-Task-Id: FN-3593
This commit is contained in:
Fusion
84
scripts/__tests__/check-test-isolation.test.mjs
Normal file
84
scripts/__tests__/check-test-isolation.test.mjs
Normal file
@@ -0,0 +1,84 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
const scriptPath = path.resolve("scripts/check-test-isolation.mjs");
|
||||
|
||||
function withFixture(fn) {
|
||||
const cwd = mkdtempSync(path.join(tmpdir(), "check-isolation-cwd-"));
|
||||
const home = mkdtempSync(path.join(tmpdir(), "check-isolation-home-"));
|
||||
mkdirSync(path.join(cwd, ".fusion"), { recursive: true });
|
||||
mkdirSync(path.join(home, ".fusion"), { recursive: true });
|
||||
try {
|
||||
fn({ cwd, home });
|
||||
} finally {
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function runScript(args, options) {
|
||||
return spawnSync(process.execPath, [scriptPath, ...args], {
|
||||
cwd: options.cwd,
|
||||
env: { ...process.env, HOME: options.home, USERPROFILE: options.home },
|
||||
encoding: "utf8",
|
||||
});
|
||||
}
|
||||
|
||||
test("passes when baseline and current state match", () => {
|
||||
withFixture(({ cwd, home }) => {
|
||||
const before = runScript(["--before"], { cwd, home });
|
||||
assert.equal(before.status, 0);
|
||||
const after = runScript([], { cwd, home });
|
||||
assert.equal(after.status, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test("fails when a tracked temp leak appears after baseline", () => {
|
||||
withFixture(({ cwd, home }) => {
|
||||
const before = runScript(["--before"], { cwd, home });
|
||||
assert.equal(before.status, 0);
|
||||
mkdirSync(path.join(tmpdir(), "fusion-test-leak-check-script"), { recursive: true });
|
||||
const after = runScript([], { cwd, home });
|
||||
assert.equal(after.status, 1);
|
||||
assert.match(after.stderr, /leaked temp director/i);
|
||||
rmSync(path.join(tmpdir(), "fusion-test-leak-check-script"), { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
test("fails when protected repo .fusion data changes after baseline", () => {
|
||||
withFixture(({ cwd, home }) => {
|
||||
const before = runScript(["--before"], { cwd, home });
|
||||
assert.equal(before.status, 0);
|
||||
writeFileSync(path.join(cwd, ".fusion", "mutated.txt"), "x");
|
||||
const after = runScript([], { cwd, home });
|
||||
assert.equal(after.status, 1);
|
||||
assert.match(after.stderr, /protected live \.fusion data changed/i);
|
||||
});
|
||||
});
|
||||
|
||||
test("fails when protected HOME .fusion data changes after baseline", () => {
|
||||
withFixture(({ cwd, home }) => {
|
||||
const before = runScript(["--before"], { cwd, home });
|
||||
assert.equal(before.status, 0);
|
||||
writeFileSync(path.join(home, ".fusion", "home-mutated.txt"), "x");
|
||||
const after = runScript([], { cwd, home });
|
||||
assert.equal(after.status, 1);
|
||||
assert.match(after.stderr, /protected live \.fusion data changed/i);
|
||||
});
|
||||
});
|
||||
|
||||
test("fails when protected .fusion existence changes after baseline", () => {
|
||||
withFixture(({ cwd, home }) => {
|
||||
rmSync(path.join(cwd, ".fusion"), { recursive: true, force: true });
|
||||
const before = runScript(["--before"], { cwd, home });
|
||||
assert.equal(before.status, 0);
|
||||
mkdirSync(path.join(cwd, ".fusion"), { recursive: true });
|
||||
const after = runScript([], { cwd, home });
|
||||
assert.equal(after.status, 1);
|
||||
assert.match(after.stderr, /protected live \.fusion data changed/i);
|
||||
});
|
||||
});
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
applyCacheToPlan,
|
||||
recordCachePass,
|
||||
cacheFilePath,
|
||||
shouldRunIsolationGuard,
|
||||
} from "../test-changed.mjs";
|
||||
|
||||
import { mkdirSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
|
||||
@@ -91,6 +92,10 @@ test("shouldForceFullSuite: returns true when scripts/test-changed.mjs changed",
|
||||
assert.equal(shouldForceFullSuite(["scripts/test-changed.mjs"]), true);
|
||||
});
|
||||
|
||||
test("shouldForceFullSuite: returns true when scripts/check-test-isolation.mjs changed", () => {
|
||||
assert.equal(shouldForceFullSuite(["scripts/check-test-isolation.mjs"]), true);
|
||||
});
|
||||
|
||||
test("shouldForceFullSuite: returns true when a GitHub workflow changed", () => {
|
||||
assert.equal(shouldForceFullSuite([".github/workflows/ci.yml"]), true);
|
||||
});
|
||||
@@ -550,3 +555,11 @@ test("cacheFilePath: ends with .fusion/test-cache.json", () => {
|
||||
const p = cacheFilePath();
|
||||
assert.ok(p.endsWith(path.join(".fusion", "test-cache.json")), `got: ${p}`);
|
||||
});
|
||||
|
||||
test("shouldRunIsolationGuard: enabled by default", () => {
|
||||
assert.equal(shouldRunIsolationGuard({}), true);
|
||||
});
|
||||
|
||||
test("shouldRunIsolationGuard: disabled when env flag is set", () => {
|
||||
assert.equal(shouldRunIsolationGuard({ FUSION_TEST_DISABLE_ISOLATION_GUARD: "1" }), false);
|
||||
});
|
||||
|
||||
@@ -1,28 +1,10 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Verifies that the test suite doesn't leak temp directories or touch the
|
||||
* real .fusion directory.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/check-test-isolation.mjs [--before]
|
||||
*
|
||||
* --before Record baseline state before running tests (writes /tmp/.fusion-isolation-baseline).
|
||||
* (default) Compare current state to baseline and fail on leaks.
|
||||
*
|
||||
* Integration:
|
||||
* node scripts/check-test-isolation.mjs --before
|
||||
* pnpm test
|
||||
* node scripts/check-test-isolation.mjs
|
||||
*/
|
||||
|
||||
import { readdirSync, statSync, existsSync, writeFileSync, readFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { readdirSync, statSync, existsSync, writeFileSync, readFileSync, realpathSync } from "node:fs";
|
||||
import { homedir, tmpdir } from "node:os";
|
||||
import { join, resolve, sep } from "node:path";
|
||||
|
||||
const BASELINE_FILE = join(tmpdir(), ".fusion-isolation-baseline");
|
||||
|
||||
// Prefixes the test suite is allowed to create under /tmp. Any dir matching
|
||||
// one of these must be cleaned up by the end of the test run.
|
||||
const TRACKED_PREFIXES = [
|
||||
"fusion-worker-",
|
||||
"fusion-test-",
|
||||
@@ -40,6 +22,14 @@ const TRACKED_PREFIXES = [
|
||||
"kb-first-run-test-",
|
||||
];
|
||||
|
||||
function stablePath(pathValue) {
|
||||
try {
|
||||
return realpathSync(pathValue);
|
||||
} catch {
|
||||
return resolve(pathValue);
|
||||
}
|
||||
}
|
||||
|
||||
function snapshotTmp() {
|
||||
const entries = readdirSync(tmpdir());
|
||||
const matching = [];
|
||||
@@ -48,46 +38,106 @@ function snapshotTmp() {
|
||||
const full = join(tmpdir(), name);
|
||||
try {
|
||||
const stat = statSync(full);
|
||||
if (stat.isDirectory()) {
|
||||
matching.push({ name, mtime: stat.mtimeMs });
|
||||
}
|
||||
if (stat.isDirectory()) matching.push({ name, mtime: stat.mtimeMs });
|
||||
} catch {
|
||||
// Ignore — could be gone already.
|
||||
// Ignore transient file-system races while scanning /tmp.
|
||||
}
|
||||
}
|
||||
return matching;
|
||||
}
|
||||
|
||||
function listProtectedFusionDirs() {
|
||||
const dirs = new Set();
|
||||
dirs.add(stablePath(join(process.cwd(), ".fusion")));
|
||||
dirs.add(stablePath(join(process.env.HOME || process.env.USERPROFILE || homedir(), ".fusion")));
|
||||
return [...dirs];
|
||||
}
|
||||
|
||||
function collectFusionSignature(rootDir, out = []) {
|
||||
if (!existsSync(rootDir)) return out;
|
||||
let stat;
|
||||
try {
|
||||
stat = statSync(rootDir);
|
||||
} catch {
|
||||
return out;
|
||||
}
|
||||
if (!stat.isDirectory()) return out;
|
||||
|
||||
const entries = readdirSync(rootDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(rootDir, entry.name);
|
||||
const relPath = fullPath.slice(rootDir.length + (rootDir.endsWith(sep) ? 0 : 1));
|
||||
let entryStat;
|
||||
try {
|
||||
entryStat = statSync(fullPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
out.push(`${relPath}|${entry.isDirectory() ? "d" : "f"}|${entryStat.size}|${Math.floor(entryStat.mtimeMs)}`);
|
||||
if (entry.isDirectory()) collectFusionSignature(fullPath, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function snapshotProtectedFusion() {
|
||||
return listProtectedFusionDirs().map((dir) => ({
|
||||
dir,
|
||||
exists: existsSync(dir),
|
||||
entries: collectFusionSignature(dir).sort(),
|
||||
}));
|
||||
}
|
||||
|
||||
function recordBaseline() {
|
||||
const baseline = snapshotTmp();
|
||||
writeFileSync(BASELINE_FILE, JSON.stringify(baseline.map((e) => e.name)));
|
||||
console.log(`[test-isolation] Baseline recorded: ${baseline.length} existing dir(s) matched patterns.`);
|
||||
const payload = {
|
||||
tmpNames: snapshotTmp().map((e) => e.name),
|
||||
protectedFusion: snapshotProtectedFusion(),
|
||||
};
|
||||
writeFileSync(BASELINE_FILE, JSON.stringify(payload));
|
||||
console.log(`[test-isolation] Baseline recorded: ${payload.tmpNames.length} temp dir(s), ${payload.protectedFusion.length} protected .fusion root(s).`);
|
||||
}
|
||||
|
||||
function checkAgainstBaseline() {
|
||||
let baselineNames = new Set();
|
||||
let baseline = { tmpNames: [], protectedFusion: [] };
|
||||
if (existsSync(BASELINE_FILE)) {
|
||||
try {
|
||||
baselineNames = new Set(JSON.parse(readFileSync(BASELINE_FILE, "utf-8")));
|
||||
baseline = JSON.parse(readFileSync(BASELINE_FILE, "utf-8"));
|
||||
} catch {
|
||||
// Ignore malformed baseline.
|
||||
// Ignore malformed baseline payloads and treat as empty baseline.
|
||||
}
|
||||
}
|
||||
const current = snapshotTmp();
|
||||
const leaks = current.filter((e) => !baselineNames.has(e.name));
|
||||
if (leaks.length === 0) {
|
||||
console.log("[test-isolation] No leaked temp directories detected.");
|
||||
|
||||
const baselineNames = new Set(baseline.tmpNames ?? []);
|
||||
const leaks = snapshotTmp().filter((e) => !baselineNames.has(e.name));
|
||||
|
||||
const baselineByDir = new Map((baseline.protectedFusion ?? []).map((entry) => [entry.dir, entry]));
|
||||
const currentProtected = snapshotProtectedFusion();
|
||||
const protectedViolations = [];
|
||||
for (const current of currentProtected) {
|
||||
const base = baselineByDir.get(current.dir) ?? { exists: false, entries: [] };
|
||||
const changedExistence = Boolean(base.exists) !== Boolean(current.exists);
|
||||
const changedEntries = JSON.stringify(base.entries) !== JSON.stringify(current.entries);
|
||||
if (changedExistence || changedEntries) {
|
||||
protectedViolations.push(current.dir);
|
||||
}
|
||||
}
|
||||
|
||||
if (leaks.length === 0 && protectedViolations.length === 0) {
|
||||
console.log("[test-isolation] No temp leaks or live .fusion mutations detected.");
|
||||
process.exit(0);
|
||||
}
|
||||
console.error(`[test-isolation] FAIL: ${leaks.length} leaked temp director${leaks.length === 1 ? "y" : "ies"}:`);
|
||||
for (const leak of leaks) {
|
||||
console.error(` ${join(tmpdir(), leak.name)}`);
|
||||
|
||||
if (leaks.length > 0) {
|
||||
console.error(`[test-isolation] FAIL: ${leaks.length} leaked temp director${leaks.length === 1 ? "y" : "ies"}:`);
|
||||
for (const leak of leaks) console.error(` ${join(tmpdir(), leak.name)}`);
|
||||
console.error("");
|
||||
}
|
||||
console.error("");
|
||||
console.error("Tests must clean up their temp directories. Use helpers from");
|
||||
console.error(" packages/core/src/__test-utils__/workspace.ts (@fusion/test-utils)");
|
||||
console.error(" - tempWorkspace(prefix) — auto-cleaned in afterEach");
|
||||
console.error(" - useIsolatedCwd(prefix) — auto-cleaned + cwd restored");
|
||||
|
||||
if (protectedViolations.length > 0) {
|
||||
console.error("[test-isolation] FAIL: protected live .fusion data changed during tests:");
|
||||
for (const dir of protectedViolations) console.error(` ${dir}`);
|
||||
console.error("Tests must use temp HOME / temp workspaces and never write repo or user .fusion data.");
|
||||
}
|
||||
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,10 @@ import { fileURLToPath } from "node:url";
|
||||
import { createHash } from "node:crypto";
|
||||
import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs";
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
const scriptDir = path.dirname(currentFilePath);
|
||||
const checkIsolationScript = path.join(scriptDir, "check-test-isolation.mjs");
|
||||
|
||||
const rootDir = process.env.FUSION_PROJECT_DIR
|
||||
? path.resolve(process.env.FUSION_PROJECT_DIR)
|
||||
: process.cwd();
|
||||
@@ -32,6 +36,26 @@ function run(command, commandArgs, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
function runIsolationCheck(before = false) {
|
||||
const args = [checkIsolationScript];
|
||||
if (before) args.push("--before");
|
||||
run(process.execPath, args);
|
||||
}
|
||||
|
||||
export function shouldRunIsolationGuard(env = process.env) {
|
||||
return env.FUSION_TEST_DISABLE_ISOLATION_GUARD !== "1";
|
||||
}
|
||||
|
||||
function runMaybeIsolated(command, commandArgs, options = {}) {
|
||||
const enabled = shouldRunIsolationGuard();
|
||||
if (enabled) runIsolationCheck(true);
|
||||
try {
|
||||
run(command, commandArgs, options);
|
||||
} finally {
|
||||
if (enabled) runIsolationCheck(false);
|
||||
}
|
||||
}
|
||||
|
||||
function gitOutput(gitArgs) {
|
||||
const result = spawnSync("git", gitArgs, {
|
||||
cwd: rootDir,
|
||||
@@ -414,7 +438,7 @@ const fullSuiteEnv = {
|
||||
};
|
||||
|
||||
function runFullSuite(forwardedArgs) {
|
||||
run("pnpm", [`-r`, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], { env: fullSuiteEnv });
|
||||
runMaybeIsolated("pnpm", [`-r`, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], { env: fullSuiteEnv });
|
||||
}
|
||||
|
||||
export function decideExecutionPlan({
|
||||
@@ -495,6 +519,10 @@ export function main(argv = process.argv.slice(2)) {
|
||||
console.log(
|
||||
`[test-changed] all changed packages are cache-fresh (${cachedPackages.join(", ")}); nothing to run.`,
|
||||
);
|
||||
if (shouldRunIsolationGuard()) {
|
||||
runIsolationCheck(true);
|
||||
runIsolationCheck(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -504,13 +532,12 @@ export function main(argv = process.argv.slice(2)) {
|
||||
console.log(`[test-changed] skipping cached packages: ${cachedPackages.join(", ")}`);
|
||||
}
|
||||
|
||||
run("pnpm", [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], { env: fullSuiteEnv });
|
||||
runMaybeIsolated("pnpm", [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], { env: fullSuiteEnv });
|
||||
|
||||
// Tests passed — record in cache (never cache failures; process.exit on failure above).
|
||||
recordCachePass(activePackages, packageDirByName, { noCache });
|
||||
}
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === currentFilePath) {
|
||||
main();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user