diff --git a/docs/testing.md b/docs/testing.md index 4f96776606..30df812b3f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -91,8 +91,10 @@ was SIGKILLed by heap pressure under workspace worker budgeting. The top-level `pretest` artifact bootstrap runs once before the orchestrator; lane subprocesses must not re-run `scripts/ensure-test-artifacts.mjs`. - -When `scripts/test-changed.mjs` runs affected-package `vitest --changed` scopes, `@fusion/engine` is split out from other scopable packages and gets a dedicated memory envelope: `NODE_OPTIONS=--max-old-space-size=6144` plus `FUSION_TEST_TOTAL_WORKERS=1`, `FUSION_TEST_CONCURRENCY=1`, and `VITEST_MAX_WORKERS=1`. This mirrors the dashboard heap-managed precedent while preserving the `runWithWatchdog` changed-class wall-clock budget. Re-measure with a wide changed selection (for example a dirty `packages/core/src/index.ts` boundary edit) before changing this envelope; the expected failure mode after the fix is a normal Vitest pass/fail verdict, not raw pnpm `SIGKILL`. + +When `scripts/test-changed.mjs` runs affected-package `vitest --changed` scopes, `@fusion/engine` and `@fusion/dashboard` are each split out from other scopable packages into their own dedicated memory-envelope run: `NODE_OPTIONS=--max-old-space-size=6144` plus `FUSION_TEST_TOTAL_WORKERS=1`, `FUSION_TEST_CONCURRENCY=1`, and `VITEST_MAX_WORKERS=1`. All other scopable packages remain in the shared non-envelope group, and packages without a Vitest config still fall back to their package `test` scripts. The envelopes preserve the `runWithWatchdog` changed-class wall-clock budget so the expected failure mode is a normal Vitest pass/fail or watchdog timeout, not raw pnpm `SIGKILL`. Re-measure with a wide changed selection (for example a dirty `packages/core/src/index.ts` boundary edit for engine, or an App/jsdom-affecting dashboard diff) before changing either envelope. Concurrency knobs: diff --git a/scripts/__tests__/test-changed.test.mjs b/scripts/__tests__/test-changed.test.mjs index 07fae0f07c..fe92d7da5a 100644 --- a/scripts/__tests__/test-changed.test.mjs +++ b/scripts/__tests__/test-changed.test.mjs @@ -34,7 +34,11 @@ import { buildForwardDependencyMap, collectTransitiveDependencies, computeOwnHash, + createDashboardScopedAffectedEnv, createEngineScopedAffectedEnv, + DASHBOARD_SCOPED_AFFECTED_HEAP_MB, + DASHBOARD_SCOPED_AFFECTED_PACKAGE, + DASHBOARD_SCOPED_AFFECTED_WORKERS, ENGINE_SCOPED_AFFECTED_HEAP_MB, ENGINE_SCOPED_AFFECTED_PACKAGE, ENGINE_SCOPED_AFFECTED_WORKERS, @@ -261,23 +265,101 @@ test("buildPackageDirByName: uses canonical workspace dirs instead of package al }); // --------------------------------------------------------------------------- -// engine scoped affected memory envelope +// scoped affected memory envelopes // --------------------------------------------------------------------------- -test("partitionScopedAffectedPackages: isolates engine from other scoped vitest lanes", () => { - const groups = partitionScopedAffectedPackages([ - "@fusion/core", - ENGINE_SCOPED_AFFECTED_PACKAGE, - "@fusion/dashboard", +function summarizeScopedAffectedGroups(packages) { + return partitionScopedAffectedPackages(packages).map((group) => ({ + packages: group.packages, + engineMemoryEnvelope: group.engineMemoryEnvelope, + memoryEnvelopePackage: group.memoryEnvelopePackage, + })); +} + +function assertScopedAffectedEnv(env, { heapMb, workers }) { + assert.match(env.NODE_OPTIONS, new RegExp(`--max-old-space-size=${heapMb}`)); + assert.match(env.NODE_OPTIONS, /--trace-warnings/); + assert.equal(env.FUSION_TEST_TOTAL_WORKERS, workers); + assert.equal(env.FUSION_TEST_CONCURRENCY, workers); + assert.equal(env.VITEST_MAX_WORKERS, workers); + assert.equal(env.HOME, "/tmp/fusion-home"); +} + +test("partitionScopedAffectedPackages: isolates dashboard and engine into separate envelope groups", () => { + assert.deepEqual(summarizeScopedAffectedGroups([DASHBOARD_SCOPED_AFFECTED_PACKAGE]), [ + { + packages: [DASHBOARD_SCOPED_AFFECTED_PACKAGE], + engineMemoryEnvelope: false, + memoryEnvelopePackage: DASHBOARD_SCOPED_AFFECTED_PACKAGE, + }, ]); - assert.deepEqual(groups, [ - { packages: ["@fusion/core", "@fusion/dashboard"], engineMemoryEnvelope: false }, - { packages: [ENGINE_SCOPED_AFFECTED_PACKAGE], engineMemoryEnvelope: true }, + assert.deepEqual(summarizeScopedAffectedGroups(["@fusion/core", DASHBOARD_SCOPED_AFFECTED_PACKAGE]), [ + { packages: ["@fusion/core"], engineMemoryEnvelope: false, memoryEnvelopePackage: null }, + { + packages: [DASHBOARD_SCOPED_AFFECTED_PACKAGE], + engineMemoryEnvelope: false, + memoryEnvelopePackage: DASHBOARD_SCOPED_AFFECTED_PACKAGE, + }, + ]); + + assert.deepEqual( + summarizeScopedAffectedGroups(["@fusion/core", DASHBOARD_SCOPED_AFFECTED_PACKAGE, ENGINE_SCOPED_AFFECTED_PACKAGE]), + [ + { packages: ["@fusion/core"], engineMemoryEnvelope: false, memoryEnvelopePackage: null }, + { + packages: [ENGINE_SCOPED_AFFECTED_PACKAGE], + engineMemoryEnvelope: true, + memoryEnvelopePackage: ENGINE_SCOPED_AFFECTED_PACKAGE, + }, + { + packages: [DASHBOARD_SCOPED_AFFECTED_PACKAGE], + engineMemoryEnvelope: false, + memoryEnvelopePackage: DASHBOARD_SCOPED_AFFECTED_PACKAGE, + }, + ], + ); + + assert.deepEqual(summarizeScopedAffectedGroups(["@fusion/core", "@runfusion/fusion"]), [ + { packages: ["@fusion/core", "@runfusion/fusion"], engineMemoryEnvelope: false, memoryEnvelopePackage: null }, ]); }); -test("createEngineScopedAffectedEnv: caps heap, lowers workers, and leaves watchdog budget finite", () => { +test("createDashboardScopedAffectedEnv: caps heap, preserves env, lowers workers, and leaves watchdog finite", () => { + const env = createDashboardScopedAffectedEnv({ + NODE_OPTIONS: "--trace-warnings", + FUSION_TEST_TOTAL_WORKERS: "8", + FUSION_TEST_CONCURRENCY: "4", + FUSION_TEST_WORKSPACE_CONCURRENCY: "1", + VITEST_MAX_WORKERS: "4", + HOME: "/tmp/fusion-home", + }); + + assertScopedAffectedEnv(env, { + heapMb: DASHBOARD_SCOPED_AFFECTED_HEAP_MB, + workers: DASHBOARD_SCOPED_AFFECTED_WORKERS, + }); + assert.equal(env.FUSION_TEST_WORKSPACE_CONCURRENCY, "1"); + + const lowConcurrencyEnv = createDashboardScopedAffectedEnv({ + NODE_OPTIONS: "--trace-warnings", + FUSION_TEST_TOTAL_WORKERS: "1", + FUSION_TEST_CONCURRENCY: "1", + FUSION_TEST_WORKSPACE_CONCURRENCY: "1", + VITEST_MAX_WORKERS: "1", + HOME: "/tmp/fusion-home", + }); + assertScopedAffectedEnv(lowConcurrencyEnv, { + heapMb: DASHBOARD_SCOPED_AFFECTED_HEAP_MB, + workers: DASHBOARD_SCOPED_AFFECTED_WORKERS, + }); + + const budgetMs = deriveBudgetMs({ klass: "changed" }); + assert.equal(Number.isFinite(budgetMs), true); + assert.equal(budgetMs > 0, true); +}); + +test("createEngineScopedAffectedEnv: preserves existing engine envelope contract", () => { const env = createEngineScopedAffectedEnv({ NODE_OPTIONS: "--trace-warnings", FUSION_TEST_TOTAL_WORKERS: "8", @@ -286,12 +368,10 @@ test("createEngineScopedAffectedEnv: caps heap, lowers workers, and leaves watch HOME: "/tmp/fusion-home", }); - assert.match(env.NODE_OPTIONS, new RegExp(`--max-old-space-size=${ENGINE_SCOPED_AFFECTED_HEAP_MB}`)); - assert.match(env.NODE_OPTIONS, /--trace-warnings/); - assert.equal(env.FUSION_TEST_TOTAL_WORKERS, ENGINE_SCOPED_AFFECTED_WORKERS); - assert.equal(env.FUSION_TEST_CONCURRENCY, ENGINE_SCOPED_AFFECTED_WORKERS); - assert.equal(env.VITEST_MAX_WORKERS, ENGINE_SCOPED_AFFECTED_WORKERS); - assert.equal(env.HOME, "/tmp/fusion-home"); + assertScopedAffectedEnv(env, { + heapMb: ENGINE_SCOPED_AFFECTED_HEAP_MB, + workers: ENGINE_SCOPED_AFFECTED_WORKERS, + }); const budgetMs = deriveBudgetMs({ klass: "changed" }); assert.equal(Number.isFinite(budgetMs), true); diff --git a/scripts/test-changed.mjs b/scripts/test-changed.mjs index fe9108afed..7edb162dfe 100644 --- a/scripts/test-changed.mjs +++ b/scripts/test-changed.mjs @@ -1314,31 +1314,71 @@ export function packageHasVitestConfig(pkgDir, projectRoot = rootDir) { export const ENGINE_SCOPED_AFFECTED_PACKAGE = "@fusion/engine"; export const ENGINE_SCOPED_AFFECTED_HEAP_MB = "6144"; export const ENGINE_SCOPED_AFFECTED_WORKERS = "1"; +export const DASHBOARD_SCOPED_AFFECTED_PACKAGE = "@fusion/dashboard"; +export const DASHBOARD_SCOPED_AFFECTED_HEAP_MB = "6144"; +export const DASHBOARD_SCOPED_AFFECTED_WORKERS = "1"; + +export const SCOPED_AFFECTED_MEMORY_ENVELOPES = Object.freeze({ + [ENGINE_SCOPED_AFFECTED_PACKAGE]: Object.freeze({ + packageName: ENGINE_SCOPED_AFFECTED_PACKAGE, + heapMb: ENGINE_SCOPED_AFFECTED_HEAP_MB, + workers: ENGINE_SCOPED_AFFECTED_WORKERS, + }), + [DASHBOARD_SCOPED_AFFECTED_PACKAGE]: Object.freeze({ + packageName: DASHBOARD_SCOPED_AFFECTED_PACKAGE, + heapMb: DASHBOARD_SCOPED_AFFECTED_HEAP_MB, + workers: DASHBOARD_SCOPED_AFFECTED_WORKERS, + }), +}); export function prependNodeOption(currentOptions, option) { return [option, currentOptions || ""].join(" ").trim(); } -export function createEngineScopedAffectedEnv(env = process.env) { +export function createScopedAffectedMemoryEnvelopeEnv(packageName, env = process.env) { + const envelope = SCOPED_AFFECTED_MEMORY_ENVELOPES[packageName]; + if (!envelope) return env; /* FNXC:TestInfrastructure 2026-06-21-11:24: The engine affected lane can select hundreds of real-git-heavy files when `vitest --changed` sees a widely imported boundary. Run that scoped lane in its own memory envelope: cap Node old-space like the dashboard heap runner and lower Vitest worker fan-out to one process so the lane returns a real pass/fail verdict instead of an OS OOM SIGKILL. Keep watchdog timing outside this env so hangs still fail through `runWithWatchdog`. + + FNXC:TestInfrastructure 2026-06-21-16:28: + FN-6874 showed the dashboard changed-mode affected lane can OOM/SIGKILL even with `FUSION_TEST_CONCURRENCY=1 FUSION_TEST_WORKSPACE_CONCURRENCY=1`, so worker fan-out alone is not the failure mode. Give each heavy scoped package its own bounded heap envelope while preserving caller env and keeping the finite changed-class watchdog outside this env so hangs still fail instead of being masked. */ return { ...env, - NODE_OPTIONS: prependNodeOption(env.NODE_OPTIONS, `--max-old-space-size=${ENGINE_SCOPED_AFFECTED_HEAP_MB}`), - FUSION_TEST_TOTAL_WORKERS: ENGINE_SCOPED_AFFECTED_WORKERS, - FUSION_TEST_CONCURRENCY: ENGINE_SCOPED_AFFECTED_WORKERS, - VITEST_MAX_WORKERS: ENGINE_SCOPED_AFFECTED_WORKERS, + NODE_OPTIONS: prependNodeOption(env.NODE_OPTIONS, `--max-old-space-size=${envelope.heapMb}`), + FUSION_TEST_TOTAL_WORKERS: envelope.workers, + FUSION_TEST_CONCURRENCY: envelope.workers, + VITEST_MAX_WORKERS: envelope.workers, }; } +export function createEngineScopedAffectedEnv(env = process.env) { + return createScopedAffectedMemoryEnvelopeEnv(ENGINE_SCOPED_AFFECTED_PACKAGE, env); +} + +export function createDashboardScopedAffectedEnv(env = process.env) { + return createScopedAffectedMemoryEnvelopeEnv(DASHBOARD_SCOPED_AFFECTED_PACKAGE, env); +} + export function partitionScopedAffectedPackages(packages) { - const regularPackages = packages.filter((pkg) => pkg !== ENGINE_SCOPED_AFFECTED_PACKAGE); + const memoryEnvelopePackages = Object.keys(SCOPED_AFFECTED_MEMORY_ENVELOPES); + const memoryEnvelopePackageSet = new Set(memoryEnvelopePackages); + const requestedPackageSet = new Set(packages); + const regularPackages = packages.filter((pkg) => !memoryEnvelopePackageSet.has(pkg)); const groups = []; - if (regularPackages.length > 0) groups.push({ packages: regularPackages, engineMemoryEnvelope: false }); - if (packages.includes(ENGINE_SCOPED_AFFECTED_PACKAGE)) { - groups.push({ packages: [ENGINE_SCOPED_AFFECTED_PACKAGE], engineMemoryEnvelope: true }); + if (regularPackages.length > 0) { + groups.push({ packages: regularPackages, engineMemoryEnvelope: false, memoryEnvelopePackage: null, memoryEnvelope: null }); + } + for (const packageName of memoryEnvelopePackages) { + if (!requestedPackageSet.has(packageName)) continue; + groups.push({ + packages: [packageName], + engineMemoryEnvelope: packageName === ENGINE_SCOPED_AFFECTED_PACKAGE, + memoryEnvelopePackage: packageName, + memoryEnvelope: SCOPED_AFFECTED_MEMORY_ENVELOPES[packageName], + }); } return groups; } @@ -1556,7 +1596,7 @@ export async function main(argv = process.argv.slice(2)) { : []; const fallbackPkgs = activePackages.filter((pkg) => !scopable.includes(pkg)); - for (const { packages, mode, engineMemoryEnvelope = false } of [ + for (const { packages, mode, memoryEnvelopePackage = null } of [ ...partitionScopedAffectedPackages(scopable).map((group) => ({ ...group, mode: "scoped" })), { packages: fallbackPkgs, mode: "full" }, ]) { @@ -1578,13 +1618,16 @@ export async function main(argv = process.argv.slice(2)) { ...forwardedArgs, ] : [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs]; + const memoryEnvelopeLabel = memoryEnvelopePackage ? ` (${memoryEnvelopePackage} memory envelope)` : ""; console.log( mode === "scoped" - ? `[test-changed] scoped (vitest --changed) run for: ${packages.join(", ")}${engineMemoryEnvelope ? " (engine memory envelope)" : ""}` + ? `[test-changed] scoped (vitest --changed) run for: ${packages.join(", ")}${memoryEnvelopeLabel}` : `[test-changed] full package-suite run for: ${packages.join(", ")} (no vitest config / no base)`, ); await runMaybeIsolated("pnpm", commandArgs, { - env: engineMemoryEnvelope ? createEngineScopedAffectedEnv(isolatedHomeEnv) : isolatedHomeEnv, + env: memoryEnvelopePackage + ? createScopedAffectedMemoryEnvelopeEnv(memoryEnvelopePackage, isolatedHomeEnv) + : isolatedHomeEnv, onBeforeAfterCheck: cleanupIsolatedHome, // Scoped runs are proportional to the diff, so the tight "changed" ceiling // applies; a hang fails fast instead of blocking the 60-min full backstop.