From caf425eaea8167d1d9a0dbec487423bfc6ec847b Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Mon, 20 Jul 2026 18:16:58 -0700 Subject: [PATCH] test(core): cap pg-gate fork fan-out for DB-bound suite The test:pg-gate suite runs only *.pg.test.ts files, each building/copying a per-file schema-template database (heavy CREATE/DROP DATABASE DDL serialized by the single shared Postgres). Worker count derived from CPU cores over-scales on high-core machines (6 forks on a 28-core box), oversubscribing the one Postgres until every beforeAll exceeds the 15s hookTimeout (23/23 hook timeouts). CI's low-core runners stay near 2 forks and pass, so it only bites high-core locals. Add a maxCap clamp to computeMaxWorkers and a dedicated vitest.pg.config.ts (maxCap=4) for the pg-gate, right-sizing concurrency to the actual constraint (a single shared Postgres) rather than raising the timeout (forbidden appeasement). Low-core machines keep their smaller CPU-derived count via min(4, cpuCap). Verified: full test:pg-gate now passes 23 files / 126 tests on a 28-core host. --- packages/core/package.json | 2 +- .../core/src/__test-utils__/vitest-workers.ts | 23 ++++++++++- .../core/src/__tests__/vitest-workers.test.ts | 38 ++++++++++++++++++ packages/core/vitest.pg.config.ts | 40 +++++++++++++++++++ 4 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 packages/core/vitest.pg.config.ts diff --git a/packages/core/package.json b/packages/core/package.json index bdded60ccd..a014f272a0 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -43,7 +43,7 @@ "typecheck": "tsc --noEmit", "test": "vitest run --silent=passed-only --reporter=dot", "test:embedded-postgres": "vitest run src/__tests__/postgres/embedded-lifecycle.test.ts --silent=passed-only --reporter=dot", - "test:pg-gate": "vitest run src/__tests__/postgres/handoff-to-review-atomicity.pg.test.ts src/__tests__/postgres/store-list.pg.test.ts src/__tests__/postgres/task-lifecycle-e2e.pg.test.ts src/__tests__/postgres/soft-delete-resurrection-FN-5233.pg.test.ts src/__tests__/postgres/agent-logs-and-monitor.pg.test.ts src/__tests__/postgres/todo-store.pg.test.ts src/__tests__/postgres/workflow-definitions.pg.test.ts src/__tests__/postgres/message-store.pg.test.ts src/__tests__/postgres/insight-store.pg.test.ts src/__tests__/postgres/insight-run-execution.pg.test.ts src/__tests__/postgres/research-store.pg.test.ts src/__tests__/postgres/mission-store.pg.test.ts src/__tests__/postgres/goal-store.pg.test.ts src/__tests__/postgres/artifacts-documents-evals.pg.test.ts src/__tests__/postgres/command-center-analytics.pg.test.ts src/__tests__/postgres/command-center-remaining-analytics.pg.test.ts src/__tests__/postgres/research-execution.pg.test.ts src/__tests__/postgres/async-store-events.pg.test.ts src/__tests__/postgres/signal-ingestion.pg.test.ts src/__tests__/postgres/mission-autopilot.pg.test.ts src/__tests__/postgres/workflow-create.pg.test.ts src/__tests__/postgres/monitor-trait-storm-guard.pg.test.ts src/__tests__/postgres/agent-wake-getagent.pg.test.ts --silent=passed-only --reporter=dot" + "test:pg-gate": "vitest run --config vitest.pg.config.ts src/__tests__/postgres/handoff-to-review-atomicity.pg.test.ts src/__tests__/postgres/store-list.pg.test.ts src/__tests__/postgres/task-lifecycle-e2e.pg.test.ts src/__tests__/postgres/soft-delete-resurrection-FN-5233.pg.test.ts src/__tests__/postgres/agent-logs-and-monitor.pg.test.ts src/__tests__/postgres/todo-store.pg.test.ts src/__tests__/postgres/workflow-definitions.pg.test.ts src/__tests__/postgres/message-store.pg.test.ts src/__tests__/postgres/insight-store.pg.test.ts src/__tests__/postgres/insight-run-execution.pg.test.ts src/__tests__/postgres/research-store.pg.test.ts src/__tests__/postgres/mission-store.pg.test.ts src/__tests__/postgres/goal-store.pg.test.ts src/__tests__/postgres/artifacts-documents-evals.pg.test.ts src/__tests__/postgres/command-center-analytics.pg.test.ts src/__tests__/postgres/command-center-remaining-analytics.pg.test.ts src/__tests__/postgres/research-execution.pg.test.ts src/__tests__/postgres/async-store-events.pg.test.ts src/__tests__/postgres/signal-ingestion.pg.test.ts src/__tests__/postgres/mission-autopilot.pg.test.ts src/__tests__/postgres/workflow-create.pg.test.ts src/__tests__/postgres/monitor-trait-storm-guard.pg.test.ts src/__tests__/postgres/agent-wake-getagent.pg.test.ts --silent=passed-only --reporter=dot" }, "devDependencies": { "@earendil-works/pi-coding-agent": "0.80.10", diff --git a/packages/core/src/__test-utils__/vitest-workers.ts b/packages/core/src/__test-utils__/vitest-workers.ts index 61061931ab..8b2c631ecd 100644 --- a/packages/core/src/__test-utils__/vitest-workers.ts +++ b/packages/core/src/__test-utils__/vitest-workers.ts @@ -2,6 +2,22 @@ import { cpus } from "node:os"; interface ComputeMaxWorkersOptions { defaultCap?: number; + /* + * FNXC:PgTestWorkerCap 2026-07-18-18:00: + * Hard upper clamp applied to the FINAL resolved worker count, AFTER every + * other resolution path (explicit VITEST_MAX_WORKERS, workspace budget, CPU + * default) and the cpuCap clamp. Purpose: a suite bound by a single shared + * external resource (the pg-gate's one Postgres server, which serializes + * CREATE/DROP DATABASE DDL) must NOT scale its fork fan-out with CPU count. + * On a 28-core dev box the CPU-derived default is 6 forks; 6 concurrent forks + * each running the DDL-heavy per-file schema-template build oversubscribe the + * one Postgres until every beforeAll exceeds the 15s hookTimeout (the whole + * pg-gate then reports 23/23 hook timeouts). CI's low-core runners stay near 2 + * and pass, which is why this only bites high-core local machines. `maxCap` + * lets the pg vitest config pin a DB-safe ceiling while low-core machines keep + * using their smaller CPU-derived count. See vitest.pg.config.ts. + */ + maxCap?: number; } function computeDefaultCap(cpuCap: number): number { @@ -26,7 +42,7 @@ function parsePositiveInt(value: string | undefined): number | undefined { // All paths clamp to (cpus - 1) so we never oversubscribe. export function computeMaxWorkers(options: ComputeMaxWorkersOptions = {}): number { const cpuCap = Math.max(1, cpus().length - 1); - const { defaultCap = computeDefaultCap(cpuCap) } = options; + const { defaultCap = computeDefaultCap(cpuCap), maxCap } = options; const explicit = parsePositiveInt(process.env.VITEST_MAX_WORKERS); const totalBudget = parsePositiveInt(process.env.FUSION_TEST_TOTAL_WORKERS); @@ -49,6 +65,11 @@ export function computeMaxWorkers(options: ComputeMaxWorkersOptions = {}): numbe } workers = Math.min(workers, cpuCap); + // DB-bound suites clamp below the CPU-derived count so fork fan-out never + // oversubscribes a single shared external resource (see maxCap docs). + if (maxCap !== undefined && maxCap > 0) { + workers = Math.max(1, Math.min(workers, maxCap)); + } process.env.VITEST_MAX_WORKERS = String(workers); return workers; } diff --git a/packages/core/src/__tests__/vitest-workers.test.ts b/packages/core/src/__tests__/vitest-workers.test.ts index b5f106754a..e8c3ee8edb 100644 --- a/packages/core/src/__tests__/vitest-workers.test.ts +++ b/packages/core/src/__tests__/vitest-workers.test.ts @@ -54,6 +54,44 @@ describe("computeMaxWorkers", () => { expect(process.env.VITEST_MAX_WORKERS).toBe("2"); }); + // FNXC:PgTestWorkerCap 2026-07-18-18:00: maxCap must clamp the FINAL count for + // DB-bound suites (pg-gate) so fork fan-out never oversubscribes one Postgres, + // and it must win even over an explicit VITEST_MAX_WORKERS above the cap. + it("clamps the final worker count to maxCap for DB-bound suites", () => { + delete process.env.VITEST_MAX_WORKERS; + delete process.env.FUSION_TEST_TOTAL_WORKERS; + delete process.env.FUSION_TEST_CONCURRENCY; + + const cpuCap = Math.max(1, cpus().length - 1); + const workers = computeMaxWorkers({ defaultCap: 6, maxCap: 4 }); + + expect(workers).toBe(Math.min(4, cpuCap)); + expect(process.env.VITEST_MAX_WORKERS).toBe(String(workers)); + }); + + it("applies maxCap even when explicit VITEST_MAX_WORKERS exceeds it", () => { + process.env.VITEST_MAX_WORKERS = "12"; + delete process.env.FUSION_TEST_TOTAL_WORKERS; + delete process.env.FUSION_TEST_CONCURRENCY; + + const cpuCap = Math.max(1, cpus().length - 1); + const workers = computeMaxWorkers({ maxCap: 4 }); + + expect(workers).toBe(Math.min(4, cpuCap)); + }); + + it("does not raise workers below maxCap on low-core machines", () => { + delete process.env.VITEST_MAX_WORKERS; + delete process.env.FUSION_TEST_TOTAL_WORKERS; + delete process.env.FUSION_TEST_CONCURRENCY; + + // A defaultCap under the maxCap must be preserved (min semantics, not a floor). + const cpuCap = Math.max(1, cpus().length - 1); + const workers = computeMaxWorkers({ defaultCap: 2, maxCap: 4 }); + + expect(workers).toBe(Math.min(2, cpuCap)); + }); + it("uses a CPU-aware default cap when no overrides are provided", () => { delete process.env.VITEST_MAX_WORKERS; delete process.env.FUSION_TEST_TOTAL_WORKERS; diff --git a/packages/core/vitest.pg.config.ts b/packages/core/vitest.pg.config.ts new file mode 100644 index 0000000000..74b4f322a0 --- /dev/null +++ b/packages/core/vitest.pg.config.ts @@ -0,0 +1,40 @@ +import { defineConfig, mergeConfig } from "vitest/config"; + +import baseConfig from "./vitest.config"; +import { computeMaxWorkers } from "./src/__test-utils__/vitest-workers"; + +/* +FNXC:PgTestWorkerCap 2026-07-18-18:00: +Dedicated vitest config for the PostgreSQL gate suite (`test:pg-gate`). The +pg-gate runs ONLY *.pg.test.ts files, each of which builds and copies a +per-file schema-template database (heavy CREATE/DROP DATABASE DDL that the one +shared Postgres server serializes). The bottleneck is that single DB, not CPU, +so fork fan-out must not scale with core count. + +The base core config derives its worker count from CPUs: on a 28-core dev box +that is 6 forks. Six concurrent forks racing the DDL-heavy per-file setup +oversubscribe the one Postgres until every `beforeAll` exceeds the 15s +hookTimeout, and the whole gate reports 23/23 hook timeouts. CI's low-core +runners land near 2 forks and pass, so the failure only bites high-core local +machines. Measured on this 28-core box: 6 forks -> all time out; 4 forks -> 23 +files / 126 tests pass in ~42s; 2 forks -> pass in ~78s. + +`maxCap: 4` pins a DB-safe ceiling: high-core machines clamp to 4 forks while +low-core machines keep their smaller CPU-derived count (min(4, cpuCap)). This +is NOT timeout appeasement — the hookTimeout is unchanged; we right-size +concurrency to the actual constraint (a single shared Postgres) per the FN-5048 +rule against oversubscribing worker/concurrency knobs. Only the pg-gate uses +this config; the interleaved full core suite is unaffected because pg files are +a small fraction of it and rarely run 6-at-once. +*/ +const PG_MAX_WORKERS = 4; + +export default mergeConfig( + baseConfig, + defineConfig({ + test: { + maxWorkers: computeMaxWorkers({ maxCap: PG_MAX_WORKERS }), + minWorkers: 1, + }, + }), +);