From fe910fcce7c875c596ed56a44f8a27c7c6eba866 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sun, 16 Aug 2026 03:49:28 -0700 Subject: [PATCH] FN-9122: align merge gate timing and policy baselines Re-establish a trustworthy W33 merge-gate timing baseline without weakening blocking coverage. - Document the controlled W33 re-measurement and future regression protocol. - Align static-validator test ledgers with all 15 canonical gate checks. - Correct gate composition, engine-core inventory, and bundle metrics in testing guidance. Files changed: .../merge-gate-w33-walltime-regression.md | 82 ++++++++++++++++++++++ docs/testing.md | 12 ++-- .../__tests__/engine-vitest-gate-policy.test.mjs | 12 ++++ scripts/__tests__/run-static-gate-checks.test.mjs | 2 + scripts/__tests__/verify-fast.test.mjs | 8 +++ 5 files changed, 111 insertions(+), 5 deletions(-) Fusion-Task-Id: FN-9122 Fusion-Task-Lineage: 47ace0d7-902d-4ea5-848d-3d2386867c42 Co-authored-by: Fusion (runfusion.ai) --- .../merge-gate-w33-walltime-regression.md | 82 +++++++++++++++++++ docs/testing.md | 12 +-- .../engine-vitest-gate-policy.test.mjs | 12 +++ .../__tests__/run-static-gate-checks.test.mjs | 2 + scripts/__tests__/verify-fast.test.mjs | 8 ++ 5 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 docs/solutions/developer-experience/merge-gate-w33-walltime-regression.md diff --git a/docs/solutions/developer-experience/merge-gate-w33-walltime-regression.md b/docs/solutions/developer-experience/merge-gate-w33-walltime-regression.md new file mode 100644 index 0000000000..4ea22b53af --- /dev/null +++ b/docs/solutions/developer-experience/merge-gate-w33-walltime-regression.md @@ -0,0 +1,82 @@ +--- +title: "Merge-gate W33 wall-time regression: controlled re-measure" +date: 2026-08-16 +category: developer-experience +module: scripts/run-static-gate-checks.mjs +problem_type: performance_investigation +applies_when: "A weekly velocity row reports a merge-gate wall-time spike and the team needs to distinguish a gate regression from a one-sample anomaly without weakening blocking coverage." +symptoms: + - "A single `pnpm test:gate` sample exceeds the 9–11 second healthy band." + - "Boot smoke and changed-only measurements do not show a matching slowdown." +root_cause: single_sample_measurement_variance +resolution_type: documented_measurement_protocol +severity: low +related_components: + - package.json + - scripts/run-static-gate-checks.mjs + - scripts/__tests__/engine-vitest-gate-policy.test.mjs + - packages/engine/vitest.config.ts + - packages/core/src/__test-utils__/pg-test-harness.ts + - packages/core/src/index.gate.ts +tags: [merge-gate, performance, test-velocity, vitest, postgres, fn-9122] +--- + +# Merge-gate W33 wall-time regression: controlled re-measure + +## Problem + +The W33 velocity row captured at `2026-08-16T06:01:44.671Z` reported `pnpm test:gate` at **14.0s**, versus **11.0s** in the preceding row. Boot smoke improved by 6.3s and changed-only improved by 116ms, so the one gate sample warranted investigation. The gate's composition documentation was also stale: it claimed 22 engine-core files and 12/14 validators while the actual gate had 21 files and 15 validators. + +## Controlled measurement + +FN-9122 used a built task worktree on Darwin 25.1.0 arm64 (28 CPUs, 256 GiB RAM), Node v26.3.0, and pnpm 10.33.0. Process checks found no concurrent Vitest, build, boot-smoke, or `fn serve` process. A priming run was discarded, then samples were serialized. + +| Run | Pre-sample load (1/5/15m) | Gate wall time | +|---|---|---:| +| Warm 1 | 7.76 / 6.89 / 6.07 | 9.5s | +| Warm 2 | 7.75 / 6.92 / 6.10 | 9.3s | +| Warm 3 | 8.27 / 7.07 / 6.16 | 9.9s | +| Cold (removed transform cache and core gate bundle) | 8.59 / 7.20 / 6.22 | 10.9s | +| Post-ledger warm 1 | 5.32 / 5.81 / 5.90 | 10.1s | +| Post-ledger warm 2 | 7.19 / 6.21 / 6.04 | 9.5s | +| Post-ledger warm 3 | 7.53 / 6.33 / 6.08 | 9.0s | + +The initial warm min/median/max was **9.3s / 9.5s / 9.9s**; the post-repair warm min/median/max was **9.0s / 9.5s / 10.1s**. Both medians are below 11.0s. W33 is therefore a **single-sample measurement anomaly**, not a reproduced gate regression. + +## Coverage-parity baseline + +The gate actually runs 15 static validators, 21 explicit engine-core files containing **432 tests**, two PG canaries, four unit-gate files, and CI-shape after successful concurrent lanes. `project-engine.test.ts` was removed from engine-core by FN-8937 (`0fbeba50d1`), so this is one fewer engine-core file than the W32 baseline. The policy contract now pins the 15-validator ordered inventory and its cardinality, and the corresponding static-runner and verify-fast test inventories match it. + +## Attribution + +| Phase | Warm timing | +|---|---:| +| Static validator fan-out | 2.0s | +| Engine-core | 5.3s | +| PG gate | 3.5s | +| Unit gate | 5.5s | +| CI-shape | 0.7s | +| Direct core gate-bundle rebuild | 0.2s | + +The concurrent test critical path was the unchanged unit-gate lane, not engine-core or PostgreSQL. The gate bundle had 411 metafile inputs and was 4,188,513 bytes; its rebuild did not explain a three-second change. Individual static checks showed `check-inert-sync-lane-conversions` at 1.7s, `check-capacity-pool-id` at 1.4s, and `check-no-getdatabase` at 1.3s; the newly admitted `check-cli-runtime-routing.mjs` (FN-9096, `1d3f6c198c`) took 0.1s and remained inside the 2.0s concurrent fan-out. + +The PG harness rewrite (`0935e27902`) was not the critical lane. The historical gate-safe barrel mirrors were also not implicated: engine-core remained 5.1–5.3s and the builder itself took 0.2s. + +## Decision + +Do not change a runtime gate seam on a single weekly measurement. For a future merge-gate spike: + +1. Run a passing `pnpm build` preflight in the task worktree. +2. Record host facts, relevant-process absence, and pre-sample load. +3. Discard one priming run, then collect at least three serialized warm `pnpm test:gate` samples. +4. Treat the regression as reproduced only when the warm median exceeds **11.0s**. +5. Measure static, engine-core, PG, unit, and CI-shape lanes before changing code. Preserve 15/21/2/4/CI-shape coverage parity and never buy time by weakening signal. + +## Precedents + +- **FN-7666** (`845fad2aa4`) established that organic full-barrel growth can inflate every engine-core fork; the gate-safe barrel remains the evidence-backed protection. +- **FN-7667** retained the exclusion policy without narrowing to hand-picked symbols; this investigation kept that policy because the barrel was not the W33 seam. +- **FN-7669** supplied the rebuild-every-run core bundle; **FN-7670** and **FN-7673** closed the engine relative-import bundling lever after no benefit and a measured regression. +- **FN-8497** (`9002fca9de`) kept only high-value PG canaries and made test lanes concurrent; **`c15c78fee`** created the PostgreSQL gate lane that was explicitly measured here. +- **FN-8783/FN-8789** established the warm-cache healthy band and transform-cache policy used for comparison. +- **FN-8937** (`0fbeba50d1`) removed, rather than admitted, an engine-core file in the measurement window; **FN-9096** (`1d3f6c198c`) added the inexpensive static validator. diff --git a/docs/testing.md b/docs/testing.md index b32c6fda77..883aed03f0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -6,25 +6,27 @@ This guide consolidates the detailed testing guidance moved from `AGENTS.md`. ## The merge gate -CI blocks PRs on exactly four checks (`.github/workflows/pr-checks.yml`): **Lint, Typecheck, Build, Gate**. The Gate job runs the boot smoke (`scripts/boot-smoke.mjs`: independent CLI `--help` and real `fn init` preflights run concurrently, the latter proving a durable `.fusion/project.json` marker, then a real `fn serve` answers `GET /api/health`, all against one isolated home) and `pnpm test:gate`: 12 static policy validators, 22 curated `engine-core` files, two PostgreSQL canaries, four core unit files, then the CI-shape test. +CI blocks PRs on exactly four checks (`.github/workflows/pr-checks.yml`): **Lint, Typecheck, Build, Gate**. The Gate job runs the boot smoke (`scripts/boot-smoke.mjs`: independent CLI `--help` and real `fn init` preflights run concurrently, the latter proving a durable `.fusion/project.json` marker, then a real `fn serve` answers `GET /api/health`, all against one isolated home) and `pnpm test:gate`: 15 static policy validators, 21 curated `engine-core` files, two PostgreSQL canaries, four core unit files, then the CI-shape test. Set `BOOT_SMOKE_TIMINGS=1` when invoking `pnpm smoke:boot` to print per-attempt help, init, health, and SIGTERM phase timings for diagnosis; the flag is off by default so normal gate output stays concise. Everything else — the 4-way shards, the engine slow tier, the dashboard inventory guard — runs NON-BLOCKING in `.github/workflows/full-suite.yml` on push to main. Gate membership is the explicit allow-list in `packages/engine/vitest.config.ts` (`engine-core` project). Admission requires evidence of value (the test catches real regressions); tests never graduate in by default. A flaky gate test is evicted by deleting its allow-list line — the eviction PR does not need the flaky test to pass. The whole `engine-core` project must stay under ~60s wall-clock. - -**Static-validator and lane ordering:** `test:gate:static` declares the 14 canonical, directly runnable read-only validators. `scripts/run-static-gate-checks.mjs` starts them concurrently and waits for **every** result, so zero, one, or multiple policy failures remain fail-closed and observable before tests start. It then starts `engine-core`, `test:pg-gate`, and `test:unit-gate` concurrently; the shell waits for all **three** and returns nonzero if any fail. CI-shape runs only after that successful wait. + +**Static-validator and lane ordering:** `test:gate:static` declares the 15 canonical, directly runnable read-only validators. `scripts/run-static-gate-checks.mjs` starts them concurrently and waits for **every** result, so zero, one, or multiple policy failures remain fail-closed and observable before tests start. It then starts `engine-core`, `test:pg-gate`, and `test:unit-gate` concurrently; the shell waits for all **three** and returns nonzero if any fail. CI-shape runs only after that successful wait. - + **FN-8783 warm result:** The paired W32 protocol recorded in task document `FN-8783/docs` measured the complete-gate median at **15.4s baseline** and **10.2s candidate** across five serialized AB/BA pairs on the same macOS arm64 host (Node 26.3.0, pnpm 10.33.0, identical lockfile). The final engine-core transform-cache profile used one priming run (6.3s), then five warm runs (**5.1, 5.2, 5.1, 5.0, 5.2s; median 5.1s**) versus the pre-cache 6.2s focused engine-core result. The residual full-gate critical path is the unchanged concurrent engine/PG/unit/CI-shape work; task evidence records commands, SHAs, preparation, raw timing order, and coverage counts. +**FN-9122 W33 re-measure:** the reported 14.0s weekly row did not reproduce. After a build preflight and priming pass, three serialized warm `pnpm test:gate` samples measured **9.5s, 9.3s, and 9.9s** (median **9.5s**); post-ledger-repair samples measured **10.1s, 9.5s, and 9.0s** (median **9.5s**). A cold-cache sample was 10.9s. Static fan-out was 2.0s, engine-core 5.3s, PG gate 3.5s, unit gate 5.5s, and CI-shape 0.7s, so no gate-only seam explained a three-second increase. Treat an isolated weekly spike as a diagnosis trigger: retain raw samples and only change the gate after at least three warm samples have median above 11.0s. See [the W33 gate investigation](solutions/developer-experience/merge-gate-w33-walltime-regression.md). + **PostgreSQL and unit gate policy:** `packages/core`'s `test:pg-gate` intentionally runs `handoff-to-review-atomicity.pg.test.ts` and `task-lifecycle-e2e.pg.test.ts`, preserving atomic-handoff and lifecycle real-backend canaries. `sync-workflow-ir-is-always-default.pg.test.ts` was evicted under the merge-gate flake rule; its coverage remains in the non-blocking core suite (see the [observed suite-only flakes register](solutions/test-failures/suite-only-flakes-observed-register.md#6-sync-workflow-ir-default-canary-setup-hook)). `test:unit-gate` runs `task-merge.test.ts`, `legacy-adoption.test.ts`, `no-hardcoded-lifecycle-columns.test.ts`, and `sync-workflow-ir-callsite-allowlist.test.ts`. Every other former PG gate member remains enabled and discovered by the non-blocking command `pnpm --filter @fusion/core test` (default config: `src/**/*.test.ts`, no PG quarantine exclusions). `scripts/__tests__/engine-vitest-gate-policy.test.mjs` pins the exact two PG and four unit files, all waits, CI-shape ordering, and every engine/static member. **Gate-safe `@fusion/core` barrel:** the `engine-core` project resolves `@fusion/core` to `packages/core/src/index.gate.ts` (a project-scoped `resolve.alias`, not the root map), not the full `packages/core/src/index.ts` barrel. `index.gate.ts` is a byte-for-byte copy of the full barrel minus the `export ... from` statements for modules added to the barrel after the last re-audit baseline — i.e. it re-exports everything the full barrel does except genuinely new, gate-irrelevant feature modules (diffed against the prior baseline commit's barrel, not hand-picked from what gate *test* files import — production modules under test pull in far more of the barrel transitively than their own imports suggest). `engine-default`/`engine-reliability`/`engine-slow` are unaffected and keep resolving the full barrel. `@fusion/engine` is untouched (no gate file imports it). When adding a new barrel module that no gate test needs, mirror the exclusion in `index.gate.ts` rather than letting gate wall-time grow — see the FNXC comment at the top of `index.gate.ts` and `packages/engine/vitest.config.ts`'s `engine-core` project for the audit procedure. -**Pre-bundled `@fusion/core` gate bundle:** FN-7668 profiled the gate's dominant wall-time cost as vitest/Vite SSR's **import-phase** — each fork worker independently re-resolves+evaluates the barrel closure from scratch with zero cross-fork sharing. `engine-core`'s `@fusion/core` alias now points at a single esbuild-bundled ESM file (`scripts/build-engine-core-gate-bundle.mjs`, entrypoint `packages/core/src/index.gate.ts`, `packages:"external"` so only the first-party closure — 220 files — is inlined) instead of directly at `index.gate.ts`'s source, collapsing 220 per-fork Vite SSR module-loader round-trips into 1 file load per fork. The bundle is **rebuilt fresh on every gate invocation** via the `engine-core` project's `globalSetup` (the builder's own esbuild dependency graph determines what gets bundled — never a hand-maintained file/symbol list, so there is no drift surface), and lives at `packages/core/.gate-bundle/core.mjs` — a gitignored, non-committed artifact placed as a **sibling of `packages/core/node_modules/`, deliberately not nested inside it**: nesting inside `node_modules` triggers Vite's SSR external-dep heuristic (loads the whole bundle via Node's native loader, bypassing Vite's mock-interception pipeline) and silently defeats `vi.mock` for imports nested inside the bundle (see the FNXC comment in the builder script for the full repro/fix). Measured A/B (5 alternating runs each, FN-7669 task docs): median real wall-time −5.5%, import-phase aggregate −14.0%, transform-phase aggregate −25.9%, with full coverage parity (335/335 gate tests, identical per-file counts) — a modest but real, reproducible, zero-downside win. `@fusion/engine` stays on the full (unbundled) barrel: no gate file imports it directly, so bundling it would be zero-benefit churn against the core↔engine circular-import DI. Bundling the `@fusion/engine` relative-import graph (`merger.ts` et al., the untouched remainder of FN-7668's ~430-file closure) is a natural, larger-payoff follow-up, filed separately. +**Pre-bundled `@fusion/core` gate bundle:** FN-7668 profiled the gate's dominant wall-time cost as vitest/Vite SSR's **import-phase** — each fork worker independently re-resolves+evaluates the barrel closure from scratch with zero cross-fork sharing. `engine-core`'s `@fusion/core` alias now points at a single esbuild-bundled ESM file (`scripts/build-engine-core-gate-bundle.mjs`, entrypoint `packages/core/src/index.gate.ts`, `packages:"external"` so only the first-party closure — 411 inputs in the FN-9122 re-measurement — is inlined) instead of directly at `index.gate.ts`'s source, collapsing those per-fork Vite SSR module-loader round-trips into 1 file load per fork. The bundle is **rebuilt fresh on every gate invocation** via the `engine-core` project's `globalSetup` (the builder's own esbuild dependency graph determines what gets bundled — never a hand-maintained file/symbol list, so there is no drift surface), and lives at `packages/core/.gate-bundle/core.mjs` — a gitignored, non-committed artifact placed as a **sibling of `packages/core/node_modules/`, deliberately not nested inside it**: nesting inside `node_modules` triggers Vite's SSR external-dep heuristic (loads the whole bundle via Node's native loader, bypassing Vite's mock-interception pipeline) and silently defeats `vi.mock` for imports nested inside the bundle (see the FNXC comment in the builder script for the full repro/fix). Measured A/B (5 alternating runs each, FN-7669 task docs): median real wall-time −5.5%, import-phase aggregate −14.0%, transform-phase aggregate −25.9%, with full coverage parity (335/335 gate tests, identical per-file counts) — a modest but real, reproducible, zero-downside win. `@fusion/engine` stays on the full (unbundled) barrel: no gate file imports it directly, so bundling it would be zero-benefit churn against the core↔engine circular-import DI. Bundling the `@fusion/engine` relative-import graph (`merger.ts` et al., the untouched remainder of FN-7668's ~430-file closure) is a natural, larger-payoff follow-up, filed separately. ## Weekly signal-per-second baseline diff --git a/scripts/__tests__/engine-vitest-gate-policy.test.mjs b/scripts/__tests__/engine-vitest-gate-policy.test.mjs index fc0db4059d..c792c96dc4 100644 --- a/scripts/__tests__/engine-vitest-gate-policy.test.mjs +++ b/scripts/__tests__/engine-vitest-gate-policy.test.mjs @@ -128,13 +128,25 @@ test("root and package gate scripts still propagate real Vitest failures", () => staticCheck("no-getdatabase"), staticCheck("prerebase-inert"), staticCheck("capacity-pool-id"), + staticCheck("cli-runtime-routing"), staticCheck("no-node-only-core-imports-in-dashboard"), staticCheck("pi-versions-pinned"), + staticCheck("workspace-package-graph"), staticCheck("no-test-timeout-appeasement"), staticCheck("changeset-format"), staticCheck("mock-completeness"), staticCheck("inert-sync-lane-conversions"), + staticCheck("runtime-skill-loader-drift"), ], "every static policy validator must remain once in the blocking composition"); + /* + FNXC:MergeGatePerformance 2026-08-16-10:29: + FN-9122's controlled W33 re-measurement closes the 14.0s row as variance, + but the correction is only meaningful when the composition is exact: 15 + concurrent static validators, 21 engine-core files, two PG canaries, and + four unit-gate files. Keep this cardinality alongside the ordered ledger so + a future declaration edit cannot silently invalidate the timing baseline. + */ + assert.equal(gateValidators.length, 15, "the W33 timing baseline requires all 15 static validators"); assert.equal(new Set(gateValidators).size, gateValidators.length, "the static validator composition must be duplicate-free"); assert.match(gate, /pnpm --filter @fusion\/engine test:core/); assert.match(gate, /pnpm --filter @fusion\/core test:pg-gate/); diff --git a/scripts/__tests__/run-static-gate-checks.test.mjs b/scripts/__tests__/run-static-gate-checks.test.mjs index b91be869c5..a765cdec6c 100644 --- a/scripts/__tests__/run-static-gate-checks.test.mjs +++ b/scripts/__tests__/run-static-gate-checks.test.mjs @@ -18,8 +18,10 @@ const EXPECTED_GATE_CHECKS = [ check("no-getdatabase"), check("prerebase-inert"), check("capacity-pool-id"), + check("cli-runtime-routing"), check("no-node-only-core-imports-in-dashboard"), check("pi-versions-pinned"), + check("workspace-package-graph"), check("no-test-timeout-appeasement"), check("changeset-format"), check("mock-completeness"), diff --git a/scripts/__tests__/verify-fast.test.mjs b/scripts/__tests__/verify-fast.test.mjs index 0be7753155..47ff140b83 100644 --- a/scripts/__tests__/verify-fast.test.mjs +++ b/scripts/__tests__/verify-fast.test.mjs @@ -60,8 +60,16 @@ const PRETEST_CHECKS = [ `PRETEST_STATIC_CHECK_SCRIPTS` already picked them up — verify:fast was RIGHT and this mirror was stale. Kept in production order, since the assertion is a deepEqual and order is part of it. */ "scripts/check-capacity-pool-id.mjs", + /* + FNXC:MergeGatePerformance 2026-08-16-10:29: + FN-9122 repaired the W33 gate inventory ledger after three validators had + entered canonical pretest/gate compositions without their test mirrors. + Keep these in manifest order so verify:fast validates the same policy set. + */ + "scripts/check-cli-runtime-routing.mjs", "scripts/check-no-node-only-core-imports-in-dashboard.mjs", "scripts/check-pi-versions-pinned.mjs", + "scripts/check-workspace-package-graph.mjs", "scripts/check-no-test-timeout-appeasement.mjs", "scripts/check-changeset-format.mjs", "scripts/check-routes-modular.mjs",