From 3916e062aa4ddf4036ef2fb9bebd885eba74746a Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Fri, 31 Jul 2026 15:01:21 -0700 Subject: [PATCH] fix(dashboard-quality): the lane runner could not be asked to run every lane (#3248) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The structural half of #2784. I re-measured its 123 failures on current main and **all four reported lanes are green** (143 / 2010 / 1961 / 5149 / 2103 passing). This fixes the reason nobody saw them. ## The mechanism `pnpm --filter @fusion/dashboard test` sets `stopScheduling = true` on the first failing lane, so the rest never run — and there was **no flag to ask for a full pass**. The report said: ``` [dashboard-quality] skipped 9 lane(s) after first failure ``` Nine lanes with **unknown** status and nine **passing** lanes produce the same absence of failure text. That is how 123 failures accumulated behind one red lane, and it is why the original issue could only be written by running all twelve lanes by hand. ## What changes, and what deliberately does not Fail-fast stays the **default** — fast feedback on a broken lane is right, and changing it would slow everyone for a rare case. - `--all` (alias `--no-fail-fast`) runs every lane and reports every failure. - `runQualityTests({ failFast })` so the behaviour is reachable from a test, not just the CLI. - The skip line now states the consequence and the remedy: lanes were **NOT RUN**, status **UNKNOWN rather than passing**, and `--all` shows the full set. ## Both halves pinned A flag nobody can prove works is the same as no flag: | test | asserts | |---|---| | DEFAULT stops after the first failing lane | `launched === ["one"]`, `skipped: 2` | | `failFast:false` runs all three | `launched === ["one","two","three"]`, `failed === [one, three]` | The second is the load-bearing one: **lane three ran even though lane one had already failed**, and both failures are reported rather than only the first. **Anti-vacuity control:** reverting the `if (failFast)` plumbing fails the second test and only it (`1 failed / 5 passed`); restoring passes `6/6`. ## Scope Runner and its tests only. No lane contents, no vitest configs, no CI workflow — CI already invokes lanes individually, so this changes local behaviour and the shared helper, not what CI runs. eslint clean; `check-fnxc-future-dates` exit 0. Suggest #2784 closes on the measured-green half and links here for the structural half, so the mechanism does not close along with the symptom that exposed it. ## Summary by CodeRabbit * **New Features** * Added an option to run all quality-test lanes, even when earlier lanes fail. * Added `--all` and `--no-fail-fast` command-line options. * Quality tests now stop on the first failure by default, with clearer output for skipped lanes. --- .../__tests__/run-quality-tests.test.ts | 45 +++++++++++++++++++ .../dashboard/scripts/run-quality-tests.mjs | 23 +++++++--- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/packages/dashboard/scripts/__tests__/run-quality-tests.test.ts b/packages/dashboard/scripts/__tests__/run-quality-tests.test.ts index ef34cc0114..dc354158a5 100644 --- a/packages/dashboard/scripts/__tests__/run-quality-tests.test.ts +++ b/packages/dashboard/scripts/__tests__/run-quality-tests.test.ts @@ -19,6 +19,7 @@ interface RunQualityTestsModule { qualityLanes: QualityLane[]; resolveConcurrency(env?: Record): number; runQualityTests(options?: { + failFast?: boolean; group?: "all" | "app" | "api"; concurrency?: number; lanes?: QualityLane[]; @@ -107,4 +108,48 @@ describe("dashboard quality orchestrator", () => { expect(result.completed).toBe(1); expect(result.skipped).toBe(0); }); + + /* + FNXC:DashboardQualityLanes 2026-07-31-18:10 (u12 — #2784's structural half): + Fail-fast hid 123 real failures behind one red lane: nine lanes were never run, and the report said + "skipped 9 lane(s)", which reads like a benign skip rather than "status unknown". These pin BOTH + halves — the default still stops (fast local feedback), and `--all` reaches every lane — because a + flag nobody can prove works is the same as no flag. + */ + it("DEFAULT stops scheduling after the first failing lane", async () => { + const { runQualityTests } = await loadModule(); + const launched: string[] = []; + + const result = await runQualityTests({ + lanes: [lane("one"), lane("two"), lane("three")], + concurrency: 1, + runner: async (qualityLane) => { + launched.push(qualityLane.name); + return { lane: qualityLane, ok: qualityLane.name !== "one" }; + }, + }); + + expect(launched).toEqual(["one"]); + expect(result).toMatchObject({ ok: false, skipped: 2 }); + }); + + it("failFast:false runs EVERY lane and reports every failure, not just the first", async () => { + const { runQualityTests } = await loadModule(); + const launched: string[] = []; + + const result = await runQualityTests({ + lanes: [lane("one"), lane("two"), lane("three")], + concurrency: 1, + failFast: false, + runner: async (qualityLane) => { + launched.push(qualityLane.name); + return { lane: qualityLane, ok: qualityLane.name === "two" }; + }, + }); + + // The half that matters: lane three ran even though lane one had already failed. + expect(launched).toEqual(["one", "two", "three"]); + expect(result).toMatchObject({ ok: false, completed: 3, skipped: 0 }); + expect(result.failed.map((f: { lane: { name: string } }) => f.lane.name)).toEqual(["one", "three"]); + }); }); diff --git a/packages/dashboard/scripts/run-quality-tests.mjs b/packages/dashboard/scripts/run-quality-tests.mjs index 6b4c15f445..0e9e335baa 100644 --- a/packages/dashboard/scripts/run-quality-tests.mjs +++ b/packages/dashboard/scripts/run-quality-tests.mjs @@ -82,6 +82,7 @@ export function resolveConcurrency(env = process.env) { function parseArgs(argv) { let group = "all"; let list = false; + let allLanes = false; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -90,6 +91,10 @@ function parseArgs(argv) { index += 1; continue; } + if (arg === "--all" || arg === "--no-fail-fast") { + allLanes = true; + continue; + } if (arg.startsWith("--group=")) { group = arg.slice("--group=".length); continue; @@ -105,7 +110,7 @@ function parseArgs(argv) { throw new Error(`Invalid --group value ${JSON.stringify(group)}; expected all, app, or api`); } - return { group, list }; + return { group, list, allLanes }; } function selectLanes(group) { @@ -152,6 +157,7 @@ export async function runQualityTests({ concurrency = resolveConcurrency(), lanes = selectLanes(group), runner = runLane, + failFast = true, } = {}) { const queue = [...lanes]; const failed = []; @@ -173,7 +179,11 @@ export async function runQualityTests({ completed += 1; if (!result.ok) { failed.push(result); - stopScheduling = true; + /* FNXC:DashboardQualityLanes 2026-07-31-18:05 (u12 — #2784's structural half): + Fail-fast stays the DEFAULT so a broken lane still gives fast local feedback, but it is + now switchable. Stopping hid 123 real failures behind one red lane (#2784): nine lanes + were never run, and "skipped 9 lane(s)" reads exactly like a benign skip. */ + if (failFast) stopScheduling = true; } if ((queue.length === 0 || stopScheduling) && running === 0) { resolve({ ok: failed.length === 0, failed, completed, skipped: queue.length }); @@ -192,7 +202,7 @@ export async function runQualityTests({ } async function main() { - const { group, list } = parseArgs(process.argv.slice(2)); + const { group, list, allLanes } = parseArgs(process.argv.slice(2)); const lanes = selectLanes(group); if (list) { @@ -202,11 +212,14 @@ async function main() { return; } - const result = await runQualityTests({ group, lanes }); + const result = await runQualityTests({ group, lanes, failFast: !allLanes }); if (!result.ok) { console.error(`[dashboard-quality] failed lane(s): ${result.failed.map(({ lane }) => lane.name).join(", ")}`); if (result.skipped > 0) { - console.error(`[dashboard-quality] skipped ${result.skipped} lane(s) after first failure`); + console.error( + `[dashboard-quality] ${result.skipped} lane(s) were NOT RUN after the first failure — their status is UNKNOWN, not passing.` + + `\n[dashboard-quality] re-run with --all (or --no-fail-fast) to execute every lane and see the full failure set.`, + ); } process.exit(1); }