FN-6445: reject quality skip-list overlaps
Prevent dashboard curated skip-list entries from masking tests already covered by quality projects. - Add validation that flags skip-list files included by quality lanes. - Cover overlap, empty-reason overlap, and genuine orphan cases in inventory tests. - Document that skip-lists are only for genuinely non-executed dashboard tests. Files changed: docs/testing.md | 1 + scripts/__tests__/check-test-inventory.test.mjs | 29 ++++++++++++++++++++++--- scripts/check-test-inventory.mjs | 9 ++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) Fusion-Task-Id: FN-6445 Fusion-Task-Lineage: 1741f684-8dfc-41fc-8f30-baa3700ac11b
This commit is contained in:
@@ -102,6 +102,7 @@ every entry needs a non-empty `reason` (empty reasons are rejected). Skip-list p
|
||||
fail in isolation) and `build-output.test.ts` (runs standalone via `test:build`
|
||||
after a Vite build). Each carries a one-line reason.
|
||||
- <!-- FNXC:DashboardTesting 2026-06-14-08:00: Skip-listed dashboard tests need actionable ownership; placeholder IDs block rescue/delete follow-through, so every non-standalone reason cites a concrete Fusion tracking task. --> Every skip-list `reason` for a pre-existing failing/orphaned test must reference a concrete `FN-NNNN` tracking task; if the test is rescued, remove the entry instead of leaving a tracking placeholder.
|
||||
- <!-- FNXC:DashboardTesting 2026-06-14-10:27: FN-6445 closes the useChatRooms.test.ts tracking drift from FN-6442: a skip-list entry that is already matched by any quality project is not a genuine ungated orphan and would overstate the orphan count. --> The guard rejects any skip-list entry whose file is already executed by a quality project. Remove the entry instead; the skip-list is only for genuinely non-executed files.
|
||||
- To remove a file from the skip-list: fix the test, confirm it passes under its
|
||||
project, delete the skip-list entry. The backfill lane then executes it.
|
||||
- The skip-list is shared verbatim with `vitest.config.ts`, which excludes the same
|
||||
|
||||
@@ -100,6 +100,17 @@ test("curated guard: fails on an unregistered (synthetic) test file", () => {
|
||||
assert.ok(errors.some((e) => e.includes("synthetic-unregistered.test.ts")));
|
||||
});
|
||||
|
||||
test("curated guard: rejects a skip-list entry that overlaps an executed quality file", () => {
|
||||
const overlappingFile = "packages/dashboard/app/hooks/__tests__/useChatRooms.test.ts";
|
||||
const { ok, errors } = validateDashboardCurated({
|
||||
includedFiles: new Set([overlappingFile]),
|
||||
allTestFiles: [overlappingFile],
|
||||
skipList: [{ file: overlappingFile, reason: "pre-existing orphan FN-6442" }],
|
||||
});
|
||||
assert.equal(ok, false);
|
||||
assert.ok(errors.some((e) => e.includes(overlappingFile) && e.includes("overlaps")));
|
||||
});
|
||||
|
||||
test("curated guard: rejects a skip-list entry with an empty reason", () => {
|
||||
const { ok, errors } = validateDashboardCurated({
|
||||
includedFiles: new Set(),
|
||||
@@ -110,13 +121,25 @@ test("curated guard: rejects a skip-list entry with an empty reason", () => {
|
||||
assert.ok(errors.some((e) => e.includes("empty")));
|
||||
});
|
||||
|
||||
test("curated guard: a skip-listed file does not trip the unregistered check", () => {
|
||||
const { ok } = validateDashboardCurated({
|
||||
test("curated guard: a skip-listed genuine orphan does not trip the overlap check", () => {
|
||||
const { ok, errors } = validateDashboardCurated({
|
||||
includedFiles: new Set(),
|
||||
allTestFiles: ["packages/dashboard/app/b.test.ts"],
|
||||
skipList: [{ file: "packages/dashboard/app/b.test.ts", reason: "pre-existing failure FN-2" }],
|
||||
});
|
||||
assert.equal(ok, true);
|
||||
assert.equal(ok, true, errors.join("; "));
|
||||
});
|
||||
|
||||
test("curated guard: overlapping skip-list entry still reports an empty reason", () => {
|
||||
const overlappingFile = "packages/dashboard/app/hooks/__tests__/useChatRooms.test.ts";
|
||||
const { ok, errors } = validateDashboardCurated({
|
||||
includedFiles: new Set([overlappingFile]),
|
||||
allTestFiles: [overlappingFile],
|
||||
skipList: [{ file: overlappingFile, reason: " " }],
|
||||
});
|
||||
assert.equal(ok, false);
|
||||
assert.ok(errors.some((e) => e.includes(overlappingFile) && e.includes("empty")));
|
||||
assert.ok(errors.some((e) => e.includes(overlappingFile) && e.includes("overlaps")));
|
||||
});
|
||||
|
||||
test("curated guard: a quarantined file is registered without returning to the skip-list", () => {
|
||||
|
||||
@@ -227,6 +227,15 @@ export function validateDashboardCurated({ includedFiles, allTestFiles, skipList
|
||||
if (typeof entry.reason !== "string" || entry.reason.trim().length === 0) {
|
||||
errors.push(`skip-list entry for ${entry.file} has an empty "reason"`);
|
||||
}
|
||||
/*
|
||||
FNXC:DashboardTesting 2026-06-14-10:27:
|
||||
FN-6445 requires the curated skip-list to enumerate only dashboard tests no quality project executes. FN-6442 found useChatRooms.test.ts was both skip-listed and matched by the hooks/utils quality lane, which overstated the genuinely ungated orphan count; reject that overlap at validation time.
|
||||
*/
|
||||
if (includedFiles.has(entry.file)) {
|
||||
errors.push(
|
||||
`skip-list entry for ${entry.file} overlaps a file already executed by a quality project; the skip-list is for genuinely non-executed files only — remove this entry`,
|
||||
);
|
||||
}
|
||||
skipByFile.set(entry.file, entry);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user