fix(merge): resolve merge conflicts with origin/main
- Renumber behavioral verification schema migration from 118→124 (main added migrations 118-123 for usage_events, knowledge_pages, deployments/incidents, token usage, source-issue closure, LOC stats) - Combine backend + timeoutMsOverride params in verification-utils.ts - Accept main's quarantine rescue for AI-merge suites (FN-6433) - Update schema version assertions to 124 across test files
This commit is contained in:
49
scripts/__tests__/check-no-test-timeout-appeasement.test.mjs
Normal file
49
scripts/__tests__/check-no-test-timeout-appeasement.test.mjs
Normal file
@@ -0,0 +1,49 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { formatFailureMessage, scanFileContent } from "../check-no-test-timeout-appeasement.mjs";
|
||||
|
||||
const emptyAllowlist = { allowlistEntries: [] };
|
||||
|
||||
test("scanFileContent reports vi.setConfig testTimeout bumps", () => {
|
||||
const source = ["import { vi } from 'vitest';", "vi.setConfig({ testTimeout: 30000 });"].join("\n");
|
||||
const matches = scanFileContent(source, "packages/x/src/a.test.ts", emptyAllowlist);
|
||||
assert.equal(matches.length, 1);
|
||||
assert.equal(matches[0].lineNumber, 2);
|
||||
assert.match(matches[0].line, /testTimeout/);
|
||||
});
|
||||
|
||||
test("scanFileContent reports hookTimeout bumps", () => {
|
||||
const matches = scanFileContent("vi.setConfig({ hookTimeout: 30000 });", "packages/x/src/a.test.ts", emptyAllowlist);
|
||||
assert.equal(matches.length, 1);
|
||||
assert.equal(matches[0].lineNumber, 1);
|
||||
assert.match(matches[0].line, /hookTimeout/);
|
||||
});
|
||||
|
||||
test("scanFileContent ignores allowlisted files with a rationale", () => {
|
||||
const matches = scanFileContent("vi.setConfig({ testTimeout: 30000 });", "packages/x/src/a.test.ts", {
|
||||
allowlistEntries: [
|
||||
{
|
||||
file: "packages/x/src/a.test.ts",
|
||||
reason: "legacy timeout pending FN-0000 removal",
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.equal(matches.length, 0);
|
||||
});
|
||||
|
||||
test("scanFileContent ignores global vitest config timeouts and non-test paths", () => {
|
||||
const configMatches = scanFileContent("testTimeout: 30_000,", "packages/x/vitest.config.ts", emptyAllowlist);
|
||||
const sourceMatches = scanFileContent("testTimeout: 30_000,", "packages/x/src/config.ts", emptyAllowlist);
|
||||
assert.equal(configMatches.length, 0);
|
||||
assert.equal(sourceMatches.length, 0);
|
||||
});
|
||||
|
||||
test("formatFailureMessage cites file, line, quarantine remediation, and allowlist", () => {
|
||||
const message = formatFailureMessage([
|
||||
{ filePath: "packages/x/src/a.test.ts", lineNumber: 3, line: "vi.setConfig({ testTimeout: 30000 });" },
|
||||
]);
|
||||
assert.match(message, /packages\/x\/src\/a\.test\.ts:3/);
|
||||
assert.match(message, /scripts\/lib\/test-quarantine\.json/);
|
||||
assert.match(message, /Do Not Add Slow Tests/);
|
||||
assert.match(message, /scripts\/lib\/test-timeout-appeasement-allowlist\.json/);
|
||||
});
|
||||
@@ -100,6 +100,17 @@ test("curated guard: fails on an unregistered (synthetic) test file", () => {
|
||||
assert.ok(errors.some((e) => e.includes("synthetic-unregistered.test.ts")));
|
||||
});
|
||||
|
||||
test("curated guard: rejects a skip-list entry that overlaps an executed quality file", () => {
|
||||
const overlappingFile = "packages/dashboard/app/hooks/__tests__/useChatRooms.test.ts";
|
||||
const { ok, errors } = validateDashboardCurated({
|
||||
includedFiles: new Set([overlappingFile]),
|
||||
allTestFiles: [overlappingFile],
|
||||
skipList: [{ file: overlappingFile, reason: "pre-existing orphan FN-6442" }],
|
||||
});
|
||||
assert.equal(ok, false);
|
||||
assert.ok(errors.some((e) => e.includes(overlappingFile) && e.includes("overlaps")));
|
||||
});
|
||||
|
||||
test("curated guard: rejects a skip-list entry with an empty reason", () => {
|
||||
const { ok, errors } = validateDashboardCurated({
|
||||
includedFiles: new Set(),
|
||||
@@ -110,13 +121,57 @@ test("curated guard: rejects a skip-list entry with an empty reason", () => {
|
||||
assert.ok(errors.some((e) => e.includes("empty")));
|
||||
});
|
||||
|
||||
test("curated guard: a skip-listed file does not trip the unregistered check", () => {
|
||||
const { ok } = validateDashboardCurated({
|
||||
test("curated guard: a skip-listed genuine orphan does not trip the overlap check", () => {
|
||||
const { ok, errors } = validateDashboardCurated({
|
||||
includedFiles: new Set(),
|
||||
allTestFiles: ["packages/dashboard/app/b.test.ts"],
|
||||
skipList: [{ file: "packages/dashboard/app/b.test.ts", reason: "pre-existing failure FN-2" }],
|
||||
});
|
||||
assert.equal(ok, true);
|
||||
assert.equal(ok, true, errors.join("; "));
|
||||
});
|
||||
|
||||
test("curated guard: overlapping skip-list entry still reports an empty reason", () => {
|
||||
const overlappingFile = "packages/dashboard/app/hooks/__tests__/useChatRooms.test.ts";
|
||||
const { ok, errors } = validateDashboardCurated({
|
||||
includedFiles: new Set([overlappingFile]),
|
||||
allTestFiles: [overlappingFile],
|
||||
skipList: [{ file: overlappingFile, reason: " " }],
|
||||
});
|
||||
assert.equal(ok, false);
|
||||
assert.ok(errors.some((e) => e.includes(overlappingFile) && e.includes("empty")));
|
||||
assert.ok(errors.some((e) => e.includes(overlappingFile) && e.includes("overlaps")));
|
||||
});
|
||||
|
||||
test("curated guard: a quarantined file is registered without returning to the skip-list", () => {
|
||||
const { ok, errors } = validateDashboardCurated({
|
||||
includedFiles: new Set(),
|
||||
allTestFiles: ["packages/dashboard/app/quarantined.test.ts"],
|
||||
skipList: [],
|
||||
quarantineList: [
|
||||
{
|
||||
file: "packages/dashboard/app/quarantined.test.ts",
|
||||
reason: "quarantined under deletion ratchet FN-4",
|
||||
quarantinedAt: "2026-06-14",
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.equal(ok, true, errors.join("; "));
|
||||
});
|
||||
|
||||
test("curated guard: rejects quarantine entries without a ratchet date", () => {
|
||||
const { ok, errors } = validateDashboardCurated({
|
||||
includedFiles: new Set(),
|
||||
allTestFiles: ["packages/dashboard/app/quarantined.test.ts"],
|
||||
skipList: [],
|
||||
quarantineList: [
|
||||
{
|
||||
file: "packages/dashboard/app/quarantined.test.ts",
|
||||
reason: "quarantined under deletion ratchet FN-4",
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.equal(ok, false);
|
||||
assert.ok(errors.some((e) => e.includes("quarantinedAt")));
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -49,6 +49,72 @@ test("fails when a tracked temp leak appears after baseline", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("ignores tracked temp dirs that disappear during the settle window", () => {
|
||||
withFixture(({ cwd, home }) => {
|
||||
const before = runScript(["--before"], { cwd, home });
|
||||
assert.equal(before.status, 0);
|
||||
|
||||
const transientName = `fusion-test-transient-worker-${process.pid}`;
|
||||
const transientPath = path.join(tmpdir(), transientName);
|
||||
mkdirSync(transientPath, { recursive: true });
|
||||
const cleanup = spawn(process.execPath, ["-e", `setTimeout(() => require("node:fs").rmSync(process.argv[1], { recursive: true, force: true }), 100)`, transientPath], {
|
||||
cwd,
|
||||
env: { ...process.env, HOME: home, USERPROFILE: home },
|
||||
stdio: "ignore",
|
||||
});
|
||||
try {
|
||||
const after = runScript([], { cwd, home });
|
||||
assert.equal(after.status, 0, after.stderr || after.stdout);
|
||||
} finally {
|
||||
cleanup.kill("SIGTERM");
|
||||
rmSync(transientPath, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("ignores active fusion-test-workers roots created after baseline", () => {
|
||||
withFixture(({ cwd, home }) => {
|
||||
const before = runScript(["--before"], { cwd, home });
|
||||
assert.equal(before.status, 0);
|
||||
|
||||
const activeRoot = path.join(tmpdir(), `fusion-test-workers-active-check-${process.pid}`);
|
||||
const owner = spawn(process.execPath, ["-e", "setTimeout(() => {}, 5000)"], {
|
||||
cwd,
|
||||
env: { ...process.env, HOME: home, USERPROFILE: home },
|
||||
stdio: "ignore",
|
||||
});
|
||||
mkdirSync(activeRoot, { recursive: true });
|
||||
writeFileSync(path.join(activeRoot, ".fusion-test-worker-root-owner"), `${owner.pid}\n`);
|
||||
|
||||
try {
|
||||
const after = runScript([], { cwd, home });
|
||||
assert.equal(after.status, 0, after.stderr || after.stdout);
|
||||
} finally {
|
||||
owner.kill("SIGTERM");
|
||||
rmSync(activeRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("fails stale fusion-test-workers roots created after baseline", () => {
|
||||
withFixture(({ cwd, home }) => {
|
||||
const before = runScript(["--before"], { cwd, home });
|
||||
assert.equal(before.status, 0);
|
||||
|
||||
const staleRoot = path.join(tmpdir(), `fusion-test-workers-stale-check-${process.pid}`);
|
||||
mkdirSync(staleRoot, { recursive: true });
|
||||
writeFileSync(path.join(staleRoot, ".fusion-test-worker-root-owner"), "424242424\n");
|
||||
|
||||
try {
|
||||
const after = runScript([], { cwd, home });
|
||||
assert.equal(after.status, 1);
|
||||
assert.match(after.stderr, /leaked temp director/i);
|
||||
} finally {
|
||||
rmSync(staleRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("ignores leaked temp dirs whose basenames appear in FUSION_TEST_ISOLATION_IGNORE_NAMES", () => {
|
||||
withFixture(({ cwd, home }) => {
|
||||
const before = runScript(["--before"], { cwd, home });
|
||||
|
||||
90
scripts/__tests__/dist-freshness.test.mjs
Normal file
90
scripts/__tests__/dist-freshness.test.mjs
Normal file
@@ -0,0 +1,90 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { computeDistStaleness, formatDistStalenessWarning } from "../lib/dist-freshness.mjs";
|
||||
|
||||
/*
|
||||
FNXC:DevWorkflow 2026-06-18-16:50:
|
||||
FN-6638 stale-dist guard tests. Verifies the startup freshness check flags a
|
||||
src-ahead-of-dist build, stays quiet when fresh, and never false-positives for
|
||||
pure-source (no dist) or packaged (no src) layouts.
|
||||
*/
|
||||
|
||||
// In-memory fs seam: paths are exact strings; dirs list children; files carry mtimeMs.
|
||||
function makeFs({ dirs, files }) {
|
||||
const dirSet = new Set(dirs);
|
||||
// files: { "<dir>": [{ name, mtimeMs, isDir? }] } keyed by parent dir
|
||||
return {
|
||||
existsSync: (p) => dirSet.has(p),
|
||||
readdirSync: (dir) =>
|
||||
(files[dir] ?? []).map((e) => ({
|
||||
name: e.name,
|
||||
isDirectory: () => Boolean(e.isDir),
|
||||
})),
|
||||
statSync: (p) => {
|
||||
// p is "<dir>/<name>"; look it up by scanning entries
|
||||
for (const [dir, entries] of Object.entries(files)) {
|
||||
for (const e of entries) {
|
||||
if (`${dir}/${e.name}` === p) return { mtimeMs: e.mtimeMs };
|
||||
}
|
||||
}
|
||||
return { mtimeMs: 0 };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const ROOT = "/repo";
|
||||
|
||||
function layout({ srcMs, distMs, withSrc = true, withDist = true }) {
|
||||
const dirs = [];
|
||||
const files = {};
|
||||
const srcDir = `${ROOT}/packages/engine/src`;
|
||||
const distDir = `${ROOT}/packages/engine/dist`;
|
||||
if (withSrc) {
|
||||
dirs.push(srcDir);
|
||||
files[srcDir] = [{ name: "executor.ts", mtimeMs: srcMs }];
|
||||
}
|
||||
if (withDist) {
|
||||
dirs.push(distDir);
|
||||
files[distDir] = [{ name: "executor.js", mtimeMs: distMs }];
|
||||
}
|
||||
return makeFs({ dirs, files });
|
||||
}
|
||||
|
||||
test("flags stale when src is newer than dist beyond slack", () => {
|
||||
const fs = layout({ srcMs: 10_000, distMs: 1_000 });
|
||||
const result = computeDistStaleness({ rootDir: ROOT, packages: ["engine"], fs });
|
||||
assert.equal(result.stale, true);
|
||||
assert.equal(result.packages[0].stale, true);
|
||||
const warning = formatDistStalenessWarning(result);
|
||||
assert.match(warning, /STALE BUILD/);
|
||||
assert.match(warning, /@fusion\/engine/);
|
||||
assert.match(warning, /pnpm build/);
|
||||
});
|
||||
|
||||
test("not stale when dist is newer than src", () => {
|
||||
const fs = layout({ srcMs: 1_000, distMs: 10_000 });
|
||||
const result = computeDistStaleness({ rootDir: ROOT, packages: ["engine"], fs });
|
||||
assert.equal(result.stale, false);
|
||||
assert.equal(formatDistStalenessWarning(result), null);
|
||||
});
|
||||
|
||||
test("not stale within slack window", () => {
|
||||
const fs = layout({ srcMs: 1_500, distMs: 1_000 }); // 500ms < 2000ms slack
|
||||
const result = computeDistStaleness({ rootDir: ROOT, packages: ["engine"], fs });
|
||||
assert.equal(result.stale, false);
|
||||
});
|
||||
|
||||
test("skips packages with no dist (pure source run)", () => {
|
||||
const fs = layout({ srcMs: 10_000, distMs: 0, withDist: false });
|
||||
const result = computeDistStaleness({ rootDir: ROOT, packages: ["engine"], fs });
|
||||
assert.equal(result.stale, false);
|
||||
assert.equal(result.packages.length, 0);
|
||||
});
|
||||
|
||||
test("skips packages with no src (packaged install)", () => {
|
||||
const fs = layout({ srcMs: 0, distMs: 10_000, withSrc: false });
|
||||
const result = computeDistStaleness({ rootDir: ROOT, packages: ["engine"], fs });
|
||||
assert.equal(result.stale, false);
|
||||
assert.equal(result.packages.length, 0);
|
||||
});
|
||||
131
scripts/__tests__/ios-acceptance.test.mjs
Normal file
131
scripts/__tests__/ios-acceptance.test.mjs
Normal file
@@ -0,0 +1,131 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
buildIosCapabilities,
|
||||
describeAvailability,
|
||||
iosHubUrl,
|
||||
redactUrl,
|
||||
resolveIosProvider,
|
||||
} from "../lib/ios-acceptance.mjs";
|
||||
|
||||
const allCredentials = {
|
||||
BROWSERSTACK_USERNAME: "browserstack-user",
|
||||
BROWSERSTACK_ACCESS_KEY: "browserstack-key",
|
||||
SAUCE_USERNAME: "sauce-user",
|
||||
SAUCE_ACCESS_KEY: "sauce-key",
|
||||
LT_USERNAME: "lt-user",
|
||||
LT_ACCESS_KEY: "lt-key",
|
||||
};
|
||||
|
||||
test("resolveIosProvider follows BrowserStack, Sauce, LambdaTest precedence", () => {
|
||||
assert.equal(resolveIosProvider(allCredentials), "browserstack");
|
||||
assert.equal(
|
||||
resolveIosProvider({
|
||||
SAUCE_USERNAME: "sauce-user",
|
||||
SAUCE_ACCESS_KEY: "sauce-key",
|
||||
}),
|
||||
"sauce",
|
||||
);
|
||||
assert.equal(
|
||||
resolveIosProvider({
|
||||
LT_USERNAME: "lt-user",
|
||||
LT_ACCESS_KEY: "lt-key",
|
||||
}),
|
||||
"lambdatest",
|
||||
);
|
||||
assert.equal(resolveIosProvider({}), null);
|
||||
});
|
||||
|
||||
test("resolveIosProvider treats whitespace-only credential values as absent", () => {
|
||||
assert.equal(
|
||||
resolveIosProvider({
|
||||
BROWSERSTACK_USERNAME: "browserstack-user",
|
||||
BROWSERSTACK_ACCESS_KEY: " ",
|
||||
SAUCE_USERNAME: "\t",
|
||||
SAUCE_ACCESS_KEY: "sauce-key",
|
||||
LT_USERNAME: "lt-user",
|
||||
LT_ACCESS_KEY: "\n",
|
||||
}),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test("describeAvailability enumerates checked and missing keys without secret values", () => {
|
||||
const availability = describeAvailability({});
|
||||
assert.equal(availability.available, false);
|
||||
assert.equal(availability.provider, null);
|
||||
assert.deepEqual(availability.checkedKeys, [
|
||||
"BROWSERSTACK_USERNAME",
|
||||
"BROWSERSTACK_ACCESS_KEY",
|
||||
"SAUCE_USERNAME",
|
||||
"SAUCE_ACCESS_KEY",
|
||||
"LT_USERNAME",
|
||||
"LT_ACCESS_KEY",
|
||||
]);
|
||||
assert.deepEqual(availability.missing, availability.checkedKeys);
|
||||
|
||||
const withSecrets = describeAvailability(allCredentials);
|
||||
const serialized = JSON.stringify(withSecrets);
|
||||
for (const value of Object.values(allCredentials)) {
|
||||
assert.equal(serialized.includes(value), false, `availability leaked credential value ${value}`);
|
||||
}
|
||||
});
|
||||
|
||||
test("buildIosCapabilities creates real iOS Safari capabilities for each provider", () => {
|
||||
const browserstack = buildIosCapabilities("browserstack", {
|
||||
deviceName: "iPhone 14",
|
||||
platformVersion: "16",
|
||||
});
|
||||
assert.equal(browserstack.browserName, "safari");
|
||||
assert.equal(browserstack.platformName, "iOS");
|
||||
assert.equal(browserstack["bstack:options"].deviceName, "iPhone 14");
|
||||
assert.equal(browserstack["bstack:options"].osVersion, "16");
|
||||
assert.equal(browserstack["bstack:options"].realMobile, true);
|
||||
|
||||
const sauce = buildIosCapabilities("sauce", {
|
||||
deviceName: "iPhone 15 Pro",
|
||||
platformVersion: "17",
|
||||
});
|
||||
assert.equal(sauce.browserName, "safari");
|
||||
assert.equal(sauce.platformName, "iOS");
|
||||
assert.equal(sauce["appium:deviceName"], "iPhone 15 Pro");
|
||||
assert.equal(sauce["appium:platformVersion"], "17");
|
||||
assert.equal(sauce["sauce:options"].realDevice, true);
|
||||
|
||||
const lambdatest = buildIosCapabilities("lambdatest", {
|
||||
deviceName: "iPhone 13",
|
||||
platformVersion: "15",
|
||||
});
|
||||
assert.equal(lambdatest.browserName, "safari");
|
||||
assert.equal(lambdatest.platformName, "iOS");
|
||||
assert.equal(lambdatest["LT:Options"].deviceName, "iPhone 13");
|
||||
assert.equal(lambdatest["LT:Options"].platformVersion, "15");
|
||||
assert.equal(lambdatest["LT:Options"].isRealMobile, true);
|
||||
});
|
||||
|
||||
test("iosHubUrl uses provider defaults, env overrides, and redacts embedded credentials", () => {
|
||||
const browserstackUrl = iosHubUrl(
|
||||
"browserstack",
|
||||
{ username: "user@example.com", accessKey: "browserstack-secret" },
|
||||
{},
|
||||
);
|
||||
assert.equal(browserstackUrl, "https://user%40example.com:browserstack-secret@hub-cloud.browserstack.com/wd/hub");
|
||||
|
||||
const sauceUrl = iosHubUrl("sauce", { username: "sauce-user", accessKey: "sauce-secret" }, {});
|
||||
assert.equal(sauceUrl, "https://sauce-user:sauce-secret@ondemand.us-west-1.saucelabs.com/wd/hub");
|
||||
|
||||
const ltUrl = iosHubUrl("lambdatest", { username: "lt-user", accessKey: "lt-secret" }, {});
|
||||
assert.equal(ltUrl, "https://lt-user:lt-secret@mobile-hub.lambdatest.com/wd/hub");
|
||||
|
||||
const overrideUrl = iosHubUrl(
|
||||
"browserstack",
|
||||
{ username: "override-user", accessKey: "override-secret" },
|
||||
{ BROWSERSTACK_HUB_URL: "https://example.test/custom/wd/hub" },
|
||||
);
|
||||
assert.equal(overrideUrl, "https://override-user:override-secret@example.test/custom/wd/hub");
|
||||
|
||||
const redacted = redactUrl(overrideUrl);
|
||||
assert.equal(redacted, "https://<redacted>:<redacted>@example.test/custom/wd/hub");
|
||||
assert.equal(redacted.includes("override-user"), false);
|
||||
assert.equal(redacted.includes("override-secret"), false);
|
||||
});
|
||||
70
scripts/__tests__/release-authorization-gate.test.mjs
Normal file
70
scripts/__tests__/release-authorization-gate.test.mjs
Normal file
@@ -0,0 +1,70 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { URL } from "node:url";
|
||||
|
||||
import {
|
||||
evaluateReleaseAuthorization,
|
||||
RELEASE_AUTHORIZATION_ENV,
|
||||
} from "../lib/release-authorization-gate.mjs";
|
||||
|
||||
test("gate blocks real release without signal in non-interactive FN-6469 path", () => {
|
||||
const result = evaluateReleaseAuthorization({ dryRun: false, env: {}, stdinIsTTY: false });
|
||||
|
||||
assert.equal(result.authorized, false);
|
||||
assert.equal(result.mode, "blocked");
|
||||
assert.match(result.reason ?? "", /non-interactive shell/);
|
||||
assert.match(result.reason ?? "", /aborted before version bump, publish, push, or tag/);
|
||||
});
|
||||
|
||||
test("gate allows real release with explicit operator signal", () => {
|
||||
const result = evaluateReleaseAuthorization({
|
||||
dryRun: false,
|
||||
env: { [RELEASE_AUTHORIZATION_ENV]: "operator-held-one-time-approval" },
|
||||
stdinIsTTY: false,
|
||||
});
|
||||
|
||||
assert.deepEqual(result, { authorized: true, mode: "env-signal" });
|
||||
});
|
||||
|
||||
test("dry-run bypasses authorization because it publishes nothing", () => {
|
||||
const result = evaluateReleaseAuthorization({ dryRun: true, env: {}, stdinIsTTY: false });
|
||||
|
||||
assert.deepEqual(result, { authorized: true, mode: "dry-run-bypass" });
|
||||
});
|
||||
|
||||
test("empty or whitespace-only authorization signal fails closed", () => {
|
||||
for (const value of ["", " ", "\n\t"]) {
|
||||
const result = evaluateReleaseAuthorization({
|
||||
dryRun: false,
|
||||
env: { [RELEASE_AUTHORIZATION_ENV]: value },
|
||||
stdinIsTTY: false,
|
||||
});
|
||||
|
||||
assert.equal(result.authorized, false, `expected ${JSON.stringify(value)} to be blocked`);
|
||||
assert.equal(result.mode, "blocked");
|
||||
}
|
||||
});
|
||||
|
||||
test("TTY presence alone does not authorize a real release", () => {
|
||||
const result = evaluateReleaseAuthorization({ dryRun: false, env: {}, stdinIsTTY: true });
|
||||
|
||||
assert.equal(result.authorized, false);
|
||||
assert.equal(result.mode, "blocked");
|
||||
assert.match(result.reason ?? "", /interactive shell/);
|
||||
});
|
||||
|
||||
test("release script imports and enforces the authorization gate after dry-run exit", () => {
|
||||
const source = readFileSync(new URL("../release.mjs", import.meta.url), "utf8");
|
||||
const importIndex = source.indexOf("./lib/release-authorization-gate.mjs");
|
||||
const dryRunExitIndex = source.indexOf("if (DRY_RUN) {");
|
||||
const gateIndex = source.indexOf("evaluateReleaseAuthorization({");
|
||||
const versionBumpIndex = source.indexOf("run(\"pnpm release:version\")");
|
||||
|
||||
assert.notEqual(importIndex, -1, "release.mjs should import the authorization helper");
|
||||
assert.notEqual(dryRunExitIndex, -1, "release.mjs should retain the dry-run early exit");
|
||||
assert.notEqual(gateIndex, -1, "release.mjs should call evaluateReleaseAuthorization()");
|
||||
assert.notEqual(versionBumpIndex, -1, "release.mjs should still run the version bump after gates");
|
||||
assert.ok(dryRunExitIndex < gateIndex, "dry-run must exit before the authorization gate call site");
|
||||
assert.ok(gateIndex < versionBumpIndex, "authorization must be checked before the first mutation");
|
||||
});
|
||||
72
scripts/__tests__/release-prompt-gate.test.mjs
Normal file
72
scripts/__tests__/release-prompt-gate.test.mjs
Normal file
@@ -0,0 +1,72 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { URL } from "node:url";
|
||||
|
||||
import { shouldPromptForVersion } from "../lib/release-prompt-gate.mjs";
|
||||
|
||||
test("dry-run is non-interactive by default for the FN-6469 no-TTY path", () => {
|
||||
assert.equal(
|
||||
shouldPromptForVersion({ dryRun: true, autoYes: false, interactive: false }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("dry-run interactive override exercises the version prompt", () => {
|
||||
assert.equal(
|
||||
shouldPromptForVersion({ dryRun: true, autoYes: false, interactive: true }),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("dry-run --yes never prompts regardless of interactive flag", () => {
|
||||
assert.equal(
|
||||
shouldPromptForVersion({ dryRun: true, autoYes: true, interactive: false }),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldPromptForVersion({ dryRun: true, autoYes: true, interactive: true }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("real releases prompt unless --yes is passed", () => {
|
||||
assert.equal(
|
||||
shouldPromptForVersion({ dryRun: false, autoYes: false, interactive: false }),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
shouldPromptForVersion({ dryRun: false, autoYes: false, interactive: true }),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
shouldPromptForVersion({ dryRun: false, autoYes: true, interactive: false }),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
shouldPromptForVersion({ dryRun: false, autoYes: true, interactive: true }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("prompt decision is independent of representative version values", () => {
|
||||
const representativeVersions = ["0.43.1", "1.0.0", "2.0.0-beta.1"];
|
||||
const decisions = representativeVersions.map(() =>
|
||||
shouldPromptForVersion({ dryRun: true, autoYes: false, interactive: false }),
|
||||
);
|
||||
|
||||
assert.deepEqual(decisions, [false, false, false]);
|
||||
});
|
||||
|
||||
test("release script dry-run exits before proceed confirmation and gates ask through helper", () => {
|
||||
const source = readFileSync(new URL("../release.mjs", import.meta.url), "utf8");
|
||||
const promptGateIndex = source.indexOf("shouldPromptForVersion({ dryRun: DRY_RUN, autoYes: AUTO_YES, interactive: INTERACTIVE })");
|
||||
const askIndex = source.indexOf("await ask(`Release version");
|
||||
const dryRunExitIndex = source.indexOf("if (DRY_RUN) {");
|
||||
const confirmIndex = source.indexOf("await confirm(`Proceed with release");
|
||||
|
||||
assert.notEqual(promptGateIndex, -1, "release.mjs should use the pure prompt gate");
|
||||
assert.notEqual(askIndex, -1, "release.mjs should still support version prompts");
|
||||
assert.ok(promptGateIndex < askIndex, "ask() must be guarded by shouldPromptForVersion()");
|
||||
assert.ok(dryRunExitIndex < confirmIndex, "dry-run must exit before proceed confirmation");
|
||||
});
|
||||
231
scripts/__tests__/run-vitest-watchdog.test.mjs
Normal file
231
scripts/__tests__/run-vitest-watchdog.test.mjs
Normal file
@@ -0,0 +1,231 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { EventEmitter } from "node:events";
|
||||
|
||||
import {
|
||||
CLASS_BUDGET_BANDS,
|
||||
DEFAULT_BUDGET_MULTIPLIER,
|
||||
TIMEOUT_EXIT_CODE,
|
||||
deriveBudgetMs,
|
||||
summarizeActiveHandles,
|
||||
captureHangDiagnostics,
|
||||
runWithWatchdog,
|
||||
} from "../lib/run-vitest-watchdog.mjs";
|
||||
|
||||
function makeFakeChild() {
|
||||
const child = new EventEmitter();
|
||||
child.pid = 999999;
|
||||
child.kill = () => {};
|
||||
return child;
|
||||
}
|
||||
|
||||
// A spawn stub that returns a controllable fake child.
|
||||
function fakeSpawn(child) {
|
||||
return () => child;
|
||||
}
|
||||
|
||||
test("deriveBudgetMs: no fresh timing falls back to the per-class ceiling", () => {
|
||||
assert.equal(deriveBudgetMs({ klass: "shard" }), CLASS_BUDGET_BANDS.shard.ceiling);
|
||||
assert.equal(
|
||||
deriveBudgetMs({ klass: "changed", expectedDurationMs: 1000, timingsFresh: false }),
|
||||
CLASS_BUDGET_BANDS.changed.ceiling,
|
||||
);
|
||||
// Zero / negative expected duration is treated as unusable → ceiling.
|
||||
assert.equal(
|
||||
deriveBudgetMs({ klass: "shard", expectedDurationMs: 0, timingsFresh: true }),
|
||||
CLASS_BUDGET_BANDS.shard.ceiling,
|
||||
);
|
||||
});
|
||||
|
||||
test("deriveBudgetMs: fresh timing tightens within the band", () => {
|
||||
// expected×multiplier between floor and ceiling → use the tightened value.
|
||||
const expected = 200_000; // 200s
|
||||
const derived = deriveBudgetMs({ klass: "shard", expectedDurationMs: expected, timingsFresh: true });
|
||||
assert.equal(derived, Math.round(expected * DEFAULT_BUDGET_MULTIPLIER));
|
||||
assert.ok(derived >= CLASS_BUDGET_BANDS.shard.floor);
|
||||
assert.ok(derived <= CLASS_BUDGET_BANDS.shard.ceiling);
|
||||
});
|
||||
|
||||
test("deriveBudgetMs: clamps to floor and ceiling", () => {
|
||||
// Tiny expected → clamps up to floor.
|
||||
assert.equal(
|
||||
deriveBudgetMs({ klass: "shard", expectedDurationMs: 1, timingsFresh: true }),
|
||||
CLASS_BUDGET_BANDS.shard.floor,
|
||||
);
|
||||
// Huge expected → clamps down to ceiling.
|
||||
assert.equal(
|
||||
deriveBudgetMs({ klass: "shard", expectedDurationMs: 10 ** 9, timingsFresh: true }),
|
||||
CLASS_BUDGET_BANDS.shard.ceiling,
|
||||
);
|
||||
});
|
||||
|
||||
test("deriveBudgetMs: unknown class falls back to the changed band", () => {
|
||||
assert.equal(deriveBudgetMs({ klass: "nonexistent" }), CLASS_BUDGET_BANDS.changed.ceiling);
|
||||
});
|
||||
|
||||
test("summarizeActiveHandles: returns a bounded string", () => {
|
||||
const summary = summarizeActiveHandles({ limit: 3 });
|
||||
assert.equal(typeof summary, "string");
|
||||
assert.ok(summary.length > 0);
|
||||
});
|
||||
|
||||
test("captureHangDiagnostics: names the invocation, elapsed, and budget", () => {
|
||||
const msg = captureHangDiagnostics({
|
||||
label: "shard 1/4",
|
||||
command: "pnpm",
|
||||
args: ["test"],
|
||||
budgetMs: 1000,
|
||||
startedAt: 0,
|
||||
lastHeartbeatAt: 500,
|
||||
now: 1500,
|
||||
});
|
||||
assert.match(msg, /HANG: shard 1\/4/);
|
||||
assert.match(msg, /elapsed 1500ms/);
|
||||
assert.match(msg, /budget 1000ms/);
|
||||
assert.match(msg, /last heartbeat: 1000ms ago/);
|
||||
});
|
||||
|
||||
test("runWithWatchdog: clean exit propagates code 0, no kill", async () => {
|
||||
const child = makeFakeChild();
|
||||
const killed = [];
|
||||
const p = runWithWatchdog({
|
||||
command: "fake",
|
||||
args: [],
|
||||
budgetMs: 10_000,
|
||||
label: "clean",
|
||||
log: () => {},
|
||||
spawn: fakeSpawn(child),
|
||||
killGroup: (sig) => killed.push(sig),
|
||||
});
|
||||
child.emit("close", 0, null);
|
||||
const result = await p;
|
||||
assert.equal(result.code, 0);
|
||||
assert.equal(result.timedOut, false);
|
||||
assert.equal(result.signal, null);
|
||||
assert.deepEqual(killed, []);
|
||||
});
|
||||
|
||||
test("runWithWatchdog: non-zero exit code is propagated unchanged", async () => {
|
||||
const child = makeFakeChild();
|
||||
const p = runWithWatchdog({
|
||||
command: "fake",
|
||||
args: [],
|
||||
budgetMs: 10_000,
|
||||
label: "fails",
|
||||
log: () => {},
|
||||
spawn: fakeSpawn(child),
|
||||
killGroup: () => {},
|
||||
});
|
||||
child.emit("close", 7, null);
|
||||
const result = await p;
|
||||
assert.equal(result.code, 7);
|
||||
assert.equal(result.timedOut, false);
|
||||
});
|
||||
|
||||
test("runWithWatchdog: timeout fires SIGTERM then SIGKILL and returns 124", async () => {
|
||||
const child = makeFakeChild();
|
||||
const killed = [];
|
||||
let diagnosticsLogged = "";
|
||||
const p = runWithWatchdog({
|
||||
command: "pnpm",
|
||||
args: ["exec", "vitest"],
|
||||
budgetMs: 30, // fire fast
|
||||
graceMs: 20,
|
||||
heartbeatMs: 1000,
|
||||
label: "hanger",
|
||||
log: (m) => {
|
||||
diagnosticsLogged += m + "\n";
|
||||
},
|
||||
spawn: fakeSpawn(child),
|
||||
killGroup: (sig) => {
|
||||
killed.push(sig);
|
||||
// Emulate the group dying only after SIGKILL.
|
||||
if (sig === "SIGKILL") setTimeout(() => child.emit("close", null, "SIGKILL"), 1);
|
||||
},
|
||||
});
|
||||
const result = await p;
|
||||
assert.equal(result.timedOut, true);
|
||||
assert.equal(result.code, TIMEOUT_EXIT_CODE);
|
||||
assert.deepEqual(killed, ["SIGTERM", "SIGKILL"]);
|
||||
assert.match(diagnosticsLogged, /HANG: hanger/);
|
||||
});
|
||||
|
||||
test("runWithWatchdog: child error rejects", async () => {
|
||||
const child = makeFakeChild();
|
||||
const p = runWithWatchdog({
|
||||
command: "fake",
|
||||
args: [],
|
||||
budgetMs: 10_000,
|
||||
label: "errors",
|
||||
log: () => {},
|
||||
spawn: fakeSpawn(child),
|
||||
killGroup: () => {},
|
||||
});
|
||||
child.emit("error", new Error("spawn failed"));
|
||||
await assert.rejects(p, /spawn failed/);
|
||||
});
|
||||
|
||||
test("runWithWatchdog: removes its process listeners after settling", async () => {
|
||||
const beforeTerm = process.listenerCount("SIGTERM");
|
||||
const beforeExit = process.listenerCount("exit");
|
||||
const child = makeFakeChild();
|
||||
const p = runWithWatchdog({
|
||||
command: "fake",
|
||||
args: [],
|
||||
budgetMs: 10_000,
|
||||
label: "cleanup",
|
||||
log: () => {},
|
||||
spawn: fakeSpawn(child),
|
||||
killGroup: () => {},
|
||||
});
|
||||
child.emit("close", 0, null);
|
||||
await p;
|
||||
assert.equal(process.listenerCount("SIGTERM"), beforeTerm);
|
||||
assert.equal(process.listenerCount("exit"), beforeExit);
|
||||
});
|
||||
|
||||
test("runWithWatchdog: forwarded signal escalates to SIGKILL after grace", async () => {
|
||||
const child = makeFakeChild();
|
||||
const killed = [];
|
||||
const p = runWithWatchdog({
|
||||
command: "pnpm",
|
||||
args: [],
|
||||
budgetMs: 10_000,
|
||||
graceMs: 15,
|
||||
heartbeatMs: 1000,
|
||||
label: "cancel",
|
||||
log: () => {},
|
||||
spawn: fakeSpawn(child),
|
||||
killGroup: (sig) => {
|
||||
killed.push(sig);
|
||||
// The child ignores SIGHUP; only SIGKILL takes it down.
|
||||
if (sig === "SIGKILL") child.emit("close", null, "SIGKILL");
|
||||
},
|
||||
});
|
||||
// Simulate external cancellation (Ctrl-C / CI cancel) reaching the wrapper.
|
||||
process.emit("SIGHUP");
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
await p;
|
||||
assert.deepEqual(killed, ["SIGHUP", "SIGKILL"]);
|
||||
});
|
||||
|
||||
test("runWithWatchdog: passes cwd through to spawn when provided", async () => {
|
||||
let capturedOpts = null;
|
||||
const child = makeFakeChild();
|
||||
const p = runWithWatchdog({
|
||||
command: "pnpm",
|
||||
args: ["test"],
|
||||
cwd: "/tmp/repo-root",
|
||||
budgetMs: 10_000,
|
||||
label: "cwd",
|
||||
log: () => {},
|
||||
spawn: (_cmd, _args, opts) => {
|
||||
capturedOpts = opts;
|
||||
return child;
|
||||
},
|
||||
killGroup: () => {},
|
||||
});
|
||||
child.emit("close", 0, null);
|
||||
await p;
|
||||
assert.equal(capturedOpts.cwd, "/tmp/repo-root");
|
||||
});
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
cleanupIsolatedHomePath,
|
||||
knownIsolatedHomeBasenames,
|
||||
__setCleanupRmSyncForTests,
|
||||
__setProcessAliveForTests,
|
||||
emitModeDecision,
|
||||
pruneFusionTestHomes,
|
||||
pruneFusionTestWorkers,
|
||||
@@ -35,7 +36,7 @@ import {
|
||||
computeOwnHash,
|
||||
} from "../test-changed.mjs";
|
||||
|
||||
import { mkdirSync, writeFileSync, mkdtempSync, rmSync, existsSync } from "node:fs";
|
||||
import { mkdirSync, writeFileSync, mkdtempSync, rmSync, existsSync, utimesSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
@@ -1058,6 +1059,77 @@ test("pruneFusionTestWorkers: skips markerless roots with live redirect sinks",
|
||||
}
|
||||
});
|
||||
|
||||
function setOldMtime(pathValue) {
|
||||
const old = new Date(Date.now() - 60_000);
|
||||
utimesSync(pathValue, old, old);
|
||||
}
|
||||
|
||||
function withAlivePid(pid, fn) {
|
||||
__setProcessAliveForTests((candidate) => candidate === pid);
|
||||
try {
|
||||
fn();
|
||||
} finally {
|
||||
__setProcessAliveForTests(null);
|
||||
}
|
||||
}
|
||||
|
||||
test("pruneFusionTestWorkers: prunes owner-marker roots when pid liveness is stale", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-stale-owner-${process.pid}-`));
|
||||
const recycledPid = 424_242;
|
||||
try {
|
||||
writeFileSync(path.join(root, ".fusion-test-worker-root-owner"), `${recycledPid}\nrunToken=prior-run\n`);
|
||||
withAlivePid(recycledPid, () => pruneFusionTestWorkers(1024));
|
||||
assert.equal(existsSync(root), false, "stale pid reuse must not preserve an orphaned worker root");
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("pruneFusionTestWorkers: preserves same-run owner-marker roots with live pids", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-current-owner-${process.pid}-`));
|
||||
const ownerPid = 515_151;
|
||||
try {
|
||||
writeFileSync(
|
||||
path.join(root, ".fusion-test-worker-root-owner"),
|
||||
`${ownerPid}\nrunToken=${process.env.FUSION_TEST_RUN_TOKEN}\n`,
|
||||
);
|
||||
withAlivePid(ownerPid, () => pruneFusionTestWorkers(1024));
|
||||
assert.equal(existsSync(root), true, "current-run live worker root must not be pruned");
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("pruneFusionTestWorkers: prunes old markerless redir roots when pid liveness is stale", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-stale-redir-${process.pid}-`));
|
||||
const recycledPid = 626_262;
|
||||
try {
|
||||
const redir = path.join(root, `redir-${recycledPid}`);
|
||||
mkdirSync(redir, { recursive: true });
|
||||
writeFileSync(path.join(redir, "payload.txt"), "stale\n");
|
||||
setOldMtime(redir);
|
||||
setOldMtime(root);
|
||||
withAlivePid(recycledPid, () => pruneFusionTestWorkers(1024));
|
||||
assert.equal(existsSync(root), false, "old markerless redir root must be pruned despite pid reuse");
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("pruneFusionTestWorkers: removes SIGKILL-style orphan roots and leaves foreign prefixes alone", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), `fusion-test-workers-sigkill-orphan-${process.pid}-`));
|
||||
const foreign = mkdtempSync(path.join(tmpdir(), `not-fusion-test-workers-${process.pid}-`));
|
||||
try {
|
||||
mkdirSync(path.join(root, `w-${process.pid}-orphan`), { recursive: true });
|
||||
pruneFusionTestWorkers(1024);
|
||||
assert.equal(existsSync(root), false, "orphaned worker root should be pruned");
|
||||
assert.equal(existsSync(foreign), true, "foreign prefixes must not be touched");
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
rmSync(foreign, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("pruneFusionTestWorkers: reclaims non-empty root after transient ENOTEMPTY", () => {
|
||||
const root = createNonEmptyPruneRoot("fusion-test-workers-", "transient");
|
||||
withTransientPruneFailure(root, pruneFusionTestWorkers);
|
||||
|
||||
90
scripts/__tests__/test-feedback-baseline.test.mjs
Normal file
90
scripts/__tests__/test-feedback-baseline.test.mjs
Normal file
@@ -0,0 +1,90 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
collectFlakeSummary,
|
||||
collectSlowestFiles,
|
||||
createBaseline,
|
||||
DEFAULT_BASELINES_PATH,
|
||||
DEFAULT_MARKDOWN_PATH,
|
||||
DEFAULT_QUARANTINE_PATH,
|
||||
DEFAULT_TIMINGS_PATH,
|
||||
main,
|
||||
renderMarkdown,
|
||||
} from "../test-feedback-baseline.mjs";
|
||||
|
||||
function writeJson(root, relativePath, value) {
|
||||
const absolutePath = path.join(root, relativePath);
|
||||
mkdirSync(path.dirname(absolutePath), { recursive: true });
|
||||
writeFileSync(absolutePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
test("collectSlowestFiles ranks timing snapshot entries across packages", () => {
|
||||
const rows = collectSlowestFiles({
|
||||
packages: {
|
||||
"@fusion/a": { files: { "a-fast.test.ts": 20, "a-slow.test.ts": 2000 } },
|
||||
"@fusion/b": { files: { "b-medium.test.ts": 1000 } },
|
||||
},
|
||||
}, 2);
|
||||
|
||||
assert.deepEqual(rows, [
|
||||
{ packageName: "@fusion/a", file: "a-slow.test.ts", durationMs: 2000 },
|
||||
{ packageName: "@fusion/b", file: "b-medium.test.ts", durationMs: 1000 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("collectFlakeSummary counts ledger entries and unique quarantined files", () => {
|
||||
const summary = collectFlakeSummary({ entries: [
|
||||
{ file: "one.test.ts" },
|
||||
{ file: "one.test.ts" },
|
||||
{ file: "two.test.ts" },
|
||||
] });
|
||||
|
||||
assert.equal(summary.flakeCount, 3);
|
||||
assert.equal(summary.uniqueQuarantinedFileCount, 2);
|
||||
assert.deepEqual(summary.quarantinedFiles, ["one.test.ts", "two.test.ts"]);
|
||||
});
|
||||
|
||||
test("renderMarkdown includes #leads summary, trend, and slowest files", () => {
|
||||
const baseline = createBaseline({
|
||||
now: new Date("2026-06-17T18:00:00.000Z"),
|
||||
gateWallTimeMs: 12_300,
|
||||
pnpmTestWallTimeMs: 45_600,
|
||||
timings: { capturedAt: "2026-06-17T17:00:00.000Z", packages: { "@fusion/core": { files: { "packages/core/src/__tests__/agent-store.test.ts": 11_600 } } } },
|
||||
quarantine: { entries: [{ file: "packages/core/src/__tests__/flake.test.ts" }] },
|
||||
});
|
||||
|
||||
const markdown = renderMarkdown([baseline]);
|
||||
|
||||
assert.match(markdown, /Latest #leads summary/);
|
||||
assert.match(markdown, /Gate suite wall-time: \*\*12\.3s\*\*/);
|
||||
assert.match(markdown, /packages\/core\/src\/__tests__\/agent-store\.test\.ts/);
|
||||
assert.match(markdown, /Quarantined tests remain on the 14-day rescue-or-delete clock/);
|
||||
});
|
||||
|
||||
test("main records a baseline and writes the markdown publication artifact", async () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "fusion-test-feedback-baseline-"));
|
||||
writeJson(root, DEFAULT_TIMINGS_PATH, {
|
||||
capturedAt: "2026-06-17T17:00:00.000Z",
|
||||
packages: { "@fusion/core": { files: { "slow.test.ts": 1500, "fast.test.ts": 100 } } },
|
||||
});
|
||||
writeJson(root, DEFAULT_QUARANTINE_PATH, { entries: [{ file: "slow.test.ts" }] });
|
||||
|
||||
const chunks = [];
|
||||
const code = await main(["--record", "--gate-ms", "1000", "--test-ms", "2000", "--print-leads"], {
|
||||
rootDir: root,
|
||||
stdout: { write: (chunk) => chunks.push(String(chunk)) },
|
||||
stderr: { write: () => {} },
|
||||
});
|
||||
|
||||
assert.equal(code, 0);
|
||||
assert.match(chunks.join(""), /gate 1\.0s, pnpm test 2\.0s/);
|
||||
const store = JSON.parse(readFileSync(path.join(root, DEFAULT_BASELINES_PATH), "utf8"));
|
||||
assert.equal(store.baselines.length, 1);
|
||||
assert.equal(store.baselines[0].flakeCount, 1);
|
||||
const markdown = readFileSync(path.join(root, DEFAULT_MARKDOWN_PATH), "utf8");
|
||||
assert.match(markdown, /slow\.test\.ts/);
|
||||
});
|
||||
121
scripts/__tests__/test-velocity-baseline.test.mjs
Normal file
121
scripts/__tests__/test-velocity-baseline.test.mjs
Normal file
@@ -0,0 +1,121 @@
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
readQuarantineCount,
|
||||
renderReport,
|
||||
topSlowestFiles,
|
||||
} from "../test-velocity-baseline.mjs";
|
||||
|
||||
function makeTimings(count = 25) {
|
||||
const files = {};
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
files[`packages/a/src/__tests__/case-${String(index).padStart(2, "0")}.test.ts`] = 100 + index;
|
||||
}
|
||||
files["packages/a/src/__tests__/tie-z.test.ts"] = 500;
|
||||
files["packages/a/src/__tests__/tie-a.test.ts"] = 500;
|
||||
return {
|
||||
packages: {
|
||||
"@pkg/b": {
|
||||
files: {
|
||||
"packages/b/src/__tests__/winner.test.ts": 1000,
|
||||
},
|
||||
},
|
||||
"@pkg/a": { files },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("topSlowestFiles", () => {
|
||||
it("returns exactly 20 rows in descending duration order with package attribution and stable ties", () => {
|
||||
const rows = topSlowestFiles(makeTimings(), 20);
|
||||
|
||||
assert.equal(rows.length, 20);
|
||||
assert.deepEqual(rows[0], {
|
||||
file: "packages/b/src/__tests__/winner.test.ts",
|
||||
ms: 1000,
|
||||
package: "@pkg/b",
|
||||
});
|
||||
assert.deepEqual(rows.slice(1, 3).map((row) => row.file), [
|
||||
"packages/a/src/__tests__/tie-a.test.ts",
|
||||
"packages/a/src/__tests__/tie-z.test.ts",
|
||||
]);
|
||||
assert.ok(rows.every((row, index) => index === 0 || rows[index - 1].ms >= row.ms));
|
||||
assert.equal(rows[1].package, "@pkg/a");
|
||||
});
|
||||
});
|
||||
|
||||
describe("readQuarantineCount", () => {
|
||||
it("counts entries by age bucket and flags deletion-due quarantines after 14 days", () => {
|
||||
const result = readQuarantineCount(
|
||||
{
|
||||
entries: [
|
||||
{ file: "fresh.test.ts", quarantinedAt: "2026-06-15" },
|
||||
{ file: "warning.test.ts", quarantinedAt: "2026-06-08" },
|
||||
{ file: "due.test.ts", quarantinedAt: "2026-06-01" },
|
||||
{ file: "unknown.test.ts", quarantinedAt: "not-a-date" },
|
||||
],
|
||||
},
|
||||
{ now: new Date("2026-06-17T12:00:00.000Z") },
|
||||
);
|
||||
|
||||
assert.equal(result.total, 4);
|
||||
assert.deepEqual(result.byAgeBucket, {
|
||||
"0-6d": 1,
|
||||
"7-13d": 1,
|
||||
deletionDue: 1,
|
||||
unknown: 1,
|
||||
});
|
||||
assert.deepEqual(result.deletionDueEntries, [
|
||||
{ file: "due.test.ts", quarantinedAt: "2026-06-01", ageDays: 16 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderReport", () => {
|
||||
it("includes metrics, slowest rows, quarantine count, and previous-run deltas", () => {
|
||||
const report = renderReport({
|
||||
gateMs: 12_000,
|
||||
bootSmokeMs: 2_000,
|
||||
testMs: 45_000,
|
||||
capturedAt: "2026-06-17T12:00:00.000Z",
|
||||
previous: {
|
||||
capturedAt: "2026-06-10T12:00:00.000Z",
|
||||
gateMs: 10_000,
|
||||
bootSmokeMs: 3_000,
|
||||
testMs: 50_000,
|
||||
quarantineCount: 3,
|
||||
},
|
||||
slowest: [
|
||||
{ file: "packages/a/src/__tests__/slow.test.ts", package: "@pkg/a", ms: 3210 },
|
||||
],
|
||||
quarantine: {
|
||||
total: 2,
|
||||
byAgeBucket: { "0-6d": 1, "7-13d": 1, deletionDue: 0, unknown: 0 },
|
||||
deletionDueEntries: [],
|
||||
deletionDueCount: 0,
|
||||
},
|
||||
});
|
||||
|
||||
assert.match(report, /\| Merge gate wall-time \(`pnpm test:gate`\) \| 12\.0s \| \+2\.0s \|/);
|
||||
assert.match(report, /\| Boot smoke wall-time \(`pnpm smoke:boot`\) \| 2\.0s \| -1\.0s \|/);
|
||||
assert.match(report, /\| Changed-only test wall-time \(`pnpm test`\) \| 45\.0s \| -5\.0s \|/);
|
||||
assert.match(report, /\| Quarantine \/ flake count \| 2 \| -1 \|/);
|
||||
assert.match(report, /`packages\/a\/src\/__tests__\/slow\.test\.ts` \| @pkg\/a \| 3\.2s/);
|
||||
assert.match(report, /FN-6612 weekly test velocity: gate 12\.0s \(\+2\.0s\)/);
|
||||
});
|
||||
|
||||
it("renders seed-baseline trend placeholders when there is no previous entry", () => {
|
||||
const report = renderReport({
|
||||
gateMs: 1_000,
|
||||
bootSmokeMs: null,
|
||||
testMs: 2_000,
|
||||
capturedAt: "2026-06-17T12:00:00.000Z",
|
||||
slowest: [],
|
||||
quarantine: { total: 0, byAgeBucket: {}, deletionDueEntries: [], deletionDueCount: 0 },
|
||||
});
|
||||
|
||||
assert.match(report, /\| Previous \| _\(seed baseline\)_ \| — \| — \| — \| — \|/);
|
||||
assert.match(report, /\| Delta \| — \| n\/a \| n\/a \| n\/a \| n\/a \|/);
|
||||
});
|
||||
});
|
||||
138
scripts/__tests__/workflow-reliability-release-check.test.mjs
Normal file
138
scripts/__tests__/workflow-reliability-release-check.test.mjs
Normal file
@@ -0,0 +1,138 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import path from "node:path";
|
||||
import { existsSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
DEFAULT_MANIFEST_PATH,
|
||||
loadManifest,
|
||||
planCommands,
|
||||
renderReport,
|
||||
summarize,
|
||||
validateManifest,
|
||||
} from "../workflow-reliability-release-check.mjs";
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(currentFilePath), "../..");
|
||||
|
||||
const expectedChecklistIds = [
|
||||
"author-import-save-discover-reload",
|
||||
"selected-workflow-execution-fail-closed",
|
||||
"gate-advisory-readonly-revise-required-artifact",
|
||||
"automerge-hard-cancel-file-scope-recovery",
|
||||
"restart-selection-progress-run-audit",
|
||||
];
|
||||
|
||||
function clone(value) {
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
}
|
||||
|
||||
test("manifest parses, covers all five release-check items, and references existing seam files", () => {
|
||||
const manifest = loadManifest(DEFAULT_MANIFEST_PATH, { rootDir: repoRoot });
|
||||
|
||||
assert.equal(manifest.version, 1);
|
||||
assert.equal(typeof manifest._comment, "string");
|
||||
assert.deepEqual(manifest.checklist.map((item) => item.id), expectedChecklistIds);
|
||||
assert.equal(manifest.manual.length, 0);
|
||||
|
||||
for (const item of manifest.checklist) {
|
||||
assert.equal(typeof item.title, "string");
|
||||
assert.equal(typeof item.journey, "string");
|
||||
assert.ok(item.seams.length >= 1, `${item.id} should map to at least one automated seam`);
|
||||
for (const seam of item.seams) {
|
||||
assert.match(seam.package, /^@fusion\/(core|dashboard|engine)$/);
|
||||
assert.ok(existsSync(path.join(repoRoot, seam.file)), `${seam.file} should exist`);
|
||||
}
|
||||
}
|
||||
|
||||
assert.deepEqual(validateManifest(manifest, { repoRoot, existsSync }), { ok: true, errors: [] });
|
||||
});
|
||||
|
||||
test("planned commands are targeted package-scoped vitest invocations", () => {
|
||||
const manifest = loadManifest(DEFAULT_MANIFEST_PATH, { rootDir: repoRoot });
|
||||
const commands = planCommands(manifest);
|
||||
|
||||
assert.deepEqual(commands.map((command) => command.package), ["@fusion/core", "@fusion/dashboard", "@fusion/engine"]);
|
||||
for (const command of commands) {
|
||||
assert.equal(command.command, "pnpm");
|
||||
assert.deepEqual(command.args.slice(0, 5), ["--filter", command.package, "exec", "vitest", "run"]);
|
||||
assert.ok(command.args.includes("--silent=passed-only"));
|
||||
assert.ok(command.args.includes("--reporter=dot"));
|
||||
assert.ok(command.files.length >= 1);
|
||||
for (const file of command.files) assert.ok(file.startsWith("packages/"));
|
||||
}
|
||||
});
|
||||
|
||||
test("validateManifest fails closed on dangling seam files and uncovered checklist items", () => {
|
||||
const manifest = loadManifest(DEFAULT_MANIFEST_PATH, { rootDir: repoRoot });
|
||||
const dangling = clone(manifest);
|
||||
dangling.checklist[0].seams[0].file = "packages/core/src/__tests__/missing-workflow-release-check.test.ts";
|
||||
|
||||
const danglingResult = validateManifest(dangling, { repoRoot, existsSync: (candidate) => !String(candidate).includes("missing-workflow-release-check") });
|
||||
assert.equal(danglingResult.ok, false);
|
||||
assert.match(danglingResult.errors.join("\n"), /file does not exist/);
|
||||
|
||||
const uncovered = clone(manifest);
|
||||
uncovered.checklist[1].seams = [];
|
||||
const uncoveredResult = validateManifest(uncovered, { repoRoot, existsSync: () => true });
|
||||
assert.equal(uncoveredResult.ok, false);
|
||||
assert.match(uncoveredResult.errors.join("\n"), /must have at least one seam or a manual automationDeferredReason/);
|
||||
});
|
||||
|
||||
test("validateManifest rejects unknown packages and accepts explicit manual deferrals", () => {
|
||||
const manifest = loadManifest(DEFAULT_MANIFEST_PATH, { rootDir: repoRoot });
|
||||
const unknownPackage = clone(manifest);
|
||||
unknownPackage.checklist[0].seams[0].package = "@fusion/unknown";
|
||||
|
||||
const unknownPackageResult = validateManifest(unknownPackage, { repoRoot, existsSync: () => true });
|
||||
assert.equal(unknownPackageResult.ok, false);
|
||||
assert.match(unknownPackageResult.errors.join("\n"), /unknown package/);
|
||||
|
||||
const manual = clone(manifest);
|
||||
manual.checklist[0].seams = [];
|
||||
manual.manual = [{
|
||||
id: manual.checklist[0].id,
|
||||
title: manual.checklist[0].title,
|
||||
automationDeferredReason: "Requires a human-only external signoff artifact.",
|
||||
}];
|
||||
assert.deepEqual(validateManifest(manual, { repoRoot, existsSync: () => true }), { ok: true, errors: [] });
|
||||
});
|
||||
|
||||
test("summarize and renderReport roll up pass, fail, manual, text, and stable JSON", () => {
|
||||
const summary = summarize([
|
||||
{
|
||||
id: "passed-item",
|
||||
title: "Passed item",
|
||||
journey: "A passing synthetic item",
|
||||
seams: [{ package: "@fusion/core", file: "one.test.ts", status: "PASS" }],
|
||||
manual: null,
|
||||
},
|
||||
{
|
||||
id: "failed-item",
|
||||
title: "Failed item",
|
||||
journey: "A failing synthetic item",
|
||||
seams: [{ package: "@fusion/engine", file: "two.test.ts", status: "FAIL", exitCode: 1 }],
|
||||
manual: null,
|
||||
},
|
||||
{
|
||||
id: "manual-item",
|
||||
title: "Manual item",
|
||||
journey: "A manual synthetic item",
|
||||
seams: [],
|
||||
manual: { id: "manual-item", title: "Manual item", automationDeferredReason: "Human inspection only." },
|
||||
},
|
||||
]);
|
||||
|
||||
assert.equal(summary.ok, false);
|
||||
assert.deepEqual(summary.counts, { pass: 1, fail: 1, manual: 1, total: 3 });
|
||||
assert.deepEqual(summary.items.map((item) => item.status), ["PASS", "FAIL", "MANUAL"]);
|
||||
|
||||
const text = renderReport(summary);
|
||||
assert.match(text, /Overall: FAIL \(1 passed, 1 failed, 1 manual, 3 total\)/);
|
||||
assert.match(text, /FAIL: failed-item/);
|
||||
assert.match(text, /MANUAL: Human inspection only\./);
|
||||
|
||||
const json = renderReport(summary, { json: true });
|
||||
assert.deepEqual(JSON.parse(json), summary);
|
||||
assert.ok(json.startsWith('{\n "ok": false,'));
|
||||
});
|
||||
119
scripts/check-no-test-timeout-appeasement.mjs
Executable file
119
scripts/check-no-test-timeout-appeasement.mjs
Executable file
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
FNXC:TestHygiene 2026-06-14-03:15:
|
||||
Repo policy forbids hiding slow or flaky Vitest suites with file-level or suite-level timeout bumps.
|
||||
This guard blocks new `testTimeout` and `hookTimeout` appeasement in tracked test files, while a dated allowlist records temporary legacy exemptions that must link to the owning cleanup or quarantine work.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const ALLOWLIST_PATH = "scripts/lib/test-timeout-appeasement-allowlist.json";
|
||||
const SCAN_ROOTS = ["packages", "plugins"];
|
||||
const TEST_FILE_PATTERN = /\.test\.(?:ts|tsx|mts|cts|mjs|cjs|js|jsx)$/;
|
||||
const VITEST_CONFIG_PATTERN = /(?:^|\/)vitest\.config\.[mc]?[jt]s$/;
|
||||
const TIMEOUT_PROPERTY_PATTERN = /\b(?:testTimeout|hookTimeout)\s*:/;
|
||||
|
||||
function isTestFile(filePath) {
|
||||
return TEST_FILE_PATTERN.test(filePath) && !VITEST_CONFIG_PATTERN.test(filePath);
|
||||
}
|
||||
|
||||
function listTrackedTargets() {
|
||||
const result = spawnSync("git", ["ls-files", "--", ...SCAN_ROOTS], {
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(result.stderr?.trim() || "git ls-files failed");
|
||||
}
|
||||
return result.stdout
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.filter(isTestFile);
|
||||
}
|
||||
|
||||
function loadAllowlistEntries(allowlistPath = ALLOWLIST_PATH) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(allowlistPath, "utf8"));
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to read ${allowlistPath}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
|
||||
if (!Array.isArray(parsed.entries)) {
|
||||
throw new Error(`${allowlistPath} must contain an entries array`);
|
||||
}
|
||||
|
||||
return parsed.entries;
|
||||
}
|
||||
|
||||
function buildAllowlistedFiles(entries) {
|
||||
const files = new Set();
|
||||
for (const [index, entry] of entries.entries()) {
|
||||
if (!entry || typeof entry.file !== "string" || entry.file.trim() === "") {
|
||||
throw new Error(`${ALLOWLIST_PATH} entries[${index}] must include a non-empty file`);
|
||||
}
|
||||
if (typeof entry.reason !== "string" || entry.reason.trim() === "") {
|
||||
throw new Error(`${ALLOWLIST_PATH} entries[${index}] for ${entry.file} must include a non-empty reason`);
|
||||
}
|
||||
files.add(entry.file);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
export function scanFileContent(content, filePath, options = {}) {
|
||||
if (!isTestFile(filePath)) return [];
|
||||
|
||||
const allowlistedFiles = options.allowlistedFiles ?? buildAllowlistedFiles(options.allowlistEntries ?? []);
|
||||
if (allowlistedFiles.has(filePath)) return [];
|
||||
|
||||
const matches = [];
|
||||
const lines = content.split(/\r?\n/);
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const line = lines[index];
|
||||
if (TIMEOUT_PROPERTY_PATTERN.test(line)) {
|
||||
matches.push({ filePath, lineNumber: index + 1, line });
|
||||
}
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
export function scanTrackedFiles(files = listTrackedTargets(), options = {}) {
|
||||
const allowlistedFiles = options.allowlistedFiles ?? buildAllowlistedFiles(options.allowlistEntries ?? loadAllowlistEntries());
|
||||
const matches = [];
|
||||
for (const filePath of files) {
|
||||
if (!isTestFile(filePath) || allowlistedFiles.has(filePath)) continue;
|
||||
let content;
|
||||
try {
|
||||
content = readFileSync(filePath, "utf8");
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
matches.push(...scanFileContent(content, filePath, { allowlistedFiles }));
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
export function formatFailureMessage(matches) {
|
||||
const lines = matches.map(
|
||||
({ filePath, lineNumber, line }) => `${filePath}:${lineNumber}: ${line.trim()}`,
|
||||
);
|
||||
return [
|
||||
"[check-no-test-timeout-appeasement] found Vitest timeout appeasement in tracked test files.",
|
||||
"Do not raise per-file/suite timeouts to mask slow/flaky tests — quarantine via `scripts/lib/test-quarantine.json` or narrow the seam; see AGENTS.md 'Do Not Add Slow Tests'.",
|
||||
`For legitimately exempt legacy cases, add a dated rationale to ${ALLOWLIST_PATH}; exemptions are temporary and should point at the owning cleanup task.`,
|
||||
...lines,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function main() {
|
||||
const matches = scanTrackedFiles();
|
||||
if (matches.length === 0) return 0;
|
||||
console.error(formatFailureMessage(matches));
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
process.exitCode = main();
|
||||
}
|
||||
@@ -27,10 +27,10 @@
|
||||
* --dashboard-curated
|
||||
* Assert that every `*.test.{ts,tsx}` file under packages/dashboard/app
|
||||
* and packages/dashboard/src is included by at least one *executed*
|
||||
* dashboard quality project, OR listed on the explicit skip-list with a
|
||||
* non-empty reason. Fails (exit 1) otherwise. This closes the curated-gate
|
||||
* coverage hole: a new dashboard test file that nobody registered trips
|
||||
* this guard.
|
||||
* dashboard quality project, OR listed on the explicit skip-list / dated
|
||||
* quarantine ledger with a non-empty reason. Fails (exit 1) otherwise.
|
||||
* This closes the curated-gate coverage hole: a new dashboard test file
|
||||
* that nobody registered trips this guard.
|
||||
*
|
||||
* The capture spec (which packages/projects to enumerate) is data, not code:
|
||||
* it lives in scripts/lib/test-inventory-spec.json so the CI shard planner and
|
||||
@@ -48,9 +48,10 @@ const REPO_ROOT = resolve(__dirname, "..");
|
||||
|
||||
const DEFAULT_SPEC_PATH = join(__dirname, "lib", "test-inventory-spec.json");
|
||||
const DASHBOARD_SKIPLIST_PATH = join(__dirname, "lib", "dashboard-curated-skiplist.json");
|
||||
const TEST_QUARANTINE_PATH = join(__dirname, "lib", "test-quarantine.json");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Spec + skip-list loading
|
||||
// Spec + skip-list / quarantine loading
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function loadSpec(specPathOverride) {
|
||||
@@ -73,6 +74,17 @@ function loadSkipList(skipListPathOverride) {
|
||||
return { skipListPath, entries: raw.entries };
|
||||
}
|
||||
|
||||
function loadQuarantineList(quarantinePathOverride) {
|
||||
const quarantinePath =
|
||||
quarantinePathOverride || process.env.FUSION_TEST_QUARANTINE || TEST_QUARANTINE_PATH;
|
||||
if (!existsSync(quarantinePath)) return { quarantinePath, entries: [] };
|
||||
const raw = JSON.parse(readFileSync(quarantinePath, "utf8"));
|
||||
if (!Array.isArray(raw.entries)) {
|
||||
throw new Error(`quarantine ledger ${quarantinePath} must have an "entries" array`);
|
||||
}
|
||||
return { quarantinePath, entries: raw.entries };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// vitest list invocation
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -201,9 +213,10 @@ function walkTestFiles(rootDir, repoRoot) {
|
||||
* @param {Set<string>} opts.includedFiles repo-relative files executed by quality projects
|
||||
* @param {string[]} opts.allTestFiles repo-relative dashboard app/src test files
|
||||
* @param {Array<{file:string,reason:string}>} opts.skipList
|
||||
* @param {Array<{file:string,reason:string,quarantinedAt?:string}>} [opts.quarantineList]
|
||||
* @returns {{ ok: boolean, errors: string[] }}
|
||||
*/
|
||||
export function validateDashboardCurated({ includedFiles, allTestFiles, skipList }) {
|
||||
export function validateDashboardCurated({ includedFiles, allTestFiles, skipList, quarantineList = [] }) {
|
||||
const errors = [];
|
||||
const skipByFile = new Map();
|
||||
for (const entry of skipList) {
|
||||
@@ -214,21 +227,48 @@ export function validateDashboardCurated({ includedFiles, allTestFiles, skipList
|
||||
if (typeof entry.reason !== "string" || entry.reason.trim().length === 0) {
|
||||
errors.push(`skip-list entry for ${entry.file} has an empty "reason"`);
|
||||
}
|
||||
/*
|
||||
FNXC:DashboardTesting 2026-06-14-10:27:
|
||||
FN-6445 requires the curated skip-list to enumerate only dashboard tests no quality project executes. FN-6442 found useChatRooms.test.ts was both skip-listed and matched by the hooks/utils quality lane, which overstated the genuinely ungated orphan count; reject that overlap at validation time.
|
||||
*/
|
||||
if (includedFiles.has(entry.file)) {
|
||||
errors.push(
|
||||
`skip-list entry for ${entry.file} overlaps a file already executed by a quality project; the skip-list is for genuinely non-executed files only — remove this entry`,
|
||||
);
|
||||
}
|
||||
skipByFile.set(entry.file, entry);
|
||||
}
|
||||
|
||||
// A skip-listed file that is actually covered is allowed but noisy; we don't
|
||||
// error on it (it keeps the guard green while a flaky file is being fixed).
|
||||
const quarantineByFile = new Map();
|
||||
for (const entry of quarantineList) {
|
||||
if (!entry || typeof entry.file !== "string" || entry.file.length === 0) {
|
||||
errors.push(`quarantine entry missing "file": ${JSON.stringify(entry)}`);
|
||||
continue;
|
||||
}
|
||||
if (typeof entry.reason !== "string" || entry.reason.trim().length === 0) {
|
||||
errors.push(`quarantine entry for ${entry.file} has an empty "reason"`);
|
||||
}
|
||||
if (typeof entry.quarantinedAt !== "string" || entry.quarantinedAt.trim().length === 0) {
|
||||
errors.push(`quarantine entry for ${entry.file} has an empty "quarantinedAt"`);
|
||||
}
|
||||
quarantineByFile.set(entry.file, entry);
|
||||
}
|
||||
|
||||
/*
|
||||
FNXC:DashboardTesting 2026-06-14-08:42:
|
||||
A quarantined dashboard test is intentionally not executed by quality projects, but it must not be re-added to the curated skip-list. Treat the dated quarantine ledger as a second explicit registration source so rescued tests can leave the skip-list while failing tests remain governed by the deletion ratchet.
|
||||
*/
|
||||
for (const file of allTestFiles) {
|
||||
if (includedFiles.has(file)) continue;
|
||||
if (skipByFile.has(file)) continue;
|
||||
if (quarantineByFile.has(file)) continue;
|
||||
errors.push(
|
||||
`dashboard test file is not executed by any quality project and is not skip-listed: ${file}`,
|
||||
`dashboard test file is not executed by any quality project and is not skip-listed or quarantined: ${file}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Stale skip-list entries pointing at deleted files are a soft error so the
|
||||
// list doesn't rot, but only when the file genuinely no longer exists.
|
||||
// Stale explicit registrations pointing at deleted files are a soft error so
|
||||
// the lists don't rot, but only when the file genuinely no longer exists.
|
||||
for (const entry of skipList) {
|
||||
if (!entry || typeof entry.file !== "string") continue;
|
||||
if (!allTestFiles.includes(entry.file) && !includedFiles.has(entry.file)) {
|
||||
@@ -238,6 +278,15 @@ export function validateDashboardCurated({ includedFiles, allTestFiles, skipList
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const entry of quarantineList) {
|
||||
if (!entry || typeof entry.file !== "string") continue;
|
||||
if (!allTestFiles.includes(entry.file) && !includedFiles.has(entry.file)) {
|
||||
const abs = join(REPO_ROOT, entry.file);
|
||||
if (!existsSync(abs)) {
|
||||
errors.push(`quarantine ledger references a non-existent file: ${entry.file}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { ok: errors.length === 0, errors };
|
||||
}
|
||||
@@ -321,7 +370,8 @@ async function main() {
|
||||
].sort();
|
||||
const includedFiles = listExecutedDashboardQualityFiles();
|
||||
const { entries: skipList } = loadSkipList();
|
||||
const { ok, errors } = validateDashboardCurated({ includedFiles, allTestFiles, skipList });
|
||||
const { entries: quarantineList } = loadQuarantineList();
|
||||
const { ok, errors } = validateDashboardCurated({ includedFiles, allTestFiles, skipList, quarantineList });
|
||||
if (!ok) {
|
||||
console.error(`✗ dashboard curated-gate guard failed (${errors.length} issue(s)):`);
|
||||
for (const e of errors) console.error(` - ${e}`);
|
||||
@@ -330,7 +380,7 @@ async function main() {
|
||||
console.log(
|
||||
`✓ dashboard curated gate complete: ${allTestFiles.length} test files, ${
|
||||
includedFiles.size
|
||||
} executed, ${skipList.length} skip-listed`,
|
||||
} executed, ${skipList.length} skip-listed, ${quarantineList.length} quarantined`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -61,6 +61,44 @@ function snapshotTmp() {
|
||||
return matching;
|
||||
}
|
||||
|
||||
function isProcessAlive(pid) {
|
||||
if (!Number.isInteger(pid) || pid <= 0) return false;
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
return Boolean(error && typeof error === "object" && error.code === "EPERM");
|
||||
}
|
||||
}
|
||||
|
||||
function readWorkerRootOwnerPid(rootPath) {
|
||||
try {
|
||||
const raw = readFileSync(join(rootPath, ".fusion-test-worker-root-owner"), "utf8").trim();
|
||||
const pid = Number.parseInt(raw.split(/\r?\n/)[0] ?? "", 10);
|
||||
return Number.isInteger(pid) && pid > 0 ? pid : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isActiveFusionTestWorkerRoot(entry) {
|
||||
if (!entry.name.startsWith("fusion-test-workers-")) return false;
|
||||
const rootPath = join(tmpdir(), entry.name);
|
||||
const ownerPid = readWorkerRootOwnerPid(rootPath);
|
||||
if (ownerPid !== null && isProcessAlive(ownerPid)) return true;
|
||||
|
||||
try {
|
||||
for (const child of readdirSync(rootPath, { withFileTypes: true })) {
|
||||
if (!child.isDirectory()) continue;
|
||||
const match = /^redir-(\d+)$/.exec(child.name);
|
||||
if (match && isProcessAlive(Number.parseInt(match[1], 10))) return true;
|
||||
}
|
||||
} catch {
|
||||
// Ignore transient removal while the worker root is being cleaned up.
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function listProtectedFusionDirs() {
|
||||
const dirs = new Set();
|
||||
dirs.add(stablePath(join(process.cwd(), ".fusion")));
|
||||
@@ -278,16 +316,34 @@ function checkAgainstBaseline() {
|
||||
.map((name) => name.trim())
|
||||
.filter(Boolean);
|
||||
for (const name of callerIgnoreNames) baselineNames.add(name);
|
||||
const leaks = snapshotTmp().filter((e) => {
|
||||
let leaks = snapshotTmp().filter((e) => {
|
||||
if (baselineNames.has(e.name)) {
|
||||
return false;
|
||||
}
|
||||
if (e.name.startsWith("fusion-test-home-root-")) {
|
||||
return false;
|
||||
}
|
||||
/*
|
||||
FNXC:TestIsolation 2026-06-14-01:20:
|
||||
Local verification can run beside another Vitest invocation from a sibling worktree.
|
||||
A fusion-test-workers-* root created after this run's baseline is not this run's leak when its owner marker or redirect sink points at a live process, so skip only those active worker roots while still failing stale worker-root leaks.
|
||||
*/
|
||||
if (isActiveFusionTestWorkerRoot(e)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
// Vitest/Node worker roots can disappear a moment after the child process
|
||||
// exits on macOS. Re-check candidate leaks after a short settle window so
|
||||
// the guard still fails durable leaks while avoiding false failures for
|
||||
// already-cleaned transient worker directories.
|
||||
if (leaks.length > 0) {
|
||||
sleepMs(500);
|
||||
const settledNames = new Set(snapshotTmp().map((e) => e.name));
|
||||
leaks = leaks.filter((e) => settledNames.has(e.name));
|
||||
}
|
||||
|
||||
const baselineByDir = new Map((baseline.protectedFusion ?? []).map((entry) => [entry.dir, entry]));
|
||||
const unstableProtectedDirs = new Set(baseline.unstableProtectedDirs ?? []);
|
||||
const currentProtected = snapshotProtectedFusion();
|
||||
@@ -381,4 +437,4 @@ if (args.includes("--before-fast")) {
|
||||
recordBaseline();
|
||||
} else {
|
||||
checkAgainstBaseline();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,14 +13,17 @@
|
||||
* keeping slices of the same package on different shards whenever possible.
|
||||
*/
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { globSync, readFileSync, writeFileSync, readdirSync, mkdirSync, renameSync } from "node:fs";
|
||||
import { cpus } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs";
|
||||
import { listWorkspacePackageInfos } from "./test-changed.mjs";
|
||||
import { deriveBudgetMs, runWithWatchdog } from "./lib/run-vitest-watchdog.mjs";
|
||||
|
||||
// Quick, non-test commands (e.g. skill-sync check) stay synchronous — they have
|
||||
// no hang risk and no benefit from the watchdog.
|
||||
function run(command, commandArgs, options = {}) {
|
||||
const result = spawnSync(command, commandArgs, {
|
||||
cwd: process.cwd(),
|
||||
@@ -33,6 +36,32 @@ function run(command, commandArgs, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test invocations run under the L2 wall-clock watchdog so a wedged vitest run
|
||||
// is SIGTERM/SIGKILLed at its budget instead of blocking to the CI job ceiling.
|
||||
// Preserves the fail-fast contract of `run` (exit non-zero on failure/timeout).
|
||||
async function runWatched(command, commandArgs, { env, budgetMs, label } = {}) {
|
||||
const { code, signal, timedOut } = await runWithWatchdog({
|
||||
command,
|
||||
args: commandArgs,
|
||||
env: env ?? process.env,
|
||||
budgetMs,
|
||||
label: label ?? command,
|
||||
log: console.error,
|
||||
spawn,
|
||||
});
|
||||
if (timedOut) {
|
||||
console.error(`[ci-test-shard] FAILED (timeout): ${label ?? command}`);
|
||||
process.exit(124);
|
||||
}
|
||||
if (signal) {
|
||||
console.error(`[ci-test-shard] FAILED (signal ${signal}): ${label ?? command}`);
|
||||
process.exit(1);
|
||||
}
|
||||
if (code !== 0) {
|
||||
process.exit(code ?? 1);
|
||||
}
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value) {
|
||||
const parsed = Number.parseInt(value ?? "", 10);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
@@ -1157,6 +1186,10 @@ export function buildShardCommands(shardEntries, options = {}) {
|
||||
commands.push({
|
||||
kind: "plain",
|
||||
label: plain.map((e) => e.name).join(", "),
|
||||
// A single plain command fans out across every packed package, so its
|
||||
// expected duration is the SUM of their weights — not a per-package value
|
||||
// (see the watchdog budget aggregation, KTD-2).
|
||||
weightMs: plain.reduce((sum, e) => sum + (e.weight ?? 0), 0),
|
||||
args: [...filters, "test", ...timingFlags()],
|
||||
});
|
||||
}
|
||||
@@ -1165,6 +1198,7 @@ export function buildShardCommands(shardEntries, options = {}) {
|
||||
commands.push({
|
||||
kind: "virtual",
|
||||
label: `${entry.name} [${entry.shardIndex}/${entry.shardCount}]`,
|
||||
weightMs: entry.weight ?? 0,
|
||||
// NB: no `--` between `test` and `--shard`; cac would treat the value as a
|
||||
// positional file filter and silently disable sharding.
|
||||
args: ["--filter", entry.name, "test", `--shard=${entry.shardIndex}/${entry.shardCount}`, ...timingFlags()],
|
||||
@@ -1175,6 +1209,7 @@ export function buildShardCommands(shardEntries, options = {}) {
|
||||
commands.push({
|
||||
kind: "dashboard-lane",
|
||||
label: `${entry.name} run ${entry.lane}`,
|
||||
weightMs: entry.weight ?? 0,
|
||||
args: ["--filter", entry.name, "run", entry.lane, ...timingFlags()],
|
||||
});
|
||||
}
|
||||
@@ -1182,7 +1217,7 @@ export function buildShardCommands(shardEntries, options = {}) {
|
||||
return commands;
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2), env = process.env) {
|
||||
export async function main(argv = process.argv.slice(2), env = process.env) {
|
||||
if (argv.includes("--write-timings")) {
|
||||
const dirIdx = argv.indexOf("--inputs-dir");
|
||||
const inputDir = dirIdx >= 0 ? argv[dirIdx + 1] : undefined;
|
||||
@@ -1264,7 +1299,11 @@ export function main(argv = process.argv.slice(2), env = process.env) {
|
||||
}
|
||||
|
||||
const { shard, total } = parseShardArgs(argv, env);
|
||||
const { units } = buildScheduleUnits();
|
||||
const { units, timings } = buildScheduleUnits();
|
||||
// Only trust timings to TIGHTEN the watchdog budget when the snapshot is
|
||||
// present and fresh; otherwise deriveBudgetMs falls back to the generous
|
||||
// per-class ceiling (KTD-2).
|
||||
const timingsFresh = Boolean(timings?.present) && !timings?.stale;
|
||||
const shardEntries = planShardAssignments(units, total)[shard - 1] || [];
|
||||
|
||||
if (shardEntries.length === 0) {
|
||||
@@ -1302,12 +1341,22 @@ export function main(argv = process.argv.slice(2), env = process.env) {
|
||||
|
||||
const commands = buildShardCommands(shardEntries, { timingFlags });
|
||||
for (const command of commands) {
|
||||
console.log(`[ci-test-shard] shard ${shard}/${total}: running ${command.label}`);
|
||||
run("pnpm", command.args, { env: shardEnv });
|
||||
const klass = command.kind === "dashboard-lane" ? "dashboard-lane" : "shard";
|
||||
const budgetMs = deriveBudgetMs({
|
||||
klass,
|
||||
expectedDurationMs: command.weightMs,
|
||||
timingsFresh,
|
||||
});
|
||||
const label = `shard ${shard}/${total}: ${command.label}`;
|
||||
console.log(`[ci-test-shard] ${label} (watchdog budget ${Math.round(budgetMs / 1000)}s)`);
|
||||
await runWatched("pnpm", command.args, { env: shardEnv, budgetMs, label });
|
||||
}
|
||||
}
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === currentFilePath) {
|
||||
main();
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -93,10 +93,30 @@ export function getPrebuildCommand(mode) {
|
||||
case "full":
|
||||
return { command: "pnpm", args: ["build"], label: "workspace build" };
|
||||
case "client":
|
||||
/*
|
||||
FNXC:DevWorkflow 2026-06-18-16:40:
|
||||
FN-6638/stale-dist: `pnpm dev dashboard` must rebuild @fusion/core and
|
||||
@fusion/engine alongside the dashboard UI, not only the client bundle.
|
||||
Although the CLI runs under `--conditions=source` (engine/core resolve to
|
||||
src), the running process and any dist-resolving consumer (plugins,
|
||||
sub-imports, a later non-dev `fn`/`pnpm local`) load built dist. Leaving
|
||||
engine/core dist stale is exactly how landed fixes (FN-6644/6647/6648,
|
||||
etc.) silently failed to run for ~2 days. pnpm builds these in dependency
|
||||
order (core → engine → dashboard); dashboard `build` runs the vite client
|
||||
bundle + server tsc, so the UI is rebuilt too.
|
||||
*/
|
||||
return {
|
||||
command: "pnpm",
|
||||
args: ["--filter", "@fusion/dashboard", "build:client"],
|
||||
label: "dashboard client build",
|
||||
args: [
|
||||
"--filter",
|
||||
"@fusion/core",
|
||||
"--filter",
|
||||
"@fusion/engine",
|
||||
"--filter",
|
||||
"@fusion/dashboard",
|
||||
"build",
|
||||
],
|
||||
label: "core + engine + dashboard build",
|
||||
};
|
||||
case "none":
|
||||
case "auto":
|
||||
|
||||
@@ -129,6 +129,25 @@ async function warnIfSourceVersionBehind() {
|
||||
|
||||
await warnIfSourceVersionBehind();
|
||||
|
||||
// FNXC:DevWorkflow 2026-06-18-16:50:
|
||||
// FN-6638 stale-dist guard. Warn (loudly, best-effort) when built dist/ is older
|
||||
// than src/ so a never-rebuilt/never-restarted process does not silently run
|
||||
// phantom-old code. When a prebuild is about to run it will refresh dist, so the
|
||||
// check is informational there; for --prebuild none / dist-resolving consumers
|
||||
// it is the safety net. Never let the check break startup.
|
||||
async function warnIfDistStale() {
|
||||
if (process.env.FUSION_SKIP_DIST_FRESHNESS_CHECK === "1") return;
|
||||
try {
|
||||
const { computeDistStaleness, formatDistStalenessWarning } = await import("./lib/dist-freshness.mjs");
|
||||
const warning = formatDistStalenessWarning(computeDistStaleness({ rootDir: process.cwd() }));
|
||||
if (warning) console.warn(warning);
|
||||
} catch {
|
||||
// Best-effort only. Startup must not depend on the freshness check.
|
||||
}
|
||||
}
|
||||
|
||||
await warnIfDistStale();
|
||||
|
||||
if (!prebuildCommand) {
|
||||
runApp(forwardedArgs);
|
||||
} else {
|
||||
|
||||
264
scripts/ios-acceptance.mjs
Executable file
264
scripts/ios-acceptance.mjs
Executable file
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* FNXC:iOSAcceptance 2026-06-18-17:02:
|
||||
* Terminal acceptance gates need a cheap run-vs-NO-OP probe and a dependency-light real-device WebDriver path. This CLI emits only structured, redacted metadata so cloud credentials can be supplied through env or Fusion secrets materialization without leaking plaintext into logs.
|
||||
*/
|
||||
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { URL } from "node:url";
|
||||
import {
|
||||
buildIosCapabilities,
|
||||
capabilityDeviceName,
|
||||
capabilityPlatformVersion,
|
||||
credentialsForProvider,
|
||||
describeAvailability,
|
||||
iosHubUrl,
|
||||
normalizeProvider,
|
||||
publicCapabilityPlan,
|
||||
redactUrl,
|
||||
} from "./lib/ios-acceptance.mjs";
|
||||
|
||||
function printJson(value) {
|
||||
console.log(JSON.stringify(value, null, 2));
|
||||
}
|
||||
|
||||
function usage() {
|
||||
return `Usage:
|
||||
node scripts/ios-acceptance.mjs --check
|
||||
node scripts/ios-acceptance.mjs --dry-run [--provider browserstack|sauce|lambdatest]
|
||||
node scripts/ios-acceptance.mjs --url <dashboardUrl> --out <screenshotPath> [--provider browserstack|sauce|lambdatest]
|
||||
|
||||
Options:
|
||||
--check Probe credential availability only; no WebDriver session.
|
||||
--url <url> Dashboard URL to open on real iOS Safari. Port 4040 is rejected.
|
||||
--out <path> Screenshot PNG path for session mode.
|
||||
--provider <name> Override provider auto-resolution.
|
||||
--dry-run Resolve credentials and print the redacted capability plan; no network.
|
||||
--help Show this help.
|
||||
`;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { check: false, dryRun: false, provider: null, url: null, out: null, help: false };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--check") {
|
||||
args.check = true;
|
||||
} else if (arg === "--dry-run") {
|
||||
args.dryRun = true;
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
args.help = true;
|
||||
} else if (arg === "--provider") {
|
||||
args.provider = argv[++index];
|
||||
} else if (arg === "--url") {
|
||||
args.url = argv[++index];
|
||||
} else if (arg === "--out") {
|
||||
args.out = argv[++index];
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function selectProvider(args, env) {
|
||||
const availability = describeAvailability(env);
|
||||
if (!args.provider) {
|
||||
return { provider: availability.provider, availability };
|
||||
}
|
||||
const provider = normalizeProvider(args.provider);
|
||||
if (!provider) {
|
||||
throw new Error(`Unsupported provider: ${args.provider}`);
|
||||
}
|
||||
const creds = credentialsForProvider(provider, env);
|
||||
return {
|
||||
provider: creds ? provider : null,
|
||||
requestedProvider: provider,
|
||||
availability: {
|
||||
...availability,
|
||||
available: Boolean(creds),
|
||||
provider: creds ? provider : null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function printMissingCredentialHint(availability) {
|
||||
console.error(`[ios:acceptance] Missing real-iOS credentials: ${availability.missing.join(", ") || "none"}`);
|
||||
console.error(
|
||||
"[ios:acceptance] If credentials are stored in Fusion, use fn_secret_get or enable env-exportable project secrets so BROWSERSTACK_*, SAUCE_*, or LT_* keys materialize into this worktree.",
|
||||
);
|
||||
console.error(
|
||||
"[ios:acceptance] NO-OP sentinel for verification gates: NO-OP: real-iOS surface unavailable — credentials missing, cannot run acceptance gate",
|
||||
);
|
||||
}
|
||||
|
||||
function assertDashboardUrl(value) {
|
||||
if (!value) {
|
||||
throw new Error("Session mode requires --url <dashboardUrl>.");
|
||||
}
|
||||
const url = new URL(value);
|
||||
if (url.port === "4040") {
|
||||
throw new Error("Port 4040 is reserved for the production dashboard; serve acceptance builds on --port 0 or another free non-4040 port.");
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function basicAuthHeader(creds) {
|
||||
return `Basic ${Buffer.from(`${creds.username}:${creds.accessKey}`).toString("base64")}`;
|
||||
}
|
||||
|
||||
function webdriverEndpoint(authenticatedHubUrl, path) {
|
||||
const url = new URL(authenticatedHubUrl);
|
||||
url.username = "";
|
||||
url.password = "";
|
||||
const basePath = url.pathname.replace(/\/+$/, "");
|
||||
const nextPath = path.replace(/^\/+/, "");
|
||||
url.pathname = `${basePath}/${nextPath}`;
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
async function webdriverFetch(authenticatedHubUrl, path, init = {}) {
|
||||
const endpoint = webdriverEndpoint(authenticatedHubUrl, path);
|
||||
const response = await fetch(endpoint, {
|
||||
...init,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...init.headers,
|
||||
},
|
||||
});
|
||||
const text = await response.text();
|
||||
let body = null;
|
||||
if (text) {
|
||||
try {
|
||||
body = JSON.parse(text);
|
||||
} catch {
|
||||
body = { raw: text.slice(0, 500) };
|
||||
}
|
||||
}
|
||||
if (!response.ok) {
|
||||
const message = body?.value?.message ?? body?.message ?? response.statusText;
|
||||
throw new Error(`WebDriver ${init.method ?? "GET"} ${path} failed (${response.status}): ${message}`);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
function sessionIdFromCreateResponse(body) {
|
||||
return body?.value?.sessionId ?? body?.sessionId ?? null;
|
||||
}
|
||||
|
||||
async function runSession({ provider, dashboardUrl, screenshotPath, env }) {
|
||||
const creds = credentialsForProvider(provider, env);
|
||||
if (!creds) {
|
||||
throw new Error(`Missing credentials for ${provider}`);
|
||||
}
|
||||
const capabilities = buildIosCapabilities(provider, { env });
|
||||
const hubUrl = iosHubUrl(provider, creds, env);
|
||||
const authHeader = basicAuthHeader(creds);
|
||||
let sessionId = null;
|
||||
try {
|
||||
const createBody = await webdriverFetch(hubUrl, "/session", {
|
||||
method: "POST",
|
||||
headers: { authorization: authHeader },
|
||||
body: JSON.stringify({ capabilities: { alwaysMatch: capabilities } }),
|
||||
});
|
||||
sessionId = sessionIdFromCreateResponse(createBody);
|
||||
if (!sessionId) {
|
||||
throw new Error("WebDriver session response did not include a sessionId.");
|
||||
}
|
||||
await webdriverFetch(hubUrl, `/session/${encodeURIComponent(sessionId)}/url`, {
|
||||
method: "POST",
|
||||
headers: { authorization: authHeader },
|
||||
body: JSON.stringify({ url: dashboardUrl }),
|
||||
});
|
||||
const screenshotBody = await webdriverFetch(hubUrl, `/session/${encodeURIComponent(sessionId)}/screenshot`, {
|
||||
method: "GET",
|
||||
headers: { authorization: authHeader },
|
||||
});
|
||||
const screenshot = screenshotBody?.value;
|
||||
if (typeof screenshot !== "string" || screenshot.length === 0) {
|
||||
throw new Error("WebDriver screenshot response did not include base64 PNG data.");
|
||||
}
|
||||
const absoluteScreenshotPath = resolve(screenshotPath);
|
||||
await mkdir(dirname(absoluteScreenshotPath), { recursive: true });
|
||||
await writeFile(absoluteScreenshotPath, Buffer.from(screenshot, "base64"));
|
||||
return {
|
||||
provider,
|
||||
device: capabilityDeviceName(provider, capabilities),
|
||||
platformVersion: capabilityPlatformVersion(provider, capabilities),
|
||||
sessionId,
|
||||
screenshotPath: absoluteScreenshotPath,
|
||||
};
|
||||
} finally {
|
||||
if (sessionId) {
|
||||
try {
|
||||
await webdriverFetch(hubUrl, `/session/${encodeURIComponent(sessionId)}`, {
|
||||
method: "DELETE",
|
||||
headers: { authorization: authHeader },
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`[ios:acceptance] Failed to delete WebDriver session ${sessionId}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (args.help) {
|
||||
console.log(usage());
|
||||
return 0;
|
||||
}
|
||||
|
||||
const { provider, requestedProvider, availability } = selectProvider(args, process.env);
|
||||
|
||||
if (args.check) {
|
||||
const result = requestedProvider
|
||||
? { ...availability, requestedProvider }
|
||||
: availability;
|
||||
printJson(result);
|
||||
if (!result.available) {
|
||||
printMissingCredentialHint(result);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!provider) {
|
||||
printJson(requestedProvider ? { ...availability, requestedProvider } : availability);
|
||||
printMissingCredentialHint(availability);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const creds = credentialsForProvider(provider, process.env);
|
||||
const hubUrl = iosHubUrl(provider, creds, process.env);
|
||||
const plan = {
|
||||
...publicCapabilityPlan(provider, { env: process.env }),
|
||||
hubUrl: redactUrl(hubUrl),
|
||||
};
|
||||
|
||||
if (args.dryRun) {
|
||||
printJson({ dryRun: true, ...plan });
|
||||
return 0;
|
||||
}
|
||||
|
||||
const dashboardUrl = assertDashboardUrl(args.url);
|
||||
if (!args.out) {
|
||||
throw new Error("Session mode requires --out <screenshotPath>.");
|
||||
}
|
||||
|
||||
console.error(`[ios:acceptance] Opening real iOS Safari via ${provider} at ${plan.hubUrl}`);
|
||||
const result = await runSession({ provider, dashboardUrl, screenshotPath: args.out, env: process.env });
|
||||
printJson(result);
|
||||
return 0;
|
||||
}
|
||||
|
||||
main()
|
||||
.then((code) => {
|
||||
process.exitCode = code;
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(`[ios:acceptance] ${error.message}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -1,137 +1,9 @@
|
||||
{
|
||||
"$comment": "Dashboard curated-gate skip-list (plan U2 / R7). Files here are NOT executed by any quality project. Every entry needs a non-empty reason. These were discovered as orphans (running in no executed project) that FAIL in isolation today, so gating them would break CI; skip-listed to keep the gate green and the failures tracked. Remove an entry once the test is fixed and add it to a backfill/quality project.",
|
||||
"$comment": "Dashboard curated-gate skip-list (plan U2 / R7). Files here are NOT executed by any quality project. Every entry needs a non-empty reason. These were discovered as orphans (running in no executed project) that FAIL in isolation today, so gating them would break CI; skip-listed to keep the gate green and the failures tracked. FNXC:DashboardTesting 2026-06-14-07:51: Placeholder tracking is not actionable; every skip-list reason must cite a concrete Fusion task ID unless the test is fixed and the entry is removed. Remove an entry once the test is fixed and add it to a backfill/quality project.",
|
||||
"entries": [
|
||||
{
|
||||
"file": "packages/dashboard/app/__tests__/build-output.test.ts",
|
||||
"reason": "asserts the built bundle; runs standalone via `pnpm --filter @fusion/dashboard test:build` (needs a prior vite build), not in the unit gate"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/ChatView.regular-composer-no-right-line.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/MissionManager.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/ModalReentry.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/ModelSelectorTab.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/NewAgentDialog.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/OAuthReloginBanner.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/PlanningModeModal.favorites.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/PlanningModeModal.questions.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/PlanningModeModal.swipe-back.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/PlanningModeModal.ui-interactions.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/SkillsView.css.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/TaskReviewTab.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/TerminalModal.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/mobile-css.test.tsx",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/hooks/__tests__/quickChatLastSessionStorage.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/hooks/__tests__/useChatRooms.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/hooks/__tests__/useTaskDiffStats.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/evals-routes.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/github-tracking-delete.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/github-tracking-periodic-reconcile-sweep.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/insights-routes.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/mission-e2e.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/planning.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-run-audit-goal-events.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-run-cited-goals.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/session-cross-tab.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/session-error-recovery.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/session-persistence-roundtrip.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/session-reconnect.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/chat-manager.test.ts",
|
||||
"reason": "pre-existing exclusion from dashboard-api-quality-backfill; tracked for rescue in FN-TBD"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/shared-branch-group-entry-points.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/usage.test.ts",
|
||||
"reason": "pre-existing failure (orphaned from curated gate, never previously executed in CI; tracked as FN-TBD)"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
110
scripts/lib/dist-freshness.mjs
Normal file
110
scripts/lib/dist-freshness.mjs
Normal file
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
FNXC:DevWorkflow 2026-06-18-16:50:
|
||||
FN-6638 stale-dist guard. The running Fusion process loads built `dist/` for
|
||||
@fusion/core, @fusion/engine, and @fusion/dashboard (directly, via plugins, via
|
||||
dist-resolving sub-imports, or whenever a non-dev/packaged `fn` runs). When a
|
||||
long-lived process or a stale build runs `dist/` that is OLDER than the `src/`
|
||||
on disk, landed fixes silently never execute — that is how FN-6644/6647/6648
|
||||
(and others) appeared "fixed" for ~2 days while the running engine still parked
|
||||
completed tasks failed. This module computes that staleness so startup can warn
|
||||
loudly (rebuild + restart) instead of running phantom-old code.
|
||||
|
||||
Design / guardrails:
|
||||
- Pure + injectable (fs + now) so it is unit-testable and never throws into the
|
||||
startup path.
|
||||
- A package is only evaluated when BOTH its `src/` and `dist/` exist. Missing
|
||||
`dist/` = running purely from source (fresh, not stale). Missing `src/` =
|
||||
packaged/published install with no source tree to compare against (not stale).
|
||||
- Staleness = newest `.ts`/`.tsx` mtime under `src/` is NEWER than the package's
|
||||
dist build marker (newest `.js` mtime under `dist/`), beyond a small slack to
|
||||
absorb filesystem mtime jitter.
|
||||
*/
|
||||
|
||||
import { existsSync, readdirSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const DEFAULT_PACKAGES = ["core", "engine", "dashboard"];
|
||||
// Slack absorbs build/checkout mtime jitter so we only flag a real source-ahead.
|
||||
const DEFAULT_SLACK_MS = 2_000;
|
||||
const SRC_EXTENSIONS = [".ts", ".tsx"];
|
||||
const DIST_EXTENSIONS = [".js"];
|
||||
// Never descend into these — they are not the package's own emitted output.
|
||||
const SKIP_DIRS = new Set(["node_modules", ".git", "__tests__", "coverage"]);
|
||||
|
||||
function newestMtimeMs(dir, extensions, fs) {
|
||||
let newest = 0;
|
||||
let stack = [dir];
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop();
|
||||
let entries;
|
||||
try {
|
||||
entries = fs.readdirSync(current, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
if (SKIP_DIRS.has(entry.name)) continue;
|
||||
stack.push(join(current, entry.name));
|
||||
continue;
|
||||
}
|
||||
if (!extensions.some((ext) => entry.name.endsWith(ext))) continue;
|
||||
try {
|
||||
const ms = fs.statSync(join(current, entry.name)).mtimeMs;
|
||||
if (ms > newest) newest = ms;
|
||||
} catch {
|
||||
// unreadable file — ignore, do not let it break the scan
|
||||
}
|
||||
}
|
||||
}
|
||||
return newest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute dist staleness for a source checkout.
|
||||
*
|
||||
* @param {object} [options]
|
||||
* @param {string} [options.rootDir] repo root (defaults to cwd)
|
||||
* @param {string[]} [options.packages] package dir names under packages/
|
||||
* @param {number} [options.slackMs] mtime slack
|
||||
* @param {object} [options.fs] fs seam ({ existsSync, readdirSync, statSync })
|
||||
* @returns {{ stale: boolean, packages: Array<{ name: string, srcNewestMs: number, distNewestMs: number, stale: boolean }> }}
|
||||
*/
|
||||
export function computeDistStaleness(options = {}) {
|
||||
const rootDir = options.rootDir ?? process.cwd();
|
||||
const packages = options.packages ?? DEFAULT_PACKAGES;
|
||||
const slackMs = options.slackMs ?? DEFAULT_SLACK_MS;
|
||||
const fs = options.fs ?? { existsSync, readdirSync, statSync };
|
||||
|
||||
const results = [];
|
||||
for (const name of packages) {
|
||||
const srcDir = join(rootDir, "packages", name, "src");
|
||||
const distDir = join(rootDir, "packages", name, "dist");
|
||||
// Both must exist: no src = packaged install; no dist = pure source run.
|
||||
if (!fs.existsSync(srcDir) || !fs.existsSync(distDir)) continue;
|
||||
const srcNewestMs = newestMtimeMs(srcDir, SRC_EXTENSIONS, fs);
|
||||
const distNewestMs = newestMtimeMs(distDir, DIST_EXTENSIONS, fs);
|
||||
if (srcNewestMs === 0 || distNewestMs === 0) continue;
|
||||
const stale = srcNewestMs - distNewestMs > slackMs;
|
||||
results.push({ name, srcNewestMs, distNewestMs, stale });
|
||||
}
|
||||
return { stale: results.some((r) => r.stale), packages: results };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the operator warning lines for a stale result (or null when fresh).
|
||||
* Kept separate from I/O so it is testable and the caller owns logging.
|
||||
*/
|
||||
export function formatDistStalenessWarning(result) {
|
||||
if (!result || !result.stale) return null;
|
||||
const staleNames = result.packages.filter((p) => p.stale).map((p) => p.name);
|
||||
return [
|
||||
"",
|
||||
`[fusion] ⚠ STALE BUILD: ${staleNames.map((n) => `@fusion/${n}`).join(", ")} dist/ is OLDER than src/.`,
|
||||
"[fusion] The running process may execute outdated compiled code, so recently landed",
|
||||
"[fusion] fixes will NOT take effect until you rebuild AND restart:",
|
||||
"[fusion] pnpm build # then restart the dashboard/engine process",
|
||||
"[fusion] (Set FUSION_SKIP_DIST_FRESHNESS_CHECK=1 to silence this check.)",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
243
scripts/lib/ios-acceptance.mjs
Normal file
243
scripts/lib/ios-acceptance.mjs
Normal file
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* FNXC:iOSAcceptance 2026-06-18-16:45:
|
||||
* Real iOS Safari is the only acceptable terminal wide-glyph gate because Playwright, desktop WebKit, jsdom, and iOS simulators did not reproduce the ASCII cell-width bug that let repeated blind fixes ship. Keep this module pure so availability probes and tests can enumerate credentials and capabilities without network access or secret logging.
|
||||
*/
|
||||
|
||||
import { URL } from "node:url";
|
||||
|
||||
export const IOS_PROVIDER_ORDER = ["browserstack", "sauce", "lambdatest"];
|
||||
|
||||
export const IOS_PROVIDER_CONFIG = {
|
||||
browserstack: {
|
||||
label: "BrowserStack",
|
||||
usernameKey: "BROWSERSTACK_USERNAME",
|
||||
accessKey: "BROWSERSTACK_ACCESS_KEY",
|
||||
hubEnvKey: "BROWSERSTACK_HUB_URL",
|
||||
defaultHubUrl: "https://hub-cloud.browserstack.com/wd/hub",
|
||||
defaultDeviceName: "iPhone 15",
|
||||
defaultPlatformVersion: "17",
|
||||
},
|
||||
sauce: {
|
||||
label: "Sauce Labs",
|
||||
usernameKey: "SAUCE_USERNAME",
|
||||
accessKey: "SAUCE_ACCESS_KEY",
|
||||
hubEnvKey: "SAUCE_HUB_URL",
|
||||
defaultHubUrl: "https://ondemand.us-west-1.saucelabs.com/wd/hub",
|
||||
defaultDeviceName: "iPhone 15",
|
||||
defaultPlatformVersion: "17",
|
||||
},
|
||||
lambdatest: {
|
||||
label: "LambdaTest",
|
||||
usernameKey: "LT_USERNAME",
|
||||
accessKey: "LT_ACCESS_KEY",
|
||||
hubEnvKey: "LT_HUB_URL",
|
||||
defaultHubUrl: "https://mobile-hub.lambdatest.com/wd/hub",
|
||||
defaultDeviceName: "iPhone 15",
|
||||
defaultPlatformVersion: "17",
|
||||
},
|
||||
};
|
||||
|
||||
export function normalizeProvider(provider) {
|
||||
const normalized = String(provider ?? "").trim().toLowerCase();
|
||||
if (normalized === "lt" || normalized === "lambda-test" || normalized === "lambda_test") {
|
||||
return "lambdatest";
|
||||
}
|
||||
if (normalized === "browser-stack" || normalized === "browser_stack") {
|
||||
return "browserstack";
|
||||
}
|
||||
if (IOS_PROVIDER_CONFIG[normalized]) {
|
||||
return normalized;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function nonEmpty(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
export function checkedCredentialKeys() {
|
||||
return IOS_PROVIDER_ORDER.flatMap((provider) => {
|
||||
const config = IOS_PROVIDER_CONFIG[provider];
|
||||
return [config.usernameKey, config.accessKey];
|
||||
});
|
||||
}
|
||||
|
||||
export function credentialsForProvider(provider, env = {}) {
|
||||
const normalized = normalizeProvider(provider);
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
const config = IOS_PROVIDER_CONFIG[normalized];
|
||||
const username = env[config.usernameKey];
|
||||
const accessKey = env[config.accessKey];
|
||||
if (!nonEmpty(username) || !nonEmpty(accessKey)) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
username: username.trim(),
|
||||
accessKey: accessKey.trim(),
|
||||
usernameKey: config.usernameKey,
|
||||
accessKeyName: config.accessKey,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveIosProvider(env = {}) {
|
||||
for (const provider of IOS_PROVIDER_ORDER) {
|
||||
if (credentialsForProvider(provider, env)) {
|
||||
return provider;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function describeAvailability(env = {}) {
|
||||
const checkedKeys = checkedCredentialKeys();
|
||||
const provider = resolveIosProvider(env);
|
||||
return {
|
||||
available: provider !== null,
|
||||
provider,
|
||||
checkedKeys,
|
||||
missing: checkedKeys.filter((key) => !nonEmpty(env[key])),
|
||||
};
|
||||
}
|
||||
|
||||
export function iosHubUrl(provider, creds, env = {}) {
|
||||
const normalized = normalizeProvider(provider);
|
||||
if (!normalized) {
|
||||
throw new Error(`Unsupported iOS provider: ${provider}`);
|
||||
}
|
||||
const config = IOS_PROVIDER_CONFIG[normalized];
|
||||
const resolvedCreds = creds ?? credentialsForProvider(normalized, env);
|
||||
if (!resolvedCreds || !nonEmpty(resolvedCreds.username) || !nonEmpty(resolvedCreds.accessKey)) {
|
||||
throw new Error(`Missing credentials for ${normalized}`);
|
||||
}
|
||||
const base = nonEmpty(env[config.hubEnvKey]) ? env[config.hubEnvKey].trim() : config.defaultHubUrl;
|
||||
const url = new URL(base);
|
||||
url.username = resolvedCreds.username.trim();
|
||||
url.password = resolvedCreds.accessKey.trim();
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export function redactSecretValue(value) {
|
||||
return nonEmpty(value) ? "<redacted>" : value;
|
||||
}
|
||||
|
||||
export function redactUrl(value) {
|
||||
if (!nonEmpty(value)) {
|
||||
return value;
|
||||
}
|
||||
try {
|
||||
const url = new URL(value);
|
||||
if (!url.username && !url.password) {
|
||||
return url.toString();
|
||||
}
|
||||
return `${url.protocol}//<redacted>:<redacted>@${url.host}${url.pathname}${url.search}${url.hash}`;
|
||||
} catch {
|
||||
return String(value).replace(/\/\/([^:@/\s]+):([^@/\s]+)@/g, "//<redacted>:<redacted>@");
|
||||
}
|
||||
}
|
||||
|
||||
function resolveCapabilityOption(opts, env, provider, optionName, envSuffix, fallbackName) {
|
||||
const config = IOS_PROVIDER_CONFIG[provider];
|
||||
const providerPrefix = provider === "browserstack" ? "BROWSERSTACK" : provider === "sauce" ? "SAUCE" : "LT";
|
||||
const envKey = `${providerPrefix}_${envSuffix}`;
|
||||
return opts[optionName] ?? env?.[envKey] ?? config[fallbackName];
|
||||
}
|
||||
|
||||
export function buildIosCapabilities(provider, opts = {}) {
|
||||
const normalized = normalizeProvider(provider);
|
||||
if (!normalized) {
|
||||
throw new Error(`Unsupported iOS provider: ${provider}`);
|
||||
}
|
||||
const env = opts.env ?? {};
|
||||
const deviceName = String(
|
||||
resolveCapabilityOption(opts, env, normalized, "deviceName", "IOS_DEVICE", "defaultDeviceName"),
|
||||
).trim();
|
||||
const platformVersion = String(
|
||||
resolveCapabilityOption(opts, env, normalized, "platformVersion", "IOS_VERSION", "defaultPlatformVersion"),
|
||||
).trim();
|
||||
const sessionName = String(opts.name ?? "Fusion real-iOS Safari acceptance").trim();
|
||||
const buildName = String(opts.build ?? "FN-6667 ios-acceptance").trim();
|
||||
|
||||
if (normalized === "browserstack") {
|
||||
return {
|
||||
browserName: "safari",
|
||||
platformName: "iOS",
|
||||
"bstack:options": {
|
||||
deviceName,
|
||||
osVersion: platformVersion,
|
||||
realMobile: true,
|
||||
projectName: "Fusion",
|
||||
buildName,
|
||||
sessionName,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (normalized === "sauce") {
|
||||
return {
|
||||
browserName: "safari",
|
||||
platformName: "iOS",
|
||||
"appium:deviceName": deviceName,
|
||||
"appium:platformVersion": platformVersion,
|
||||
"appium:automationName": "XCUITest",
|
||||
"sauce:options": {
|
||||
name: sessionName,
|
||||
build: buildName,
|
||||
realDevice: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
browserName: "safari",
|
||||
platformName: "iOS",
|
||||
"LT:Options": {
|
||||
deviceName,
|
||||
platformVersion,
|
||||
platformName: "iOS",
|
||||
isRealMobile: true,
|
||||
name: sessionName,
|
||||
build: buildName,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function publicCapabilityPlan(provider, opts = {}) {
|
||||
const normalized = normalizeProvider(provider);
|
||||
const capabilities = buildIosCapabilities(normalized, opts);
|
||||
return {
|
||||
provider: normalized,
|
||||
device: capabilityDeviceName(normalized, capabilities),
|
||||
platformVersion: capabilityPlatformVersion(normalized, capabilities),
|
||||
capabilities,
|
||||
};
|
||||
}
|
||||
|
||||
export function capabilityDeviceName(provider, capabilities) {
|
||||
const normalized = normalizeProvider(provider);
|
||||
if (normalized === "browserstack") {
|
||||
return capabilities["bstack:options"]?.deviceName ?? null;
|
||||
}
|
||||
if (normalized === "sauce") {
|
||||
return capabilities["appium:deviceName"] ?? null;
|
||||
}
|
||||
if (normalized === "lambdatest") {
|
||||
return capabilities["LT:Options"]?.deviceName ?? null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function capabilityPlatformVersion(provider, capabilities) {
|
||||
const normalized = normalizeProvider(provider);
|
||||
if (normalized === "browserstack") {
|
||||
return capabilities["bstack:options"]?.osVersion ?? null;
|
||||
}
|
||||
if (normalized === "sauce") {
|
||||
return capabilities["appium:platformVersion"] ?? null;
|
||||
}
|
||||
if (normalized === "lambdatest") {
|
||||
return capabilities["LT:Options"]?.platformVersion ?? null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
30
scripts/lib/release-authorization-gate.mjs
Normal file
30
scripts/lib/release-authorization-gate.mjs
Normal file
@@ -0,0 +1,30 @@
|
||||
export const RELEASE_AUTHORIZATION_ENV = "FUSION_RELEASE_AUTHORIZED";
|
||||
|
||||
/**
|
||||
* FNXC:ReleaseScript 2026-06-15-02:41:
|
||||
* FN-6469 proved that branch and working-tree preflight checks are not an authorization boundary because an agent can clone `main` into a fresh directory and rerun `pnpm release --yes`.
|
||||
* Real releases are not agent-initiable: the publish path requires an explicit operator-held environment signal that is outside repo state and cannot be self-granted by reproducing `main`; dry-runs bypass this gate because they publish nothing.
|
||||
*
|
||||
* @param {{ dryRun: boolean, env?: Record<string, string | undefined>, stdinIsTTY?: boolean }} options
|
||||
* @returns {{ authorized: boolean, mode: "dry-run-bypass" | "env-signal" | "blocked", reason?: string }}
|
||||
*/
|
||||
export function evaluateReleaseAuthorization({ dryRun, env = {}, stdinIsTTY = false }) {
|
||||
if (dryRun === true) {
|
||||
return { authorized: true, mode: "dry-run-bypass" };
|
||||
}
|
||||
|
||||
const signal = env[RELEASE_AUTHORIZATION_ENV];
|
||||
if (typeof signal === "string" && signal.trim() !== "") {
|
||||
return { authorized: true, mode: "env-signal" };
|
||||
}
|
||||
|
||||
const shellContext = stdinIsTTY
|
||||
? "No operator authorization signal was present in this interactive shell."
|
||||
: "No operator authorization signal was present in this non-interactive shell.";
|
||||
|
||||
return {
|
||||
authorized: false,
|
||||
mode: "blocked",
|
||||
reason: `${shellContext} Real releases require explicit operator authorization via ${RELEASE_AUTHORIZATION_ENV}; aborted before version bump, publish, push, or tag.`,
|
||||
};
|
||||
}
|
||||
13
scripts/lib/release-prompt-gate.mjs
Normal file
13
scripts/lib/release-prompt-gate.mjs
Normal file
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* FNXC:ReleaseScript 2026-06-14-23:08:
|
||||
* Dry-run releases must be non-interactive by default because FN-6469 showed non-TTY agent shells can hang on unsettled top-level await and exit 13 when the version prompt reads stdin.
|
||||
* `--interactive` is the explicit dry-run opt-in for maintainers who intentionally want to exercise the version prompt; real releases keep prompting unless `--yes` is passed.
|
||||
*
|
||||
* @param {{ dryRun: boolean, autoYes: boolean, interactive: boolean }} options
|
||||
* @returns {boolean} true when the release script should prompt for a version override.
|
||||
*/
|
||||
export function shouldPromptForVersion({ dryRun, autoYes, interactive }) {
|
||||
if (autoYes) return false;
|
||||
if (dryRun) return interactive;
|
||||
return true;
|
||||
}
|
||||
290
scripts/lib/run-vitest-watchdog.mjs
Normal file
290
scripts/lib/run-vitest-watchdog.mjs
Normal file
@@ -0,0 +1,290 @@
|
||||
|
||||
/**
|
||||
* Shared, bounded test-invocation runner (the L2 watchdog layer).
|
||||
*
|
||||
* Generalizes the process-group lifecycle proven in
|
||||
* packages/dashboard/scripts/run-vitest-with-heap.mjs so that
|
||||
* scripts/ci-test-shard.mjs and scripts/test-changed.mjs can wrap each vitest
|
||||
* invocation in a wall-clock killer instead of letting a wedged run block to
|
||||
* the CI 6h ceiling (or hang a local run forever).
|
||||
*
|
||||
* Design notes:
|
||||
* - `runWithWatchdog` spawns the command DETACHED (its own process group) and,
|
||||
* on timeout, SIGTERMs the whole group, then SIGKILLs after a grace window —
|
||||
* the same lifecycle the dashboard runner uses. It returns a result object
|
||||
* ({ code, signal, timedOut }) rather than calling process.exit, so each
|
||||
* caller decides its own exit/signal-re-raise behavior. This lets the shard
|
||||
* runner loop over many invocations in one process without leaking handlers.
|
||||
* - Budgets are NOT a single flat constant. `deriveBudgetMs` uses per-class
|
||||
* floor/ceiling bands as the load-bearing safety net; a fresh timings value
|
||||
* only TIGHTENS within the band. With no fresh timings, the generous ceiling
|
||||
* is used so a stale snapshot can never produce a too-tight (false-kill)
|
||||
* budget. This is not an assertion-timeout widening — it bounds a currently
|
||||
* unbounded outer wait. See the plan KTD-2.
|
||||
* - On timeout the watchdog emits inline hang diagnostics (the wrapper-side
|
||||
* half of U2): which invocation hung, for how long, and the wrapper's own
|
||||
* active-handle summary. The child's own open-handle dump is produced inside
|
||||
* the vitest process by the SIGTERM diagnostics in vitest-setup.ts.
|
||||
*/
|
||||
|
||||
const MINUTE = 60_000;
|
||||
|
||||
/**
|
||||
* Per-invocation-class budget bands (milliseconds). The floor/ceiling are the
|
||||
* safety net; timings tighten within them. Tune against a freshly refreshed
|
||||
* scripts/test-timings.json (see the plan's Deferred Implementation Notes).
|
||||
*/
|
||||
export const CLASS_BUDGET_BANDS = {
|
||||
// One CI shard command (may fan out across several packages via --filter).
|
||||
shard: { floor: 5 * MINUTE, ceiling: 30 * MINUTE },
|
||||
// One local changed-file package invocation.
|
||||
changed: { floor: 2 * MINUTE, ceiling: 20 * MINUTE },
|
||||
// One dashboard quality lane (heap-managed). Matches the historical 15min.
|
||||
"dashboard-lane": { floor: 15 * MINUTE, ceiling: 30 * MINUTE },
|
||||
};
|
||||
|
||||
export const DEFAULT_BUDGET_MULTIPLIER = 3.5;
|
||||
export const DEFAULT_GRACE_MS = 5_000;
|
||||
export const DEFAULT_HEARTBEAT_MS = 5_000;
|
||||
export const TIMEOUT_EXIT_CODE = 124;
|
||||
|
||||
/**
|
||||
* Derive a wall-clock budget for one invocation.
|
||||
*
|
||||
* @param {object} opts
|
||||
* @param {keyof typeof CLASS_BUDGET_BANDS} opts.klass
|
||||
* @param {number|null} [opts.expectedDurationMs] aggregated expected duration
|
||||
* across every package/lane packed into this invocation (sum, not a single
|
||||
* package lookup), or null when unknown.
|
||||
* @param {boolean} [opts.timingsFresh] whether the timings snapshot feeding
|
||||
* expectedDurationMs is fresh enough to trust.
|
||||
* @param {number} [opts.multiplier]
|
||||
* @returns {number} budget in milliseconds
|
||||
*/
|
||||
export function deriveBudgetMs({
|
||||
klass,
|
||||
expectedDurationMs = null,
|
||||
timingsFresh = false,
|
||||
multiplier = DEFAULT_BUDGET_MULTIPLIER,
|
||||
} = {}) {
|
||||
const band = CLASS_BUDGET_BANDS[klass] ?? CLASS_BUDGET_BANDS.changed;
|
||||
// No usable, fresh timing → fall back to the generous ceiling. A stale or
|
||||
// missing snapshot must never yield a tighter-than-ceiling budget.
|
||||
if (!timingsFresh || expectedDurationMs == null || !(expectedDurationMs > 0)) {
|
||||
return band.ceiling;
|
||||
}
|
||||
const derived = Math.round(expectedDurationMs * multiplier);
|
||||
return Math.max(band.floor, Math.min(band.ceiling, derived));
|
||||
}
|
||||
|
||||
/**
|
||||
* Summarize the wrapper process's active handles/requests, bounded so a hang
|
||||
* dump cannot itself flood CI logs. Reports handle TYPE counts only (never
|
||||
* payloads) so there is nothing to redact.
|
||||
*/
|
||||
export function summarizeActiveHandles({ limit = 12 } = {}) {
|
||||
const handles =
|
||||
typeof process._getActiveHandles === "function" ? process._getActiveHandles() : [];
|
||||
const requests =
|
||||
typeof process._getActiveRequests === "function" ? process._getActiveRequests() : [];
|
||||
|
||||
const counts = new Map();
|
||||
for (const h of [...handles, ...requests]) {
|
||||
const name = h?.constructor?.name ?? typeof h;
|
||||
counts.set(name, (counts.get(name) ?? 0) + 1);
|
||||
}
|
||||
if (counts.size === 0) return "no pending handles in wrapper process";
|
||||
|
||||
const sorted = [...counts.entries()].sort((a, b) => b[1] - a[1]);
|
||||
const shown = sorted.slice(0, limit).map(([name, n]) => `${name}×${n}`);
|
||||
const remainder = sorted.length - shown.length;
|
||||
const suffix = remainder > 0 ? ` (+${remainder} more types)` : "";
|
||||
return shown.join(", ") + suffix;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the inline hang-diagnostic summary emitted on timeout (U2, wrapper side).
|
||||
*/
|
||||
export function captureHangDiagnostics({ label, command, args, budgetMs, startedAt, lastHeartbeatAt, now }) {
|
||||
const elapsedMs = now - startedAt;
|
||||
const sinceHeartbeat = lastHeartbeatAt ? now - lastHeartbeatAt : null;
|
||||
const lines = [
|
||||
`[watchdog] HANG: ${label} exceeded budget ${budgetMs}ms (elapsed ${elapsedMs}ms)`,
|
||||
`[watchdog] command: ${command} ${args.join(" ")}`,
|
||||
lastHeartbeatAt != null
|
||||
? `[watchdog] last heartbeat: ${sinceHeartbeat}ms ago`
|
||||
: `[watchdog] last heartbeat: none observed`,
|
||||
`[watchdog] wrapper handles: ${summarizeActiveHandles({})}`,
|
||||
`[watchdog] (child open-handle dump, if any, is printed by the vitest process on SIGTERM)`,
|
||||
];
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a command under a wall-clock watchdog in its own process group.
|
||||
*
|
||||
* Resolves to { code, signal, timedOut, diagnostics } and never rejects for an
|
||||
* ordinary child failure — callers translate the result into their own exit
|
||||
* behavior. Installs SIGINT/SIGTERM/SIGHUP forwarders and an exit cleanup hook
|
||||
* for the lifetime of THIS invocation only, removing them once the child
|
||||
* settles so sequential invocations in a loop don't accumulate handlers.
|
||||
*
|
||||
* @param {object} opts
|
||||
* @param {string} opts.command
|
||||
* @param {string[]} opts.args
|
||||
* @param {NodeJS.ProcessEnv} [opts.env]
|
||||
* @param {number} opts.budgetMs wall-clock budget; <=0 or non-finite disables the killer
|
||||
* @param {number} [opts.graceMs] SIGTERM→SIGKILL grace window
|
||||
* @param {number} [opts.heartbeatMs]
|
||||
* @param {string} [opts.label]
|
||||
* @param {(msg: string) => void} [opts.log]
|
||||
* @param {object} opts.spawn injected spawn (node:child_process spawn); required for testability
|
||||
* @param {string} [opts.cwd] working directory for the spawned child (preserves callers that
|
||||
* ran the test command from a fixed root, e.g. test-changed.mjs's rootDir)
|
||||
* @param {() => number} [opts.now] injected clock (defaults to Date.now)
|
||||
* @param {(signal: string) => void} [opts.killGroup] injected group-signaller
|
||||
* (defaults to a process-group `process.kill(-pid)` with child.kill fallback);
|
||||
* override in tests so signals are captured instead of hitting real groups.
|
||||
*/
|
||||
export function runWithWatchdog({
|
||||
command,
|
||||
args,
|
||||
env = process.env,
|
||||
cwd = null,
|
||||
budgetMs,
|
||||
graceMs = DEFAULT_GRACE_MS,
|
||||
heartbeatMs = DEFAULT_HEARTBEAT_MS,
|
||||
label = command,
|
||||
log = console.error,
|
||||
spawn,
|
||||
now = () => Date.now(),
|
||||
killGroup = null,
|
||||
}) {
|
||||
if (typeof spawn !== "function") {
|
||||
throw new Error("runWithWatchdog requires an injected `spawn` function");
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const startedAt = now();
|
||||
let lastHeartbeatAt = null;
|
||||
let timedOut = false;
|
||||
let diagnostics = null;
|
||||
let forceKillTimer = null;
|
||||
let settled = false;
|
||||
|
||||
// process-supervisor-allowlist: foreground wrapper signals the whole vitest
|
||||
// process group on death/timeout; not a background daemon.
|
||||
const child = spawn(command, args, {
|
||||
detached: true,
|
||||
stdio: "inherit",
|
||||
env,
|
||||
...(cwd ? { cwd } : {}),
|
||||
});
|
||||
|
||||
const heartbeat = setInterval(() => {
|
||||
lastHeartbeatAt = now();
|
||||
log(`[watchdog] still running: ${label}`);
|
||||
}, heartbeatMs);
|
||||
heartbeat.unref?.();
|
||||
|
||||
function defaultSignalGroup(signal) {
|
||||
try {
|
||||
process.kill(-child.pid, signal);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error) || !("code" in error)) throw error;
|
||||
if (error.code !== "ESRCH" && error.code !== "EPERM") throw error;
|
||||
}
|
||||
try {
|
||||
child.kill(signal);
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error) || !("code" in error) || error.code !== "ESRCH") throw error;
|
||||
}
|
||||
}
|
||||
const signalGroup = typeof killGroup === "function" ? killGroup : defaultSignalGroup;
|
||||
|
||||
// Arm the SIGTERM→SIGKILL grace ladder once. Used by BOTH the timeout path
|
||||
// and external-cancellation forwarding so a child that ignores SIGTERM can't
|
||||
// keep the wrapper pending until the full budget (the original handlers
|
||||
// suppressed Node's default exit behavior, so Ctrl-C / CI cancellation could
|
||||
// otherwise hang for the whole per-command ceiling).
|
||||
function armForceKill(triggerSignal) {
|
||||
if (forceKillTimer) return;
|
||||
forceKillTimer = setTimeout(() => {
|
||||
log(`[watchdog] grace expired after ${triggerSignal}; SIGKILL: ${label}`);
|
||||
signalGroup("SIGKILL");
|
||||
}, Math.max(1, graceMs));
|
||||
forceKillTimer.unref?.();
|
||||
}
|
||||
|
||||
const watchdog =
|
||||
Number.isFinite(budgetMs) && budgetMs > 0
|
||||
? setTimeout(() => {
|
||||
timedOut = true;
|
||||
diagnostics = captureHangDiagnostics({
|
||||
label,
|
||||
command,
|
||||
args,
|
||||
budgetMs,
|
||||
startedAt,
|
||||
lastHeartbeatAt,
|
||||
now: now(),
|
||||
});
|
||||
log(diagnostics);
|
||||
signalGroup("SIGTERM");
|
||||
armForceKill("timeout");
|
||||
}, budgetMs)
|
||||
: null;
|
||||
watchdog?.unref?.();
|
||||
|
||||
const forwardedSignals = ["SIGINT", "SIGTERM", "SIGHUP"];
|
||||
const signalHandlers = new Map();
|
||||
for (const sig of forwardedSignals) {
|
||||
const handler = () => {
|
||||
log(`[watchdog] received ${sig}; forwarding to group: ${label}`);
|
||||
signalGroup(sig);
|
||||
armForceKill(sig);
|
||||
};
|
||||
signalHandlers.set(sig, handler);
|
||||
process.on(sig, handler);
|
||||
}
|
||||
|
||||
function onProcExit() {
|
||||
// Best-effort: don't leave an orphaned group if the wrapper itself dies.
|
||||
// Route through signalGroup so the injection contract holds everywhere.
|
||||
try {
|
||||
signalGroup("SIGTERM");
|
||||
} catch {
|
||||
/* group already gone */
|
||||
}
|
||||
}
|
||||
process.on("exit", onProcExit);
|
||||
|
||||
function cleanup() {
|
||||
clearInterval(heartbeat);
|
||||
if (watchdog) clearTimeout(watchdog);
|
||||
if (forceKillTimer) clearTimeout(forceKillTimer);
|
||||
for (const [sig, handler] of signalHandlers) process.removeListener(sig, handler);
|
||||
process.removeListener("exit", onProcExit);
|
||||
}
|
||||
|
||||
child.on("error", (error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(error);
|
||||
});
|
||||
|
||||
child.on("close", (code, signal) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
resolve({
|
||||
code: timedOut ? TIMEOUT_EXIT_CODE : code,
|
||||
signal: timedOut ? null : signal,
|
||||
timedOut,
|
||||
diagnostics,
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -2,49 +2,15 @@
|
||||
"$comment": "Flaky-test quarantine ledger (deletion ratchet — see AGENTS.md 'Flaky tests: quarantine on sight' and docs/testing.md 'Quarantine ledger and the deletion ratchet'). A test observed failing without a corresponding real bug is quarantined ON SIGHT: add an entry here AND a matching one-line `exclude` entry in that package's vitest config, in the same commit. Every entry needs a non-empty `reason` (link the failing run) and a `quarantinedAt` date — the entry expires 14 days later, at which point the test file is DELETED unless someone rescues it with evidence it catches real regressions plus a root-cause fix (never appeasement). There is deliberately no loader module and no automation around this file: it is a dated record, the vitest config exclude is the mechanism, and the sweep is policy executed by whoever touches the suite.",
|
||||
"entries": [
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/project-engine-manager.test.ts",
|
||||
"reason": "Flake: setInterval-driven reconciliation races with vitest fake-timer contention under full reliability-suite load. Test passes standalone (~46ms) but times out (30s) when reliability-pool files are co-resident. FN-6206.",
|
||||
"quarantinedAt": "2026-06-10"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/merger-file-scope-invariant.test.ts",
|
||||
"reason": "Flake: vi.mock('node:child_process') occasionally doesn't take under workspace-concurrent runs, letting real git binary leak and report staged files unrelated to test scope (trips FileScopeViolationError). Same logic covered by real-git fixture tests in reliability-interactions/workflow-and-file-scope. FN-6206.",
|
||||
"quarantinedAt": "2026-06-10"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/merger-file-scope-invariant.test.ts",
|
||||
"reason": "Flake: same mock-contention mode as the sibling changeset-file test above (vi.mock('node:child_process') not taking under concurrent load). FN-6206.",
|
||||
"quarantinedAt": "2026-06-10"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/merger-ai-cleanup-active-session.test.ts",
|
||||
"reason": "Flake: pruneExistingAiMergeWorktrees skips active-session paths — active-session temp AI merge dir was unexpectedly pruned during pnpm --filter @fusion/engine test in FN-6206 verification, while the same file passed standalone. Root cause suspected: realpathSync resolution mismatch or readdirSync mock interaction with activeSessionRegistry singleton under concurrent engine suite load. Discovered during FN-6206.",
|
||||
"quarantinedAt": "2026-06-10"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/merger-ai-cleanup.test.ts",
|
||||
"reason": "Flake observed during FN-6206 verification: `pruneExistingAiMergeWorktrees skips active-session paths` failed in full `pnpm --filter @fusion/engine test` runs while the file passed standalone, indicating suite-order/concurrency sensitivity. Follow-up FN-6207.",
|
||||
"quarantinedAt": "2026-06-10"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/merger-ai.test.ts",
|
||||
"reason": "Flake observed during FN-6238 verification: full `pnpm --filter @fusion/engine test` failed in two merger-ai tests with git ENOENT / unable to read current working directory after a temp checkout disappeared, while the file passed standalone (23/23). Follow-up FN-6248.",
|
||||
"quarantinedAt": "2026-06-11"
|
||||
"file": "packages/dashboard/src/__tests__/session-cross-tab.test.ts",
|
||||
"reason": "FN-6690 local workspace `pnpm test` observed ENOTEMPTY while removing the test's temp .fusion directory in dashboard-api-quality-backfill shard; isolated rerun passed, indicating cleanup flake rather than a lazy-view CSS regression.",
|
||||
"quarantinedAt": "2026-06-19"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/QuickEntryBox.test.tsx",
|
||||
"reason": "Flake observed during FN-6239 verification: broad `pnpm test` in dashboard backfill shard 4/4 could not find `quick-entry-priority-button` immediately after a successful task creation, while the named test passed standalone. Indicates suite-order/concurrency sensitivity unrelated to QuickChatFAB coverage.",
|
||||
"quarantinedAt": "2026-06-11"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/soft-delete-blocker-residue.test.ts",
|
||||
"reason": "Flake observed during FN-6294 verification and reproduced during FN-6319 broad `pnpm --filter @fusion/engine test`: `clearStaleBlockedBy handles missed task:deleted event with soft-deleted-blocker reason` failed because the log entry was absent, while the same file passed standalone and the narrow three-file reproduction passed. Product-code cross-check: `clearStaleBlockedBy` still has the soft-deleted-blocker branch and soft-delete-deadlock-scan-exclusion.test.ts covers it via a deterministic store double, indicating suite-order/concurrency sensitivity in this reliability-interactions fixture rather than a confirmed product bug.",
|
||||
"quarantinedAt": "2026-06-12"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-settings.test.ts",
|
||||
"reason": "Flake observed during FN-6354 broad `pnpm test`: `GET /api/memory/audit > preserves extraction metadata across extract then audit requests` received HTTP 503 instead of 200 in the dashboard api:curated lane, while the same named test passed standalone immediately afterward. FN-6354 only changed the task-detail Chat composer UI/tests, so this is classified as unrelated suite-order/concurrency sensitivity in the dashboard API quality lane.",
|
||||
"quarantinedAt": "2026-06-13"
|
||||
"reason": "FN-6697 local workspace `pnpm test` observed the post-submission focus restoration test fail in the broad dashboard app backfill shard, while a targeted rerun of QuickEntryBox with MailboxModal passed the QuickEntryBox assertions; quarantine the focus-timing flake instead of appeasing it while the terminal shortcut fix remains scoped.",
|
||||
"quarantinedAt": "2026-06-19"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
4
scripts/lib/test-timeout-appeasement-allowlist.json
Normal file
4
scripts/lib/test-timeout-appeasement-allowlist.json
Normal file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"$comment": "Vitest timeout-appeasement allowlist (temporary exemption ledger — see AGENTS.md 'Do Not Add Slow Tests' and docs/testing.md 'Quarantine ledger and the deletion ratchet'). The guard blocks new per-file or suite-level `testTimeout` / `hookTimeout` bumps in tracked test files. Every entry needs a repo-relative `file`, non-empty `reason` linking the owning cleanup/quarantine work, and `allowlistedAt` date. The goal is removal, not permanence: quarantine the flaky test or narrow the slow seam, then delete the timeout bump and this entry.",
|
||||
"entries": []
|
||||
}
|
||||
102
scripts/lib/workflow-reliability-release-check.json
Normal file
102
scripts/lib/workflow-reliability-release-check.json
Normal file
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"version": 1,
|
||||
"_comment": "FNXC:CustomWorkflowReliability 2026-06-19-00:00: FN-6694 requires this manifest to encode goal G-MPW67VQR-0001-97S3's release-check checklist as objective, runnable seam evidence instead of prose-only QA signoff.",
|
||||
"checklist": [
|
||||
{
|
||||
"id": "author-import-save-discover-reload",
|
||||
"title": "Custom workflow authoring, import, invalid rejection, save, discovery, selection, and reload",
|
||||
"journey": "A custom workflow can be authored/imported, rejected on invalid IR, saved, discovered, selected, and reloaded.",
|
||||
"seams": [
|
||||
{
|
||||
"package": "@fusion/core",
|
||||
"file": "packages/core/src/__tests__/workflow-definition-store.test.ts"
|
||||
},
|
||||
{
|
||||
"package": "@fusion/dashboard",
|
||||
"file": "packages/dashboard/src/routes/__tests__/workflow-import-export.test.ts"
|
||||
},
|
||||
{
|
||||
"package": "@fusion/dashboard",
|
||||
"file": "packages/dashboard/src/routes/__tests__/workflow-design-route.test.ts"
|
||||
},
|
||||
{
|
||||
"package": "@fusion/core",
|
||||
"file": "packages/core/src/__tests__/workflow-selection-store.test.ts"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "selected-workflow-execution-fail-closed",
|
||||
"title": "Selected workflow execution and fail-closed missing custom workflow IDs",
|
||||
"journey": "A task can execute the selected workflow through runtime primitives, and explicit missing custom workflow IDs fail closed.",
|
||||
"seams": [
|
||||
{
|
||||
"package": "@fusion/core",
|
||||
"file": "packages/core/src/__tests__/workflow-selection-store.test.ts"
|
||||
},
|
||||
{
|
||||
"package": "@fusion/engine",
|
||||
"file": "packages/engine/src/__tests__/workflow-task-runtime.test.ts"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "gate-advisory-readonly-revise-required-artifact",
|
||||
"title": "Gate, advisory, readonly, REVISE, and required-artifact observability",
|
||||
"journey": "Gate/advisory/readonly/REVISE/required-artifact behavior is observable in task state, workflow results, task documents, and logs.",
|
||||
"seams": [
|
||||
{
|
||||
"package": "@fusion/engine",
|
||||
"file": "packages/engine/src/__tests__/workflow-malformed-verdict-gate.test.ts"
|
||||
},
|
||||
{
|
||||
"package": "@fusion/engine",
|
||||
"file": "packages/engine/src/__tests__/workflow-required-artifact-gate.test.ts"
|
||||
},
|
||||
{
|
||||
"package": "@fusion/engine",
|
||||
"file": "packages/engine/src/__tests__/workflow-step-readonly-allowlist.test.ts"
|
||||
},
|
||||
{
|
||||
"package": "@fusion/engine",
|
||||
"file": "packages/engine/src/__tests__/executor-workflow-revision-scope.test.ts"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "automerge-hard-cancel-file-scope-recovery",
|
||||
"title": "autoMerge:false, hard-cancel, file-scope, and recovery invariants",
|
||||
"journey": "autoMerge:false, hard-cancel, file-scope, and recovery invariants are preserved under custom workflow execution.",
|
||||
"seams": [
|
||||
{
|
||||
"package": "@fusion/engine",
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/workflow-and-file-scope.test.ts"
|
||||
},
|
||||
{
|
||||
"package": "@fusion/engine",
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/workflow-interpreter-cutover.test.ts"
|
||||
},
|
||||
{
|
||||
"package": "@fusion/engine",
|
||||
"file": "packages/engine/src/__tests__/self-healing-custom-workflow-recovery.test.ts"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "restart-selection-progress-run-audit",
|
||||
"title": "Restart durability for workflow selection, progress, and recovery evidence",
|
||||
"journey": "Engine/scheduler restart preserves workflow selection and progress, and any recovery emits typed run-audit evidence instead of silent lifecycle mutation.",
|
||||
"seams": [
|
||||
{
|
||||
"package": "@fusion/core",
|
||||
"file": "packages/core/src/__tests__/workflow-restart-durability.test.ts"
|
||||
},
|
||||
{
|
||||
"package": "@fusion/engine",
|
||||
"file": "packages/engine/src/__tests__/self-healing-custom-workflow-recovery.test.ts"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"manual": []
|
||||
}
|
||||
151
scripts/mobile-run-android.sh
Executable file
151
scripts/mobile-run-android.sh
Executable file
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# FNXC:MobileAndroidRun 2026-06-16-17:30:
|
||||
# Convenience deploy script for the Fusion Capacitor Android app. Codifies the
|
||||
# environment the native build needs so a contributor can go from "connected
|
||||
# phone" to "app installed" in one command, without re-discovering toolchain
|
||||
# requirements each time.
|
||||
#
|
||||
# Requirements encoded here (learned during the first manual deploy):
|
||||
# - Capacitor 7's :capacitor-android library compiles at source release 21, so
|
||||
# Gradle MUST run under a JDK 21 toolchain. JDK 17 fails with
|
||||
# "invalid source release: 21". We pin JAVA_HOME to Homebrew openjdk@21.
|
||||
# - The Android SDK lives at the Homebrew cmdline-tools root, not the default
|
||||
# ~/Library/Android/sdk. We export ANDROID_HOME/ANDROID_SDK_ROOT and write
|
||||
# android/local.properties (sdk.dir=...) so Gradle resolves the SDK.
|
||||
# - The target device connects over network ADB (Tailscale), which drops
|
||||
# between commands. We re-run `adb connect` for FUSION_ANDROID_DEVICE right
|
||||
# before deploy so Capacitor can see the device as a valid target.
|
||||
# - When FUSION_SERVER_URL is set, the webview loads the live backend
|
||||
# (assets + API) from that origin instead of the bundled static client.
|
||||
# This is the working path until the mobile shell host-context wiring lands
|
||||
# (see shell-host.ts detectShellHostContext: it has no Capacitor branch, so
|
||||
# a bundled build self-identifies as a plain browser and calls /api against
|
||||
# its own static origin -> "API returned HTML instead of JSON").
|
||||
#
|
||||
# Usage:
|
||||
# FUSION_ANDROID_DEVICE=100.96.156.40:5555 \
|
||||
# FUSION_SERVER_URL=http://100.97.197.105:4040 \
|
||||
# pnpm mobile:run:android
|
||||
#
|
||||
# Env vars:
|
||||
# FUSION_ANDROID_DEVICE adb target id (host:port for network adb, or serial).
|
||||
# If unset, Capacitor auto-selects the only device.
|
||||
# FUSION_SERVER_URL Optional. If set, the app loads from this backend URL
|
||||
# (remote/live mode). If unset, ships the bundled client.
|
||||
# ANDROID_HOME Optional override for the SDK root.
|
||||
# JAVA_HOME Optional override for the JDK 21 home.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
MOBILE_DIR="$REPO_ROOT/packages/mobile"
|
||||
|
||||
# --- Resolve Android SDK root ------------------------------------------------
|
||||
if [[ -z "${ANDROID_HOME:-}" ]]; then
|
||||
for candidate in \
|
||||
"/opt/homebrew/share/android-commandlinetools" \
|
||||
"$HOME/Library/Android/sdk" \
|
||||
"/usr/local/share/android-commandlinetools"; do
|
||||
if [[ -d "$candidate" ]]; then
|
||||
ANDROID_HOME="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [[ -z "${ANDROID_HOME:-}" || ! -d "$ANDROID_HOME" ]]; then
|
||||
echo "[mobile:run:android] Could not locate the Android SDK. Set ANDROID_HOME." >&2
|
||||
exit 1
|
||||
fi
|
||||
export ANDROID_HOME
|
||||
export ANDROID_SDK_ROOT="$ANDROID_HOME"
|
||||
|
||||
# --- Resolve JDK 21 ----------------------------------------------------------
|
||||
if [[ -z "${JAVA_HOME:-}" ]]; then
|
||||
for candidate in \
|
||||
"/opt/homebrew/opt/openjdk@21/libexec/openjdk.jdk/Contents/Home" \
|
||||
"/usr/local/opt/openjdk@21/libexec/openjdk.jdk/Contents/Home"; do
|
||||
if [[ -d "$candidate" ]]; then
|
||||
JAVA_HOME="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [[ -z "${JAVA_HOME:-}" || ! -x "$JAVA_HOME/bin/java" ]]; then
|
||||
echo "[mobile:run:android] JDK 21 not found. Install with: brew install openjdk@21" >&2
|
||||
echo "[mobile:run:android] Or set JAVA_HOME to a JDK 21 home." >&2
|
||||
exit 1
|
||||
fi
|
||||
export JAVA_HOME
|
||||
export PATH="$JAVA_HOME/bin:$ANDROID_HOME/platform-tools:$PATH"
|
||||
|
||||
echo "[mobile:run:android] ANDROID_HOME=$ANDROID_HOME"
|
||||
echo "[mobile:run:android] JAVA_HOME=$JAVA_HOME ($("$JAVA_HOME/bin/java" -version 2>&1 | head -1))"
|
||||
|
||||
# --- Ensure the Android platform project exists ------------------------------
|
||||
if [[ ! -d "$MOBILE_DIR/android" ]]; then
|
||||
echo "[mobile:run:android] Android project missing; adding it (cap add android)..."
|
||||
(cd "$MOBILE_DIR" && npx cap add android)
|
||||
fi
|
||||
|
||||
# Gradle reads the SDK location from local.properties.
|
||||
printf "sdk.dir=%s\n" "$ANDROID_HOME" > "$MOBILE_DIR/android/local.properties"
|
||||
|
||||
# FNXC:MobileShell 2026-06-16-18:40:
|
||||
# The Android project is generated (gitignored), so re-apply the edge-to-edge
|
||||
# enablement that @capacitor-community/safe-area needs in MainActivity. Idempotent:
|
||||
# only rewrites when EdgeToEdge is not already wired. Without this the status bar
|
||||
# overlaps the app top on Android 15+ (API 35+).
|
||||
MAIN_ACTIVITY="$MOBILE_DIR/android/app/src/main/java/com/fusion/mobile/MainActivity.java"
|
||||
if [[ -f "$MAIN_ACTIVITY" ]] && ! grep -q "EdgeToEdge" "$MAIN_ACTIVITY"; then
|
||||
echo "[mobile:run:android] Patching MainActivity for edge-to-edge (safe-area insets)..."
|
||||
cat > "$MAIN_ACTIVITY" <<'JAVA'
|
||||
package com.fusion.mobile;
|
||||
|
||||
import android.os.Bundle;
|
||||
import androidx.activity.EdgeToEdge;
|
||||
import com.getcapacitor.BridgeActivity;
|
||||
|
||||
// FNXC:MobileShell 2026-06-16-18:40:
|
||||
// Enable Android edge-to-edge so @capacitor-community/safe-area passes status-bar
|
||||
// insets to the WebView as env(safe-area-inset-*). Re-applied by
|
||||
// scripts/mobile-run-android.sh because android/ is generated (gitignored).
|
||||
public class MainActivity extends BridgeActivity {
|
||||
@Override
|
||||
public void onCreate(Bundle savedInstanceState) {
|
||||
super.onCreate(savedInstanceState);
|
||||
EdgeToEdge.enable(this);
|
||||
}
|
||||
}
|
||||
JAVA
|
||||
fi
|
||||
|
||||
# --- Build web client --------------------------------------------------------
|
||||
echo "[mobile:run:android] Building dashboard web client..."
|
||||
pnpm --filter @fusion/dashboard build
|
||||
|
||||
RUN_ENV=()
|
||||
if [[ -n "${FUSION_SERVER_URL:-}" ]]; then
|
||||
echo "[mobile:run:android] Remote mode: app will load from $FUSION_SERVER_URL"
|
||||
RUN_ENV+=("FUSION_LIVE_RELOAD=true" "FUSION_SERVER_URL=$FUSION_SERVER_URL")
|
||||
fi
|
||||
|
||||
# --- Reconnect network ADB device right before deploy ------------------------
|
||||
# FNXC:MobileAndroidRun 2026-06-16-17:55: Network ADB (Tailscale) drops on idle,
|
||||
# so reconnect AFTER the web build (which takes seconds) and immediately before
|
||||
# `cap run`, otherwise Capacitor sees no device and rejects the target id.
|
||||
DEVICE="${FUSION_ANDROID_DEVICE:-}"
|
||||
if [[ -n "$DEVICE" ]]; then
|
||||
echo "[mobile:run:android] Reconnecting adb device $DEVICE..."
|
||||
adb connect "$DEVICE" || true
|
||||
sleep 1
|
||||
fi
|
||||
echo "[mobile:run:android] Attached devices:"
|
||||
adb devices
|
||||
|
||||
cd "$MOBILE_DIR"
|
||||
if [[ -n "$DEVICE" ]]; then
|
||||
env "${RUN_ENV[@]}" npx cap run android --target "$DEVICE"
|
||||
else
|
||||
env "${RUN_ENV[@]}" npx cap run android
|
||||
fi
|
||||
@@ -10,11 +10,15 @@
|
||||
// - clean working tree on `main`, up to date with origin
|
||||
// - at least one pending changeset in .changeset/
|
||||
// - `npm login` already completed (publish uses the active npm token)
|
||||
// - real releases require an operator-held FUSION_RELEASE_AUTHORIZED signal;
|
||||
// dry-runs do not require it because they make no file/git/npm changes
|
||||
//
|
||||
// Usage:
|
||||
// pnpm release # interactive: review changesets, accept or override version, confirm
|
||||
// pnpm release --yes # accept the proposed version, skip confirmation prompt
|
||||
// pnpm release --dry-run # preview only — exit before any file/git/npm changes
|
||||
// pnpm release # interactive: review changesets, accept or override version, confirm, then require operator authorization before mutation
|
||||
// pnpm release --yes # accept the proposed version, skip confirmation prompt, still require operator authorization before mutation
|
||||
// pnpm release --dry-run # preview only; non-interactive by default; no authorization signal or file/git/npm changes
|
||||
// pnpm release --dry-run --interactive
|
||||
// # preview only, but exercise the version prompt override
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { readFileSync, readdirSync, writeFileSync, statSync, existsSync, unlinkSync, mkdtempSync, rmSync } from "node:fs";
|
||||
@@ -23,11 +27,18 @@ import { tmpdir } from "node:os";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
import { stdin, stdout } from "node:process";
|
||||
|
||||
import { evaluateReleaseAuthorization } from "./lib/release-authorization-gate.mjs";
|
||||
import { extractVersionNotes } from "./lib/extract-version-notes.mjs";
|
||||
import { shouldPromptForVersion } from "./lib/release-prompt-gate.mjs";
|
||||
|
||||
const args = new Set(process.argv.slice(2));
|
||||
/*
|
||||
* FNXC:ReleaseScript 2026-06-14-23:08:
|
||||
* `--dry-run` must not read stdin in the default agent-shell path; `--interactive` is the explicit maintainer override for prompt coverage while preserving real-release prompts.
|
||||
*/
|
||||
const DRY_RUN = args.has("--dry-run");
|
||||
const AUTO_YES = args.has("--yes") || args.has("-y");
|
||||
const INTERACTIVE = args.has("--interactive");
|
||||
|
||||
const color = (c, s) => `\x1b[${c}m${s}\x1b[0m`;
|
||||
const info = (s) => console.log(color(36, "▶ ") + s);
|
||||
@@ -506,7 +517,7 @@ console.log(` Bumped packages : ${releases.map((r) => r.name).join(", ")}`);
|
||||
console.log("");
|
||||
|
||||
let chosenVersion = proposedVersion;
|
||||
if (!AUTO_YES) {
|
||||
if (shouldPromptForVersion({ dryRun: DRY_RUN, autoYes: AUTO_YES, interactive: INTERACTIVE })) {
|
||||
while (true) {
|
||||
const answer = await ask(`Release version [${proposedVersion}]: `);
|
||||
if (answer === "") break;
|
||||
@@ -529,6 +540,23 @@ if (DRY_RUN) {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
/*
|
||||
* FNXC:ReleaseScript 2026-06-15-02:45:
|
||||
* FN-6469 showed `main`-branch preflight is bypassable by cloning a clean `main`; require an out-of-tree operator-held authorization signal before any version bump, publish, push, tag, GitHub Release, or Homebrew tap mutation can begin.
|
||||
* Dry-run exits above so agents can still inspect release plans without the signal.
|
||||
*/
|
||||
const releaseAuthorization = evaluateReleaseAuthorization({
|
||||
dryRun: DRY_RUN,
|
||||
env: process.env,
|
||||
stdinIsTTY: process.stdin.isTTY === true,
|
||||
});
|
||||
if (!releaseAuthorization.authorized) {
|
||||
fail(
|
||||
`${releaseAuthorization.reason ?? "Release is not authorized."}\n` +
|
||||
"Releases are not agent-initiable. A human operator must provide the operator-held FUSION_RELEASE_AUTHORIZED signal from outside the repository before invoking a real release.",
|
||||
);
|
||||
}
|
||||
|
||||
if (!(await confirm(`Proceed with release v${chosenVersion} (build, publish, tag)?`))) {
|
||||
warn("Aborted by user.");
|
||||
process.exit(0);
|
||||
|
||||
@@ -1,15 +1,19 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { readFileSync, readdirSync, writeFileSync, mkdirSync, renameSync, mkdtempSync, rmSync, realpathSync, globSync, existsSync } from "node:fs";
|
||||
import { readFileSync, readdirSync, writeFileSync, mkdirSync, renameSync, mkdtempSync, rmSync, realpathSync, globSync, existsSync, statSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { cpus, tmpdir } from "node:os";
|
||||
import { createRequire } from "node:module";
|
||||
import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs";
|
||||
import { isSkillSyncCheckCached } from "./sync-fusion-skill-tools.mjs";
|
||||
import { computeContentHash, createRepoContentSnapshot } from "./lib/content-hash.mjs";
|
||||
import { deriveBudgetMs, runWithWatchdog } from "./lib/run-vitest-watchdog.mjs";
|
||||
|
||||
/** Generous local full-suite budget (60min): far above a real full run, far below an infinite hang. */
|
||||
const FULL_SUITE_BUDGET_MS = 60 * 60 * 1000;
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
const scriptDir = path.dirname(currentFilePath);
|
||||
@@ -174,13 +178,32 @@ const PRUNE_REMOVE_RETRIES = 3;
|
||||
const PRUNE_REMOVE_DELAY_MS = 75;
|
||||
const PRUNE_DIAGNOSTIC_CHILD_LIMIT = 8;
|
||||
const FUSION_WORKER_ROOT_OWNER_FILE = ".fusion-test-worker-root-owner";
|
||||
const FUSION_TEST_RUN_TOKEN_ENV = "FUSION_TEST_RUN_TOKEN";
|
||||
const LEGACY_MARKERLESS_ACTIVE_ROOT_MAX_AGE_MS = 30_000;
|
||||
|
||||
function ensureFusionTestRunToken(env = process.env) {
|
||||
const existing = env[FUSION_TEST_RUN_TOKEN_ENV];
|
||||
if (typeof existing === "string" && existing.trim().length > 0) return existing;
|
||||
const token = randomUUID();
|
||||
env[FUSION_TEST_RUN_TOKEN_ENV] = token;
|
||||
return token;
|
||||
}
|
||||
|
||||
ensureFusionTestRunToken();
|
||||
|
||||
function isEnoentError(err) {
|
||||
return Boolean(err && typeof err === "object" && "code" in err && err.code === "ENOENT");
|
||||
}
|
||||
|
||||
let processAliveForTests = null;
|
||||
|
||||
export function __setProcessAliveForTests(nextProcessAlive) {
|
||||
processAliveForTests = typeof nextProcessAlive === "function" ? nextProcessAlive : null;
|
||||
}
|
||||
|
||||
function isProcessAlive(pid) {
|
||||
if (!Number.isInteger(pid) || pid <= 0) return false;
|
||||
if (processAliveForTests) return Boolean(processAliveForTests(pid));
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
@@ -189,28 +212,61 @@ function isProcessAlive(pid) {
|
||||
}
|
||||
}
|
||||
|
||||
function readWorkerRootOwnerPid(rootPath) {
|
||||
function readWorkerRootOwnerInfo(rootPath) {
|
||||
try {
|
||||
const raw = readFileSync(path.join(rootPath, FUSION_WORKER_ROOT_OWNER_FILE), "utf8").trim();
|
||||
const pid = Number.parseInt(raw, 10);
|
||||
return Number.isInteger(pid) && pid > 0 ? pid : null;
|
||||
const lines = raw.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
||||
const pid = Number.parseInt(lines[0] ?? "", 10);
|
||||
if (!Number.isInteger(pid) || pid <= 0) return null;
|
||||
const info = { pid, runToken: null };
|
||||
for (const line of lines.slice(1)) {
|
||||
const match = /^runToken=(.+)$/.exec(line);
|
||||
if (match) info.runToken = match[1];
|
||||
}
|
||||
return info;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function isActiveFusionWorkerRoot(rootPath) {
|
||||
const ownerPid = readWorkerRootOwnerPid(rootPath);
|
||||
if (ownerPid !== null && isProcessAlive(ownerPid)) return true;
|
||||
function hasCurrentRunToken(ownerInfo) {
|
||||
const currentToken = process.env[FUSION_TEST_RUN_TOKEN_ENV];
|
||||
return Boolean(ownerInfo?.runToken && currentToken && ownerInfo.runToken === currentToken);
|
||||
}
|
||||
|
||||
// Backward-compatible guard for worker roots created before the owner marker
|
||||
// landed, or marker writes that failed: an alive redir-<pid> child means a
|
||||
// Vitest worker still owns temp workspaces beneath this root.
|
||||
function isFreshLegacyMarkerlessRoot(rootPath) {
|
||||
try {
|
||||
return Date.now() - statSync(rootPath).mtimeMs <= LEGACY_MARKERLESS_ACTIVE_ROOT_MAX_AGE_MS;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function isActiveFusionWorkerRoot(rootPath) {
|
||||
const ownerInfo = readWorkerRootOwnerInfo(rootPath);
|
||||
if (ownerInfo !== null && isProcessAlive(ownerInfo.pid)) {
|
||||
if (ownerInfo.pid === process.pid || hasCurrentRunToken(ownerInfo)) return true;
|
||||
// FN-6396/FN-6360 recurrence: bare pid liveness is not enough evidence.
|
||||
// macOS can recycle a dead Vitest owner's pid to an unrelated process, so
|
||||
// the pnpm-test prune must require the same-run token before preserving the
|
||||
// root. Otherwise stale fusion-test-workers-* shells survive to the after
|
||||
// check-test-isolation pass and fail the merge gate.
|
||||
}
|
||||
|
||||
// Backward-compatible guard for markerless roots. New roots are marked by
|
||||
// globalSetup or by vitest-setup's self-minted fallback path; old markerless
|
||||
// redir roots are only considered active while very fresh, preventing stale
|
||||
// redir-<pid> pid reuse from keeping orphans alive forever.
|
||||
try {
|
||||
for (const child of readdirSync(rootPath, { withFileTypes: true })) {
|
||||
if (!child.isDirectory()) continue;
|
||||
const match = /^redir-(\d+)$/.exec(child.name);
|
||||
if (match && isProcessAlive(Number.parseInt(match[1], 10))) return true;
|
||||
if (!match) continue;
|
||||
const redirPid = Number.parseInt(match[1], 10);
|
||||
if (!isProcessAlive(redirPid)) continue;
|
||||
if (redirPid === process.pid || (ownerInfo && hasCurrentRunToken(ownerInfo)) || isFreshLegacyMarkerlessRoot(rootPath)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// If we cannot inspect it, fall through to normal best-effort pruning.
|
||||
@@ -288,13 +344,45 @@ export function pruneFusionTestWorkers(maxEntries = PRUNE_MAX_ENTRIES, retryOpti
|
||||
pruneFusionTestRoots("fusion-test-workers-", maxEntries, retryOptions);
|
||||
}
|
||||
|
||||
function runMaybeIsolated(command, commandArgs, options = {}) {
|
||||
// Run a test invocation under the L2 wall-clock watchdog (async). Throws on
|
||||
// failure/timeout/signal with an `.exitCode` — same shape as `run` — so the
|
||||
// caller's catch and the `finally` cleanup below behave identically. On a
|
||||
// watchdog kill the child group is already dead, and the `finally` prune +
|
||||
// isolation post-check then reap any leaked isolated HOME (no leak slips past
|
||||
// the guard).
|
||||
async function runWatchedTest(command, commandArgs, { env, budgetMs, label } = {}) {
|
||||
const { code, signal, timedOut } = await runWithWatchdog({
|
||||
command,
|
||||
args: commandArgs,
|
||||
env: env ?? process.env,
|
||||
// Preserve the original `run`'s fixed working directory; pnpm must execute
|
||||
// from the repo root regardless of where test-changed was invoked.
|
||||
cwd: rootDir,
|
||||
budgetMs,
|
||||
label: label ?? `${command} ${commandArgs.join(" ")}`,
|
||||
log: console.error,
|
||||
spawn,
|
||||
});
|
||||
if (timedOut || signal || code !== 0) {
|
||||
const reason = timedOut ? "watchdog timeout" : signal ? `signal ${signal}` : `exit code ${code}`;
|
||||
const error = new Error(`${command} ${commandArgs.join(" ")} failed (${reason})`);
|
||||
error.exitCode = timedOut ? 124 : signal ? 1 : code ?? 1;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function runMaybeIsolated(command, commandArgs, options = {}) {
|
||||
const enabled = shouldRunIsolationGuard();
|
||||
const env = options.env ?? process.env;
|
||||
const { onBeforeAfterCheck, ...spawnOptions } = options;
|
||||
/*
|
||||
* FNXC:TestInfrastructure 2026-06-15-12:12:
|
||||
* Conflict resolution for PR #1669 must keep main's per-run temp-root token so cleanup can distinguish live roots while still passing watchdog budgets and labels into async test invocations.
|
||||
*/
|
||||
const env = { ...(options.env ?? process.env), [FUSION_TEST_RUN_TOKEN_ENV]: ensureFusionTestRunToken(options.env ?? process.env) };
|
||||
const { onBeforeAfterCheck, budgetMs, label, ...spawnOptions } = options;
|
||||
void spawnOptions; // cwd/stdio defaults live in the watchdog/spawn path now
|
||||
if (enabled) runIsolationCheck(true, env, /* fastBefore */ true);
|
||||
try {
|
||||
run(command, commandArgs, spawnOptions);
|
||||
await runWatchedTest(command, commandArgs, { env, budgetMs, label });
|
||||
} finally {
|
||||
if (typeof onBeforeAfterCheck === "function") {
|
||||
onBeforeAfterCheck();
|
||||
@@ -1096,8 +1184,19 @@ export function createIsolatedHomeEnv(env = process.env) {
|
||||
return { env: nextEnv, isolatedHome };
|
||||
}
|
||||
|
||||
export function createTestProcessEnv(env = process.env) {
|
||||
/*
|
||||
FNXC:TestInfrastructure 2026-06-17-17:02:
|
||||
Developer shells and release scripts can export NODE_ENV=production, but Vitest must resolve React, Testing Library, and Vite transforms through their test/development paths. Normalize spawned test processes here so pnpm test cannot inherit production React test-utils and stall/fail jsdom lanes.
|
||||
*/
|
||||
return {
|
||||
...env,
|
||||
NODE_ENV: "test",
|
||||
};
|
||||
}
|
||||
|
||||
const fullSuiteEnv = {
|
||||
...process.env,
|
||||
...createTestProcessEnv(process.env),
|
||||
FUSION_TEST_TOTAL_WORKERS: process.env.FUSION_TEST_TOTAL_WORKERS || String(totalWorkers),
|
||||
FUSION_TEST_CONCURRENCY: process.env.FUSION_TEST_CONCURRENCY || String(concurrency),
|
||||
};
|
||||
@@ -1160,7 +1259,7 @@ export function normalizeForwardedArgs(argv) {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function main(argv = process.argv.slice(2)) {
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
// The full suite is explicit opt-in ONLY (--full / FUSION_TEST_FULL=1).
|
||||
// CI no longer routes through this script (the gate job runs `pnpm
|
||||
// test:gate`; the demoted tier runs `test:ci:shard` in full-suite.yml), so
|
||||
@@ -1291,9 +1390,11 @@ export function main(argv = process.argv.slice(2)) {
|
||||
|
||||
if (plan.mode === "full") {
|
||||
// Explicit opt-in only ("forced": --full / FUSION_TEST_FULL=1).
|
||||
runMaybeIsolated("pnpm", [`-r`, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
|
||||
await runMaybeIsolated("pnpm", [`-r`, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
|
||||
env: isolatedHomeEnv,
|
||||
onBeforeAfterCheck: cleanupIsolatedHome,
|
||||
budgetMs: FULL_SUITE_BUDGET_MS,
|
||||
label: "test:full (-r)",
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -1312,9 +1413,11 @@ export function main(argv = process.argv.slice(2)) {
|
||||
}
|
||||
console.log("[test-changed] need the full sweep instead? run `pnpm test:full` (explicit opt-in).");
|
||||
|
||||
runMaybeIsolated("pnpm", ["test:gate"], {
|
||||
await runMaybeIsolated("pnpm", ["test:gate"], {
|
||||
env: isolatedHomeEnv,
|
||||
onBeforeAfterCheck: cleanupIsolatedHome,
|
||||
budgetMs: deriveBudgetMs({ klass: "changed" }),
|
||||
label: "test:gate",
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -1327,7 +1430,11 @@ export function main(argv = process.argv.slice(2)) {
|
||||
// Run the gate under the same isolation guard as the affected set — a gate
|
||||
// suite leak must trip the checker, not silently become the "before" state
|
||||
// of the later run.
|
||||
runMaybeIsolated("pnpm", ["test:gate"], { env: isolatedHomeEnv });
|
||||
await runMaybeIsolated("pnpm", ["test:gate"], {
|
||||
env: isolatedHomeEnv,
|
||||
budgetMs: deriveBudgetMs({ klass: "changed" }),
|
||||
label: "test:gate (pre-affected)",
|
||||
});
|
||||
|
||||
const filterArgs = activePackages.flatMap((pkg) => ["--filter", pkg]);
|
||||
console.log(`[test-changed] running tests for changed packages: ${activePackages.join(", ")}`);
|
||||
@@ -1335,9 +1442,14 @@ export function main(argv = process.argv.slice(2)) {
|
||||
console.log(`[test-changed] skipping cached packages: ${cachedPackages.join(", ")}`);
|
||||
}
|
||||
|
||||
runMaybeIsolated("pnpm", [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
|
||||
await runMaybeIsolated("pnpm", [...filterArgs, `--workspace-concurrency=${workspaceConcurrency}`, "test", ...forwardedArgs], {
|
||||
env: isolatedHomeEnv,
|
||||
onBeforeAfterCheck: cleanupIsolatedHome,
|
||||
// Affected sets can include dashboard (13 inner-watchdog'd lanes); use the
|
||||
// generous full-suite backstop rather than the tight changed ceiling so a
|
||||
// legitimately long local run is never false-killed.
|
||||
budgetMs: FULL_SUITE_BUDGET_MS,
|
||||
label: `affected: ${activePackages.join(", ")}`,
|
||||
});
|
||||
|
||||
// Tests passed — record in cache (never cache failures; process.exit on failure above).
|
||||
@@ -1353,12 +1465,11 @@ export function main(argv = process.argv.slice(2)) {
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === currentFilePath) {
|
||||
try {
|
||||
main();
|
||||
} catch (error) {
|
||||
main().catch((error) => {
|
||||
if (error?.exitCode) {
|
||||
process.exit(error.exitCode);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
192
scripts/test-feedback-baseline.mjs
Normal file
192
scripts/test-feedback-baseline.mjs
Normal file
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(currentFilePath), "..");
|
||||
|
||||
export const DEFAULT_TIMINGS_PATH = "scripts/test-timings.json";
|
||||
export const DEFAULT_QUARANTINE_PATH = "scripts/lib/test-quarantine.json";
|
||||
export const DEFAULT_BASELINES_PATH = "docs/test-feedback-loop-baselines.json";
|
||||
export const DEFAULT_MARKDOWN_PATH = "docs/test-feedback-loop-baseline.md";
|
||||
|
||||
function readJson(relativePath, fallback = null, rootDir = repoRoot) {
|
||||
const absolutePath = path.join(rootDir, relativePath);
|
||||
if (!existsSync(absolutePath)) return fallback;
|
||||
return JSON.parse(readFileSync(absolutePath, "utf8"));
|
||||
}
|
||||
|
||||
function writeJson(relativePath, value, rootDir = repoRoot) {
|
||||
const absolutePath = path.join(rootDir, relativePath);
|
||||
mkdirSync(path.dirname(absolutePath), { recursive: true });
|
||||
writeFileSync(absolutePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function writeText(relativePath, value, rootDir = repoRoot) {
|
||||
const absolutePath = path.join(rootDir, relativePath);
|
||||
mkdirSync(path.dirname(absolutePath), { recursive: true });
|
||||
writeFileSync(absolutePath, value, "utf8");
|
||||
}
|
||||
|
||||
function normalizeMs(value) {
|
||||
if (value == null || value === "") return null;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed) || parsed < 0) {
|
||||
throw new Error(`Expected a non-negative millisecond value, got ${value}`);
|
||||
}
|
||||
return Math.round(parsed);
|
||||
}
|
||||
|
||||
function formatDuration(ms) {
|
||||
if (ms == null) return "pending measurement";
|
||||
const sign = ms < 0 ? "-" : "";
|
||||
const absoluteMs = Math.abs(ms);
|
||||
if (absoluteMs < 1000) return `${sign}${absoluteMs}ms`;
|
||||
const seconds = absoluteMs / 1000;
|
||||
if (seconds < 60) return `${sign}${seconds.toFixed(1)}s`;
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remaining = Math.round(seconds - minutes * 60);
|
||||
return `${sign}${minutes}m ${String(remaining).padStart(2, "0")}s`;
|
||||
}
|
||||
|
||||
function isoWeek(date) {
|
||||
const working = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
||||
const day = working.getUTCDay() || 7;
|
||||
working.setUTCDate(working.getUTCDate() + 4 - day);
|
||||
const yearStart = new Date(Date.UTC(working.getUTCFullYear(), 0, 1));
|
||||
const week = Math.ceil(((working - yearStart) / 86_400_000 + 1) / 7);
|
||||
return `${working.getUTCFullYear()}-W${String(week).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
export function collectSlowestFiles(timings, limit = 20) {
|
||||
const rows = [];
|
||||
for (const [packageName, packageTiming] of Object.entries(timings?.packages ?? {})) {
|
||||
for (const [file, durationMs] of Object.entries(packageTiming?.files ?? {})) {
|
||||
rows.push({ packageName, file, durationMs: Number(durationMs) || 0 });
|
||||
}
|
||||
}
|
||||
|
||||
rows.sort((a, b) => b.durationMs - a.durationMs || a.file.localeCompare(b.file));
|
||||
return rows.slice(0, limit);
|
||||
}
|
||||
|
||||
export function collectFlakeSummary(quarantine) {
|
||||
const entries = Array.isArray(quarantine?.entries) ? quarantine.entries : [];
|
||||
const uniqueFiles = [...new Set(entries.map((entry) => entry.file).filter(Boolean))].sort();
|
||||
return {
|
||||
flakeCount: entries.length,
|
||||
uniqueQuarantinedFileCount: uniqueFiles.length,
|
||||
quarantinedFiles: uniqueFiles,
|
||||
};
|
||||
}
|
||||
|
||||
export function createBaseline({ now = new Date(), gateWallTimeMs = null, pnpmTestWallTimeMs = null, timings, quarantine, notes = "" } = {}) {
|
||||
const slowest20 = collectSlowestFiles(timings, 20);
|
||||
const flakeSummary = collectFlakeSummary(quarantine);
|
||||
return {
|
||||
capturedAt: now.toISOString(),
|
||||
cycle: isoWeek(now),
|
||||
gateWallTimeMs: normalizeMs(gateWallTimeMs),
|
||||
pnpmTestWallTimeMs: normalizeMs(pnpmTestWallTimeMs),
|
||||
timingSnapshotCapturedAt: timings?.capturedAt ?? null,
|
||||
slowest20,
|
||||
...flakeSummary,
|
||||
notes,
|
||||
};
|
||||
}
|
||||
|
||||
function trendDelta(latest, previous, field) {
|
||||
if (!previous || latest?.[field] == null || previous?.[field] == null) return "n/a";
|
||||
const delta = latest[field] - previous[field];
|
||||
const sign = delta > 0 ? "+" : "";
|
||||
return `${sign}${formatDuration(delta)}`;
|
||||
}
|
||||
|
||||
export function renderMarkdown(baselines) {
|
||||
const sorted = [...baselines].sort((a, b) => String(a.capturedAt).localeCompare(String(b.capturedAt)));
|
||||
const latest = sorted.at(-1);
|
||||
const previous = sorted.at(-2);
|
||||
|
||||
const slowRows = (latest?.slowest20 ?? [])
|
||||
.map((row, index) => `| ${index + 1} | \`${row.file}\` | ${row.packageName} | ${formatDuration(row.durationMs)} |`)
|
||||
.join("\n");
|
||||
const trendRows = sorted
|
||||
.map((row) => `| ${row.cycle} | ${row.capturedAt} | ${formatDuration(row.gateWallTimeMs)} | ${formatDuration(row.pnpmTestWallTimeMs)} | ${row.flakeCount ?? 0} | ${row.uniqueQuarantinedFileCount ?? 0} |`)
|
||||
.join("\n");
|
||||
|
||||
return `# Test feedback-loop baseline\n\n> Publish this page's latest-cycle summary in #leads each week. The objective is signal-per-second: keep the merge gate thin, keep \`pnpm test\` flat or faster, and ratchet flaky/low-signal tests toward rescue or deletion.\n\n## Latest #leads summary\n\n- Cycle: **${latest?.cycle ?? "none"}** (${latest?.capturedAt ?? "not captured"})\n- Gate suite wall-time: **${formatDuration(latest?.gateWallTimeMs)}** (trend: ${trendDelta(latest, previous, "gateWallTimeMs")})\n- \`pnpm test\` wall-time: **${formatDuration(latest?.pnpmTestWallTimeMs)}** (trend: ${trendDelta(latest, previous, "pnpmTestWallTimeMs")})\n- Flake/quarantine count: **${latest?.flakeCount ?? 0}** ledger entr${(latest?.flakeCount ?? 0) === 1 ? "y" : "ies"} across **${latest?.uniqueQuarantinedFileCount ?? 0}** file${(latest?.uniqueQuarantinedFileCount ?? 0) === 1 ? "" : "s"}\n- Timing snapshot source: \`${DEFAULT_TIMINGS_PATH}\` captured at **${latest?.timingSnapshotCapturedAt ?? "unknown"}**\n\n## Slowest 20 test files\n\n| Rank | File | Package | Duration |\n|---:|---|---|---:|\n${slowRows || "| — | — | — | — |"}\n\n## Trend\n\n| Cycle | Captured at | Gate suite | \`pnpm test\` | Quarantine entries | Quarantined files |\n|---|---|---:|---:|---:|---:|\n${trendRows || "| — | — | — | — | — | — |"}\n\n## Operating rules\n\n- Record a new row weekly with \`node scripts/test-feedback-baseline.mjs --record --gate-ms <ms> --test-ms <ms>\` after running \`pnpm test:gate\` and \`pnpm test\`.\n- Use the slowest-file list as the candidate queue for FN-5048 rewrites or deletion-ratchet review; do not add coverage for its own sake.\n- Quarantined tests remain on the 14-day rescue-or-delete clock in \`scripts/lib/test-quarantine.json\`; deleting a low-signal expired test is a valid positive outcome.\n`;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { record: false, printLeads: false, gateMs: null, testMs: null, notes: "" };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--record") args.record = true;
|
||||
else if (arg === "--print-leads") args.printLeads = true;
|
||||
else if (arg === "--gate-ms") args.gateMs = argv[++index];
|
||||
else if (arg === "--test-ms") args.testMs = argv[++index];
|
||||
else if (arg === "--notes") args.notes = argv[++index] ?? "";
|
||||
else if (arg === "--help" || arg === "-h") args.help = true;
|
||||
else throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function renderLeadsSummary(baselines) {
|
||||
const latest = [...baselines].sort((a, b) => String(a.capturedAt).localeCompare(String(b.capturedAt))).at(-1);
|
||||
if (!latest) return "No test feedback-loop baseline has been recorded yet.";
|
||||
const topFive = (latest.slowest20 ?? []).slice(0, 5).map((row, index) => `${index + 1}. ${row.file} (${formatDuration(row.durationMs)})`).join("; ");
|
||||
return `Test feedback-loop ${latest.cycle}: gate ${formatDuration(latest.gateWallTimeMs)}, pnpm test ${formatDuration(latest.pnpmTestWallTimeMs)}, quarantine ledger ${latest.flakeCount} entries/${latest.uniqueQuarantinedFileCount} files. Slowest files: ${topFive || "none"}.`;
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2), { rootDir = repoRoot, stdout = process.stdout, stderr = process.stderr } = {}) {
|
||||
let args;
|
||||
try {
|
||||
args = parseArgs(argv);
|
||||
} catch (err) {
|
||||
stderr.write(`${err.message}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (args.help) {
|
||||
stdout.write("Usage: node scripts/test-feedback-baseline.mjs [--record --gate-ms <ms> --test-ms <ms>] [--print-leads]\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
const store = readJson(DEFAULT_BASELINES_PATH, { baselines: [] }, rootDir);
|
||||
const baselines = Array.isArray(store?.baselines) ? store.baselines : [];
|
||||
|
||||
if (args.record || baselines.length === 0) {
|
||||
const timings = readJson(DEFAULT_TIMINGS_PATH, { packages: {} }, rootDir);
|
||||
const quarantine = readJson(DEFAULT_QUARANTINE_PATH, { entries: [] }, rootDir);
|
||||
baselines.push(createBaseline({
|
||||
gateWallTimeMs: args.gateMs,
|
||||
pnpmTestWallTimeMs: args.testMs,
|
||||
timings,
|
||||
quarantine,
|
||||
notes: args.notes,
|
||||
}));
|
||||
writeJson(DEFAULT_BASELINES_PATH, { baselines }, rootDir);
|
||||
}
|
||||
|
||||
const markdown = renderMarkdown(baselines);
|
||||
writeText(DEFAULT_MARKDOWN_PATH, markdown, rootDir);
|
||||
|
||||
if (args.printLeads) {
|
||||
stdout.write(`${renderLeadsSummary(baselines)}\n`);
|
||||
} else {
|
||||
stdout.write(`Updated ${DEFAULT_MARKDOWN_PATH}\n`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
/*
|
||||
FNXC:TestFeedbackVelocity 2026-06-17-18:30:
|
||||
The CEO mandate requires a weekly #leads-visible baseline for test signal-per-second, not more coverage. Keep this script stdlib-only so any engineer or scheduled job can refresh gate time, pnpm-test time, slowest files, and quarantine count without booting Fusion services.
|
||||
*/
|
||||
const exitCode = await main();
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
339
scripts/test-velocity-baseline.mjs
Executable file
339
scripts/test-velocity-baseline.mjs
Executable file
@@ -0,0 +1,339 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { spawn } from "node:child_process";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(currentFilePath), "..");
|
||||
|
||||
export const DEFAULT_TIMINGS_PATH = "scripts/test-timings.json";
|
||||
export const DEFAULT_QUARANTINE_PATH = "scripts/lib/test-quarantine.json";
|
||||
export const DEFAULT_HISTORY_PATH = "scripts/test-velocity-history.json";
|
||||
export const DEFAULT_REPORT_PATH = "docs/test-velocity-baseline.md";
|
||||
export const DEFAULT_MEASURE_TIMEOUT_MS = 10 * 60 * 1000;
|
||||
export const DELETION_CLOCK_DAYS = 14;
|
||||
|
||||
const MEASURE_COMMANDS = [
|
||||
{ key: "gateMs", label: "Merge gate (`pnpm test:gate`)", command: "pnpm", args: ["test:gate"] },
|
||||
{ key: "bootSmokeMs", label: "Boot smoke (`pnpm smoke:boot`)", command: "pnpm", args: ["smoke:boot"] },
|
||||
{ key: "testMs", label: "Changed-only tests (`pnpm test`)", command: "pnpm", args: ["test"] },
|
||||
];
|
||||
|
||||
function readJson(relativePath, fallback = null, rootDir = repoRoot) {
|
||||
const absolutePath = path.join(rootDir, relativePath);
|
||||
if (!existsSync(absolutePath)) return fallback;
|
||||
return JSON.parse(readFileSync(absolutePath, "utf8"));
|
||||
}
|
||||
|
||||
function writeJson(relativePath, value, rootDir = repoRoot) {
|
||||
const absolutePath = path.join(rootDir, relativePath);
|
||||
mkdirSync(path.dirname(absolutePath), { recursive: true });
|
||||
writeFileSync(absolutePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function writeText(relativePath, value, rootDir = repoRoot) {
|
||||
const absolutePath = path.join(rootDir, relativePath);
|
||||
mkdirSync(path.dirname(absolutePath), { recursive: true });
|
||||
writeFileSync(absolutePath, value, "utf8");
|
||||
}
|
||||
|
||||
function normalizeMs(value) {
|
||||
if (value == null || value === "") return null;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed) || parsed < 0) {
|
||||
throw new Error(`Expected a non-negative millisecond value, got ${value}`);
|
||||
}
|
||||
return Math.round(parsed);
|
||||
}
|
||||
|
||||
function toDate(value) {
|
||||
if (!value) return null;
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
||||
}
|
||||
|
||||
function ageDays(quarantinedAt, now) {
|
||||
const quarantinedAtDate = toDate(quarantinedAt);
|
||||
if (!quarantinedAtDate) return null;
|
||||
return Math.floor((now.getTime() - quarantinedAtDate.getTime()) / 86_400_000);
|
||||
}
|
||||
|
||||
function isoWeek(date) {
|
||||
const working = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
||||
const day = working.getUTCDay() || 7;
|
||||
working.setUTCDate(working.getUTCDate() + 4 - day);
|
||||
const yearStart = new Date(Date.UTC(working.getUTCFullYear(), 0, 1));
|
||||
const week = Math.ceil(((working - yearStart) / 86_400_000 + 1) / 7);
|
||||
return `${working.getUTCFullYear()}-W${String(week).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
export function formatDuration(ms) {
|
||||
if (ms == null) return "unavailable";
|
||||
const rounded = Math.round(ms);
|
||||
const sign = rounded < 0 ? "-" : "";
|
||||
const absoluteMs = Math.abs(rounded);
|
||||
if (absoluteMs < 1000) return `${sign}${absoluteMs}ms`;
|
||||
const seconds = absoluteMs / 1000;
|
||||
if (seconds < 60) return `${sign}${seconds.toFixed(1)}s`;
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remainingSeconds = Math.round(seconds - minutes * 60);
|
||||
return `${sign}${minutes}m ${String(remainingSeconds).padStart(2, "0")}s`;
|
||||
}
|
||||
|
||||
export function readQuarantineCount(json, { now = new Date() } = {}) {
|
||||
const entries = Array.isArray(json?.entries) ? json.entries : [];
|
||||
const buckets = {
|
||||
"0-6d": 0,
|
||||
"7-13d": 0,
|
||||
deletionDue: 0,
|
||||
unknown: 0,
|
||||
};
|
||||
const deletionDueEntries = [];
|
||||
|
||||
for (const entry of entries) {
|
||||
const age = ageDays(entry?.quarantinedAt, now);
|
||||
const normalized = {
|
||||
file: entry?.file ?? "unknown",
|
||||
quarantinedAt: entry?.quarantinedAt ?? null,
|
||||
ageDays: age,
|
||||
};
|
||||
if (age == null || age < 0) {
|
||||
buckets.unknown += 1;
|
||||
} else if (age >= DELETION_CLOCK_DAYS) {
|
||||
buckets.deletionDue += 1;
|
||||
deletionDueEntries.push(normalized);
|
||||
} else if (age >= 7) {
|
||||
buckets["7-13d"] += 1;
|
||||
} else {
|
||||
buckets["0-6d"] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
total: entries.length,
|
||||
byAgeBucket: buckets,
|
||||
deletionDueEntries,
|
||||
deletionDueCount: deletionDueEntries.length,
|
||||
};
|
||||
}
|
||||
|
||||
export function topSlowestFiles(timingsJson, n = 20) {
|
||||
const rows = [];
|
||||
for (const [packageName, packageTiming] of Object.entries(timingsJson?.packages ?? {})) {
|
||||
for (const [file, ms] of Object.entries(packageTiming?.files ?? {})) {
|
||||
const parsed = Number(ms);
|
||||
rows.push({ file, ms: Number.isFinite(parsed) ? parsed : 0, package: packageName });
|
||||
}
|
||||
}
|
||||
|
||||
rows.sort((a, b) => b.ms - a.ms || a.package.localeCompare(b.package) || a.file.localeCompare(b.file));
|
||||
return rows.slice(0, n);
|
||||
}
|
||||
|
||||
function delta(latest, previous, field) {
|
||||
if (!previous || latest?.[field] == null || previous?.[field] == null) return "n/a";
|
||||
const diff = latest[field] - previous[field];
|
||||
const sign = diff > 0 ? "+" : "";
|
||||
return `${sign}${formatDuration(diff)}`;
|
||||
}
|
||||
|
||||
function renderMetricRow(name, latest, previous, field) {
|
||||
return `| ${name} | ${formatDuration(latest?.[field])} | ${delta(latest, previous, field)} |`;
|
||||
}
|
||||
|
||||
function trendCell(current, prior) {
|
||||
if (current == null || prior == null) return "n/a";
|
||||
const diff = current - prior;
|
||||
return `${diff > 0 ? "+" : ""}${diff}`;
|
||||
}
|
||||
|
||||
export function renderReport({ gateMs, bootSmokeMs, testMs, slowest = [], quarantine, capturedAt, previous = null, measurementFailures = [], timingSnapshotCapturedAt = null } = {}) {
|
||||
const latest = {
|
||||
gateMs: normalizeMs(gateMs),
|
||||
bootSmokeMs: normalizeMs(bootSmokeMs),
|
||||
testMs: normalizeMs(testMs),
|
||||
quarantineCount: quarantine?.total ?? quarantine?.quarantineCount ?? 0,
|
||||
capturedAt: capturedAt ?? new Date().toISOString(),
|
||||
};
|
||||
const cycle = isoWeek(new Date(latest.capturedAt));
|
||||
const slowRows = slowest
|
||||
.map((row, index) => `| ${index + 1} | \`${row.file}\` | ${row.package} | ${formatDuration(row.ms)} |`)
|
||||
.join("\n");
|
||||
const dueRows = (quarantine?.deletionDueEntries ?? [])
|
||||
.map((entry) => `| \`${entry.file}\` | ${entry.quarantinedAt ?? "unknown"} | ${entry.ageDays ?? "unknown"} |`)
|
||||
.join("\n");
|
||||
const failures = measurementFailures.length > 0
|
||||
? measurementFailures.map((failure) => `- ${failure.label}: ${failure.status}`).join("\n")
|
||||
: "- None recorded.";
|
||||
const previousRows = previous
|
||||
? `| Previous | ${previous.capturedAt ?? "unknown"} | ${formatDuration(previous.gateMs)} | ${formatDuration(previous.bootSmokeMs)} | ${formatDuration(previous.testMs)} | ${previous.quarantineCount ?? "n/a"} |\n| Latest | ${latest.capturedAt} | ${formatDuration(latest.gateMs)} | ${formatDuration(latest.bootSmokeMs)} | ${formatDuration(latest.testMs)} | ${latest.quarantineCount} |\n| Delta | — | ${delta(latest, previous, "gateMs")} | ${delta(latest, previous, "bootSmokeMs")} | ${delta(latest, previous, "testMs")} | ${trendCell(latest.quarantineCount, previous.quarantineCount)} |`
|
||||
: `| Previous | _(seed baseline)_ | — | — | — | — |\n| Latest | ${latest.capturedAt} | ${formatDuration(latest.gateMs)} | ${formatDuration(latest.bootSmokeMs)} | ${formatDuration(latest.testMs)} | ${latest.quarantineCount} |\n| Delta | — | n/a | n/a | n/a | n/a |`;
|
||||
|
||||
return `# Test velocity baseline\n\n> Weekly FN-6612 signal-per-second baseline. Measure and report feedback-loop velocity; do **not** add slow tests or wire this report into blocking PR checks. The merge gate remains the existing thin Lint, Typecheck, Build, and Gate path.\n\n## Latest baseline\n\n- Cycle: **${cycle}**\n- Captured at: **${latest.capturedAt}**\n- Timing snapshot: \`${DEFAULT_TIMINGS_PATH}\`${timingSnapshotCapturedAt ? ` captured at **${timingSnapshotCapturedAt}**` : ""}\n- Quarantine ledger: \`${DEFAULT_QUARANTINE_PATH}\`\n\n## Metrics\n\n| Metric | Current | Delta vs previous |\n|---|---:|---:|\n${renderMetricRow("Merge gate wall-time (`pnpm test:gate`)", latest, previous, "gateMs")}\n${renderMetricRow("Boot smoke wall-time (`pnpm smoke:boot`)", latest, previous, "bootSmokeMs")}\n${renderMetricRow("Changed-only test wall-time (`pnpm test`)", latest, previous, "testMs")}\n| Quarantine / flake count | ${latest.quarantineCount} | ${trendCell(latest.quarantineCount, previous?.quarantineCount)} |\n| Deletion-due quarantines | ${quarantine?.deletionDueCount ?? 0} | n/a |\n\n## Measurement failures\n\n${failures}\n\n## Slowest 20 test files\n\n| Rank | File | Package | Duration |\n|---:|---|---|---:|\n${slowRows || "| — | — | — | — |"}\n\n## Quarantine age buckets\n\n| Age bucket | Count |\n|---|---:|\n| 0-6 days | ${quarantine?.byAgeBucket?.["0-6d"] ?? 0} |\n| 7-13 days | ${quarantine?.byAgeBucket?.["7-13d"] ?? 0} |\n| deletion due (>=14 days) | ${quarantine?.byAgeBucket?.deletionDue ?? 0} |\n| unknown/future | ${quarantine?.byAgeBucket?.unknown ?? 0} |\n\n### Deletion-due entries\n\n| File | Quarantined at | Age (days) |\n|---|---:|---:|\n${dueRows || "| — | — | — |"}\n\n## Before / after trend\n\n| Row | Captured at | Gate | Boot smoke | \`pnpm test\` | Quarantine count |\n|---|---|---:|---:|---:|---:|\n${previousRows}\n\n_Future weekly rows append to \`${DEFAULT_HISTORY_PATH}\`; compare the latest row against the previous row before posting to #leads._\n\n## Post to #leads\n\n\`\`\`text\nFN-6612 weekly test velocity: gate ${formatDuration(latest.gateMs)} (${delta(latest, previous, "gateMs")}), boot smoke ${formatDuration(latest.bootSmokeMs)} (${delta(latest, previous, "bootSmokeMs")}), pnpm test ${formatDuration(latest.testMs)} (${delta(latest, previous, "testMs")}), quarantine ledger ${latest.quarantineCount} (${trendCell(latest.quarantineCount, previous?.quarantineCount)}). Slowest file: ${slowest[0]?.file ?? "none"} at ${formatDuration(slowest[0]?.ms)}. Deletion-due quarantines: ${quarantine?.deletionDueCount ?? 0}.\n\`\`\`\n\n## How to refresh\n\n\`\`\`bash\npnpm test:velocity -- --measure --write-report\n\`\`\`\n\nReport-only regeneration is cheap and does not run any suite:\n\n\`\`\`bash\npnpm test:velocity\n\`\`\`\n`;
|
||||
}
|
||||
|
||||
function historyEntries(history) {
|
||||
if (Array.isArray(history)) return history;
|
||||
if (Array.isArray(history?.entries)) return history.entries;
|
||||
return [];
|
||||
}
|
||||
|
||||
function createEntry({ capturedAt = new Date().toISOString(), gateMs = null, bootSmokeMs = null, testMs = null, quarantine, slowest, measurementFailures = [], timingSnapshotCapturedAt = null }) {
|
||||
return {
|
||||
capturedAt,
|
||||
gateMs: normalizeMs(gateMs),
|
||||
bootSmokeMs: normalizeMs(bootSmokeMs),
|
||||
testMs: normalizeMs(testMs),
|
||||
quarantineCount: quarantine?.total ?? 0,
|
||||
slowestTop20: slowest,
|
||||
measurementFailures,
|
||||
timingSnapshotCapturedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { measure: false, writeReport: false, reportOnly: true, timeoutMs: DEFAULT_MEASURE_TIMEOUT_MS, help: false };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--") continue;
|
||||
else if (arg === "--measure") args.measure = true;
|
||||
else if (arg === "--write-report") args.writeReport = true;
|
||||
else if (arg === "--report-only") args.reportOnly = true;
|
||||
else if (arg === "--timeout-ms") args.timeoutMs = Number(argv[++index]);
|
||||
else if (arg === "--help" || arg === "-h") args.help = true;
|
||||
else throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
if (!Number.isFinite(args.timeoutMs) || args.timeoutMs <= 0) {
|
||||
throw new Error(`Expected --timeout-ms to be a positive number, got ${args.timeoutMs}`);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
async function timeCommand({ command, args, label, timeoutMs, cwd, stdout, stderr }) {
|
||||
const started = performance.now();
|
||||
stderr.write(`[test-velocity] measuring ${label}\n`);
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command, args, { cwd, shell: false, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let settled = false;
|
||||
const timer = setTimeout(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
child.kill("SIGTERM");
|
||||
const elapsedMs = Math.round(performance.now() - started);
|
||||
resolve({ ms: null, failure: { label, status: `timeout after ${formatDuration(timeoutMs)} (${elapsedMs}ms elapsed)` } });
|
||||
}, timeoutMs);
|
||||
|
||||
child.stdout.on("data", (chunk) => stdout.write(chunk));
|
||||
child.stderr.on("data", (chunk) => stderr.write(chunk));
|
||||
child.on("error", (error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve({ ms: null, failure: { label, status: `spawn error: ${error.message}` } });
|
||||
});
|
||||
child.on("close", (code, signal) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
const elapsedMs = Math.round(performance.now() - started);
|
||||
if (code === 0 && !signal) {
|
||||
resolve({ ms: elapsedMs, failure: null });
|
||||
} else {
|
||||
resolve({ ms: null, failure: { label, status: signal ? `signal ${signal} after ${formatDuration(elapsedMs)}` : `exit ${code} after ${formatDuration(elapsedMs)}` } });
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function measureCommands({ timeoutMs, cwd, stdout, stderr }) {
|
||||
const results = {};
|
||||
const failures = [];
|
||||
for (const measurement of MEASURE_COMMANDS) {
|
||||
const result = await timeCommand({ ...measurement, timeoutMs, cwd, stdout, stderr });
|
||||
results[measurement.key] = result.ms;
|
||||
if (result.failure) failures.push(result.failure);
|
||||
}
|
||||
return { ...results, measurementFailures: failures };
|
||||
}
|
||||
|
||||
function renderFromEntry(entry, previous, quarantine) {
|
||||
return renderReport({
|
||||
gateMs: entry?.gateMs,
|
||||
bootSmokeMs: entry?.bootSmokeMs,
|
||||
testMs: entry?.testMs,
|
||||
slowest: entry?.slowestTop20 ?? [],
|
||||
quarantine: quarantine ?? { total: entry?.quarantineCount ?? 0, byAgeBucket: {}, deletionDueEntries: [], deletionDueCount: 0 },
|
||||
capturedAt: entry?.capturedAt,
|
||||
previous,
|
||||
measurementFailures: entry?.measurementFailures ?? [],
|
||||
timingSnapshotCapturedAt: entry?.timingSnapshotCapturedAt ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2), { rootDir = repoRoot, stdout = process.stdout, stderr = process.stderr, now = new Date() } = {}) {
|
||||
let args;
|
||||
try {
|
||||
args = parseArgs(argv);
|
||||
} catch (error) {
|
||||
stderr.write(`${error.message}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (args.help) {
|
||||
stdout.write("Usage: node scripts/test-velocity-baseline.mjs [--measure] [--write-report] [--report-only] [--timeout-ms <ms>]\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
const history = readJson(DEFAULT_HISTORY_PATH, { entries: [] }, rootDir);
|
||||
const entries = historyEntries(history);
|
||||
const timings = readJson(DEFAULT_TIMINGS_PATH, { packages: {} }, rootDir);
|
||||
const quarantineJson = readJson(DEFAULT_QUARANTINE_PATH, { entries: [] }, rootDir);
|
||||
const quarantine = readQuarantineCount(quarantineJson, { now });
|
||||
const slowest = topSlowestFiles(timings, 20);
|
||||
|
||||
if (args.measure) {
|
||||
const measured = await measureCommands({ timeoutMs: args.timeoutMs, cwd: rootDir, stdout, stderr });
|
||||
const entry = createEntry({
|
||||
capturedAt: now.toISOString(),
|
||||
gateMs: measured.gateMs,
|
||||
bootSmokeMs: measured.bootSmokeMs,
|
||||
testMs: measured.testMs,
|
||||
quarantine,
|
||||
slowest,
|
||||
measurementFailures: measured.measurementFailures,
|
||||
timingSnapshotCapturedAt: timings?.capturedAt ?? null,
|
||||
});
|
||||
entries.push(entry);
|
||||
writeJson(DEFAULT_HISTORY_PATH, { entries }, rootDir);
|
||||
}
|
||||
|
||||
const latest = entries.at(-1) ?? createEntry({
|
||||
capturedAt: now.toISOString(),
|
||||
quarantine,
|
||||
slowest,
|
||||
timingSnapshotCapturedAt: timings?.capturedAt ?? null,
|
||||
});
|
||||
const previous = entries.length > 1 ? entries.at(-2) : null;
|
||||
const report = renderFromEntry(latest, previous, quarantine);
|
||||
|
||||
if (args.writeReport || args.reportOnly) {
|
||||
writeText(DEFAULT_REPORT_PATH, report, rootDir);
|
||||
stdout.write(`Updated ${DEFAULT_REPORT_PATH}\n`);
|
||||
} else {
|
||||
stdout.write(report);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
/*
|
||||
FNXC:TestVelocityBaseline 2026-06-17-00:00:
|
||||
FN-6612 requires a weekly signal-per-second baseline for gate time, boot smoke time, changed-only test time, slowest files, and quarantine age. This script measures and reports those values only; it must stay out of blocking PR checks so the merge gate remains thin.
|
||||
*/
|
||||
const exitCode = await main();
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
342
scripts/test-velocity-history.json
Normal file
342
scripts/test-velocity-history.json
Normal file
@@ -0,0 +1,342 @@
|
||||
{
|
||||
"entries": [
|
||||
{
|
||||
"capturedAt": "2026-06-18T02:53:34.158Z",
|
||||
"gateMs": 8478,
|
||||
"bootSmokeMs": null,
|
||||
"testMs": 38405,
|
||||
"quarantineCount": 2,
|
||||
"slowestTop20": [
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/shared-branch-group-lifecycle.test.ts",
|
||||
"ms": 13900,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/agent-store.test.ts",
|
||||
"ms": 11600,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-agents.test.ts",
|
||||
"ms": 11200,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/mission-store.test.ts",
|
||||
"ms": 10700,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/db.test.ts",
|
||||
"ms": 10100,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-git.test.ts",
|
||||
"ms": 9400,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-automerge-precedence.test.ts",
|
||||
"ms": 9000,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/merger-ai.test.ts",
|
||||
"ms": 8700,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-merge-routing.test.ts",
|
||||
"ms": 8400,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-promotion-gate.test.ts",
|
||||
"ms": 8400,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/task-documents.test.ts",
|
||||
"ms": 8300,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/runtimes/__tests__/in-process-runtime.test.ts",
|
||||
"ms": 7800,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/cli/src/__tests__/extension.test.ts",
|
||||
"ms": 7000,
|
||||
"package": "@runfusion/fusion"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/run-audit.test.ts",
|
||||
"ms": 6900,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-promotion.test.ts",
|
||||
"ms": 6100,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-planning.test.ts",
|
||||
"ms": 5600,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/store-merge-queue.test.ts",
|
||||
"ms": 5200,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/FileEditor.test.tsx",
|
||||
"ms": 5100,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/integration-worktree-state.test.ts",
|
||||
"ms": 4900,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/self-healing-already-merged.real-git.test.ts",
|
||||
"ms": 4900,
|
||||
"package": "@fusion/engine"
|
||||
}
|
||||
],
|
||||
"measurementFailures": [
|
||||
{
|
||||
"label": "Boot smoke (`pnpm smoke:boot`)",
|
||||
"status": "exit 1 after 411ms"
|
||||
}
|
||||
],
|
||||
"timingSnapshotCapturedAt": "2026-06-03T23:45:49.672Z"
|
||||
},
|
||||
{
|
||||
"capturedAt": "2026-06-18T03:04:28.794Z",
|
||||
"gateMs": 6177,
|
||||
"bootSmokeMs": 18220,
|
||||
"testMs": 7740,
|
||||
"quarantineCount": 2,
|
||||
"slowestTop20": [
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/shared-branch-group-lifecycle.test.ts",
|
||||
"ms": 13900,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/agent-store.test.ts",
|
||||
"ms": 11600,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-agents.test.ts",
|
||||
"ms": 11200,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/mission-store.test.ts",
|
||||
"ms": 10700,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/db.test.ts",
|
||||
"ms": 10100,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-git.test.ts",
|
||||
"ms": 9400,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-automerge-precedence.test.ts",
|
||||
"ms": 9000,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/merger-ai.test.ts",
|
||||
"ms": 8700,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-merge-routing.test.ts",
|
||||
"ms": 8400,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-promotion-gate.test.ts",
|
||||
"ms": 8400,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/task-documents.test.ts",
|
||||
"ms": 8300,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/runtimes/__tests__/in-process-runtime.test.ts",
|
||||
"ms": 7800,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/cli/src/__tests__/extension.test.ts",
|
||||
"ms": 7000,
|
||||
"package": "@runfusion/fusion"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/run-audit.test.ts",
|
||||
"ms": 6900,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-promotion.test.ts",
|
||||
"ms": 6100,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-planning.test.ts",
|
||||
"ms": 5600,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/store-merge-queue.test.ts",
|
||||
"ms": 5200,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/FileEditor.test.tsx",
|
||||
"ms": 5100,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/integration-worktree-state.test.ts",
|
||||
"ms": 4900,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/self-healing-already-merged.real-git.test.ts",
|
||||
"ms": 4900,
|
||||
"package": "@fusion/engine"
|
||||
}
|
||||
],
|
||||
"measurementFailures": [],
|
||||
"timingSnapshotCapturedAt": "2026-06-03T23:45:49.672Z"
|
||||
},
|
||||
{
|
||||
"capturedAt": "2026-06-18T16:12:01.248Z",
|
||||
"gateMs": 5398,
|
||||
"bootSmokeMs": 18097,
|
||||
"testMs": 7240,
|
||||
"quarantineCount": 0,
|
||||
"slowestTop20": [
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/shared-branch-group-lifecycle.test.ts",
|
||||
"ms": 13900,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/agent-store.test.ts",
|
||||
"ms": 11600,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-agents.test.ts",
|
||||
"ms": 11200,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/mission-store.test.ts",
|
||||
"ms": 10700,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/db.test.ts",
|
||||
"ms": 10100,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-git.test.ts",
|
||||
"ms": 9400,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-automerge-precedence.test.ts",
|
||||
"ms": 9000,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/merger-ai.test.ts",
|
||||
"ms": 8700,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-merge-routing.test.ts",
|
||||
"ms": 8400,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-promotion-gate.test.ts",
|
||||
"ms": 8400,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/task-documents.test.ts",
|
||||
"ms": 8300,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/runtimes/__tests__/in-process-runtime.test.ts",
|
||||
"ms": 7800,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/cli/src/__tests__/extension.test.ts",
|
||||
"ms": 7000,
|
||||
"package": "@runfusion/fusion"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/run-audit.test.ts",
|
||||
"ms": 6900,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/branch-group-promotion.test.ts",
|
||||
"ms": 6100,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/src/__tests__/routes-planning.test.ts",
|
||||
"ms": 5600,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/core/src/__tests__/store-merge-queue.test.ts",
|
||||
"ms": 5200,
|
||||
"package": "@fusion/core"
|
||||
},
|
||||
{
|
||||
"file": "packages/dashboard/app/components/__tests__/FileEditor.test.tsx",
|
||||
"ms": 5100,
|
||||
"package": "@fusion/dashboard"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/reliability-interactions/integration-worktree-state.test.ts",
|
||||
"ms": 4900,
|
||||
"package": "@fusion/engine"
|
||||
},
|
||||
{
|
||||
"file": "packages/engine/src/__tests__/self-healing-already-merged.real-git.test.ts",
|
||||
"ms": 4900,
|
||||
"package": "@fusion/engine"
|
||||
}
|
||||
],
|
||||
"measurementFailures": [],
|
||||
"timingSnapshotCapturedAt": "2026-06-03T23:45:49.672Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
312
scripts/workflow-reliability-release-check.mjs
Normal file
312
scripts/workflow-reliability-release-check.mjs
Normal file
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { spawn } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
const repoRoot = path.resolve(path.dirname(currentFilePath), "..");
|
||||
|
||||
export const DEFAULT_MANIFEST_PATH = "scripts/lib/workflow-reliability-release-check.json";
|
||||
export const KNOWN_PACKAGES = new Map([
|
||||
["@fusion/core", "packages/core"],
|
||||
["@fusion/dashboard", "packages/dashboard"],
|
||||
["@fusion/engine", "packages/engine"],
|
||||
]);
|
||||
export const DEFAULT_COMMAND_TIMEOUT_MS = 300_000;
|
||||
|
||||
/**
|
||||
* FNXC:CustomWorkflowReliability 2026-06-19-00:00:
|
||||
* FN-6694 makes the custom-workflow reliability release checklist executable for QA without adding it to the merge gate. Keep this harness on-demand, targeted to manifest-listed Vitest seams only, and fail closed when a manifest seam path is missing so release evidence cannot silently drift.
|
||||
*/
|
||||
export function loadManifest(manifestPath = DEFAULT_MANIFEST_PATH, { rootDir = repoRoot } = {}) {
|
||||
const absolutePath = path.isAbsolute(manifestPath) ? manifestPath : path.join(rootDir, manifestPath);
|
||||
return JSON.parse(readFileSync(absolutePath, "utf8"));
|
||||
}
|
||||
|
||||
function normalizeManual(manifest) {
|
||||
const manualEntries = Array.isArray(manifest?.manual) ? manifest.manual : [];
|
||||
return new Map(manualEntries.map((entry) => [entry?.id, entry]));
|
||||
}
|
||||
|
||||
function validateSeam(seam, item, index, seamIndex, { repoRoot: rootDir, existsSync: fileExists }) {
|
||||
const errors = [];
|
||||
const label = `checklist[${index}] ${item?.id ?? "<missing-id>"} seams[${seamIndex}]`;
|
||||
if (!seam || typeof seam !== "object") {
|
||||
errors.push(`${label} must be an object`);
|
||||
return errors;
|
||||
}
|
||||
if (!KNOWN_PACKAGES.has(seam.package)) {
|
||||
errors.push(`${label} references unknown package ${JSON.stringify(seam.package)}`);
|
||||
}
|
||||
if (!seam.file || typeof seam.file !== "string") {
|
||||
errors.push(`${label} must include a repo-relative file`);
|
||||
} else {
|
||||
const normalizedFile = path.normalize(seam.file);
|
||||
if (path.isAbsolute(seam.file) || normalizedFile.startsWith("..") || normalizedFile.includes(`${path.sep}..${path.sep}`)) {
|
||||
errors.push(`${label} file must stay within the repository: ${seam.file}`);
|
||||
} else if (!fileExists(path.join(rootDir, seam.file))) {
|
||||
errors.push(`${label} file does not exist: ${seam.file}`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
export function validateManifest(manifest, { repoRoot: rootDir = repoRoot, existsSync: fileExists = existsSync } = {}) {
|
||||
const errors = [];
|
||||
if (!manifest || typeof manifest !== "object") {
|
||||
return { ok: false, errors: ["manifest must be an object"] };
|
||||
}
|
||||
if (manifest.version !== 1) errors.push("manifest.version must be 1");
|
||||
if (!Array.isArray(manifest.checklist)) errors.push("manifest.checklist must be an array");
|
||||
|
||||
const manualById = normalizeManual(manifest);
|
||||
const checklist = Array.isArray(manifest.checklist) ? manifest.checklist : [];
|
||||
const seenIds = new Set();
|
||||
checklist.forEach((item, index) => {
|
||||
const label = `checklist[${index}]`;
|
||||
if (!item || typeof item !== "object") {
|
||||
errors.push(`${label} must be an object`);
|
||||
return;
|
||||
}
|
||||
if (!item.id || typeof item.id !== "string") {
|
||||
errors.push(`${label} must include an id`);
|
||||
} else if (seenIds.has(item.id)) {
|
||||
errors.push(`${label} duplicates id ${item.id}`);
|
||||
} else {
|
||||
seenIds.add(item.id);
|
||||
}
|
||||
if (!item.title || typeof item.title !== "string") errors.push(`${label} ${item.id ?? "<missing-id>"} must include a title`);
|
||||
if (!item.journey || typeof item.journey !== "string") errors.push(`${label} ${item.id ?? "<missing-id>"} must include a journey`);
|
||||
|
||||
const seams = Array.isArray(item.seams) ? item.seams : [];
|
||||
const manual = manualById.get(item.id);
|
||||
const manualReason = typeof manual?.automationDeferredReason === "string" ? manual.automationDeferredReason.trim() : "";
|
||||
if (seams.length === 0 && manualReason.length === 0) {
|
||||
errors.push(`${label} ${item.id ?? "<missing-id>"} must have at least one seam or a manual automationDeferredReason`);
|
||||
}
|
||||
seams.forEach((seam, seamIndex) => {
|
||||
errors.push(...validateSeam(seam, item, index, seamIndex, { repoRoot: rootDir, existsSync: fileExists }));
|
||||
});
|
||||
});
|
||||
|
||||
const manual = Array.isArray(manifest.manual) ? manifest.manual : [];
|
||||
manual.forEach((entry, index) => {
|
||||
if (!entry?.id || typeof entry.id !== "string") errors.push(`manual[${index}] must include an id`);
|
||||
if (!entry?.title || typeof entry.title !== "string") errors.push(`manual[${index}] ${entry?.id ?? "<missing-id>"} must include a title`);
|
||||
if (!entry?.automationDeferredReason || typeof entry.automationDeferredReason !== "string" || entry.automationDeferredReason.trim().length === 0) {
|
||||
errors.push(`manual[${index}] ${entry?.id ?? "<missing-id>"} must include a non-empty automationDeferredReason`);
|
||||
}
|
||||
if (entry?.id && !seenIds.has(entry.id)) errors.push(`manual[${index}] references unknown checklist id ${entry.id}`);
|
||||
});
|
||||
|
||||
return { ok: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
function distinctSeams(manifest) {
|
||||
const seen = new Set();
|
||||
const seams = [];
|
||||
for (const item of manifest.checklist ?? []) {
|
||||
for (const seam of item.seams ?? []) {
|
||||
const key = `${seam.package}\u0000${seam.file}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
seams.push({ package: seam.package, file: seam.file });
|
||||
}
|
||||
}
|
||||
return seams;
|
||||
}
|
||||
|
||||
export function planCommands(manifest) {
|
||||
const byPackage = new Map();
|
||||
for (const seam of distinctSeams(manifest)) {
|
||||
if (!byPackage.has(seam.package)) byPackage.set(seam.package, []);
|
||||
byPackage.get(seam.package).push(seam.file);
|
||||
}
|
||||
|
||||
return [...byPackage.entries()].map(([packageName, files]) => {
|
||||
const packageRoot = KNOWN_PACKAGES.get(packageName);
|
||||
const packageRelativeFiles = files.map((file) => path.relative(packageRoot, file));
|
||||
return {
|
||||
package: packageName,
|
||||
files,
|
||||
command: "pnpm",
|
||||
args: ["--filter", packageName, "exec", "vitest", "run", ...packageRelativeFiles, "--silent=passed-only", "--reporter=dot"],
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function runProcess(command, args, { cwd, timeoutMs, stdout = process.stdout, stderr = process.stderr } = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command, args, { cwd, shell: false, detached: false, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stdoutText = "";
|
||||
let stderrText = "";
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
child.kill("SIGTERM");
|
||||
}, timeoutMs);
|
||||
|
||||
child.stdout.on("data", (chunk) => {
|
||||
const text = String(chunk);
|
||||
stdoutText += text;
|
||||
stdout?.write?.(text);
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
const text = String(chunk);
|
||||
stderrText += text;
|
||||
stderr?.write?.(text);
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ exitCode: 1, timedOut, stdout: stdoutText, stderr: `${stderrText}${error.message}\n` });
|
||||
});
|
||||
child.on("close", (exitCode, signal) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ exitCode: exitCode ?? 1, signal, timedOut, stdout: stdoutText, stderr: stderrText });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function runReleaseCheck(manifest, { rootDir = repoRoot, timeoutMs = DEFAULT_COMMAND_TIMEOUT_MS, stdout = process.stdout, stderr = process.stderr } = {}) {
|
||||
const commands = planCommands(manifest);
|
||||
const seamResults = new Map();
|
||||
for (const planned of commands) {
|
||||
stdout?.write?.(`Running: ${planned.command} ${planned.args.join(" ")}\n`);
|
||||
const result = await runProcess(planned.command, planned.args, { cwd: rootDir, timeoutMs, stdout, stderr });
|
||||
for (const file of planned.files) {
|
||||
seamResults.set(file, {
|
||||
package: planned.package,
|
||||
file,
|
||||
status: result.exitCode === 0 && !result.timedOut ? "PASS" : "FAIL",
|
||||
exitCode: result.exitCode,
|
||||
timedOut: result.timedOut,
|
||||
command: `${planned.command} ${planned.args.join(" ")}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return (manifest.checklist ?? []).map((item) => {
|
||||
const seams = (item.seams ?? []).map((seam) => seamResults.get(seam.file) ?? { ...seam, status: "FAIL", exitCode: null, timedOut: false, command: null });
|
||||
const manual = (manifest.manual ?? []).find((entry) => entry.id === item.id) ?? null;
|
||||
return { id: item.id, title: item.title, journey: item.journey, seams, manual };
|
||||
});
|
||||
}
|
||||
|
||||
export function summarize(results) {
|
||||
const items = results.map((result) => {
|
||||
const seamStatuses = (result.seams ?? []).map((seam) => seam.status);
|
||||
const hasFailure = seamStatuses.includes("FAIL");
|
||||
const hasPass = seamStatuses.includes("PASS");
|
||||
const manualReason = typeof result.manual?.automationDeferredReason === "string" ? result.manual.automationDeferredReason.trim() : "";
|
||||
const status = hasFailure ? "FAIL" : hasPass ? "PASS" : manualReason ? "MANUAL" : "FAIL";
|
||||
return { ...result, status };
|
||||
});
|
||||
const counts = {
|
||||
pass: items.filter((item) => item.status === "PASS").length,
|
||||
fail: items.filter((item) => item.status === "FAIL").length,
|
||||
manual: items.filter((item) => item.status === "MANUAL").length,
|
||||
total: items.length,
|
||||
};
|
||||
return { ok: counts.fail === 0, counts, items };
|
||||
}
|
||||
|
||||
export function renderReport(summary, { json = false } = {}) {
|
||||
if (json) return `${JSON.stringify(summary, null, 2)}\n`;
|
||||
const lines = [
|
||||
"Custom workflow reliability release-check",
|
||||
`Overall: ${summary.ok ? "PASS" : "FAIL"} (${summary.counts.pass} passed, ${summary.counts.fail} failed, ${summary.counts.manual} manual, ${summary.counts.total} total)`,
|
||||
"",
|
||||
];
|
||||
for (const item of summary.items) {
|
||||
lines.push(`${item.status}: ${item.id} — ${item.title}`);
|
||||
for (const seam of item.seams ?? []) {
|
||||
lines.push(` - ${seam.status}: ${seam.file} (${seam.package})`);
|
||||
}
|
||||
if (item.manual?.automationDeferredReason) {
|
||||
lines.push(` - MANUAL: ${item.manual.automationDeferredReason}`);
|
||||
}
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { dryRun: false, json: false, manifestPath: DEFAULT_MANIFEST_PATH, timeoutMs: DEFAULT_COMMAND_TIMEOUT_MS };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--dry-run") args.dryRun = true;
|
||||
else if (arg === "--json") args.json = true;
|
||||
else if (arg === "--manifest") args.manifestPath = argv[++index];
|
||||
else if (arg === "--timeout-ms") args.timeoutMs = Number(argv[++index]);
|
||||
else if (arg === "--help" || arg === "-h") args.help = true;
|
||||
else throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
if (!Number.isFinite(args.timeoutMs) || args.timeoutMs <= 0) throw new Error("--timeout-ms must be a positive number");
|
||||
return args;
|
||||
}
|
||||
|
||||
function renderDryRun(manifest) {
|
||||
const commands = planCommands(manifest);
|
||||
const results = (manifest.checklist ?? []).map((item) => ({
|
||||
id: item.id,
|
||||
title: item.title,
|
||||
journey: item.journey,
|
||||
seams: (item.seams ?? []).map((seam) => ({ ...seam, status: "PASS", exitCode: 0, timedOut: false, command: null })),
|
||||
manual: (manifest.manual ?? []).find((entry) => entry.id === item.id) ?? null,
|
||||
}));
|
||||
return { commands, summary: summarize(results) };
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2), { rootDir = repoRoot, stdout = process.stdout, stderr = process.stderr } = {}) {
|
||||
let args;
|
||||
try {
|
||||
args = parseArgs(argv);
|
||||
} catch (error) {
|
||||
stderr.write(`${error.message}\n`);
|
||||
return 1;
|
||||
}
|
||||
if (args.help) {
|
||||
stdout.write("Usage: node scripts/workflow-reliability-release-check.mjs [--dry-run] [--json] [--manifest <path>] [--timeout-ms <ms>]\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
let manifest;
|
||||
try {
|
||||
manifest = loadManifest(args.manifestPath, { rootDir });
|
||||
} catch (error) {
|
||||
const summary = { ok: false, counts: { pass: 0, fail: 1, manual: 0, total: 1 }, items: [{ id: "manifest", title: "Manifest load", status: "FAIL", seams: [], error: error.message }] };
|
||||
stdout.write(renderReport(summary, { json: args.json }));
|
||||
return 1;
|
||||
}
|
||||
|
||||
const validation = validateManifest(manifest, { repoRoot: rootDir, existsSync });
|
||||
if (!validation.ok) {
|
||||
const summary = { ok: false, counts: { pass: 0, fail: validation.errors.length, manual: 0, total: validation.errors.length }, items: validation.errors.map((error, index) => ({ id: `manifest-${index + 1}`, title: error, status: "FAIL", seams: [] })) };
|
||||
stdout.write(renderReport(summary, { json: args.json }));
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (args.dryRun) {
|
||||
const dryRun = renderDryRun(manifest);
|
||||
if (args.json) {
|
||||
stdout.write(`${JSON.stringify({ ok: dryRun.summary.ok, dryRun: true, commands: dryRun.commands, summary: dryRun.summary }, null, 2)}\n`);
|
||||
} else {
|
||||
stdout.write("Dry run: manifest is valid. Planned commands:\n");
|
||||
for (const command of dryRun.commands) stdout.write(`- ${command.command} ${command.args.join(" ")}\n`);
|
||||
stdout.write("\n");
|
||||
stdout.write(renderReport(dryRun.summary));
|
||||
}
|
||||
return dryRun.summary.ok ? 0 : 1;
|
||||
}
|
||||
|
||||
const results = await runReleaseCheck(manifest, { rootDir, timeoutMs: args.timeoutMs, stdout: args.json ? { write: () => {} } : stdout, stderr });
|
||||
const summary = summarize(results);
|
||||
stdout.write(renderReport(summary, { json: args.json }));
|
||||
return summary.ok ? 0 : 1;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const exitCode = await main();
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
Reference in New Issue
Block a user