From 610d473e779c1daa1dfb5879c3c33efd6e253379 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 3 Jun 2026 19:12:28 -0700 Subject: [PATCH] refactor(test): simplify-pass cleanups from 4-angle review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - content-hash: createRepoContentSnapshot — 2 repo-wide git spawns shared across all hash computations (was ~2 spawns x N packages x 2 passes, ~0.6-1.6s per cache-miss run); snapshot-equivalence test pins zero-spawn path - test-changed: one hash memo + snapshot shared between applyCacheToPlan and recordCachePass (record pass now re-hashes nothing) - ci-test-shard: listPackageTestFiles single source of truth for the test-file glob (was triplicated) - check-test-inventory: curatedProjects defaults to projects (removes duplicated 11-entry list in spec) - ensure-test-artifacts: drop detectMissingArtifacts passthrough alias - drop dead statSync re-export; cross-reference comments on the two shared-input path lists Skipped deliberately: --cold-start-probe/--check-timings-staleness removal (plan artifacts for U8 re-eval + scheduled refresh), best-fit dedup + threshold² (behavior-preserving refactor of verified shard math — follow-up), worker-budget duplication (pre-existing on main) --- scripts/__tests__/content-hash.test.mjs | 56 ++++++- .../__tests__/ensure-test-artifacts.test.mjs | 24 +-- scripts/check-test-inventory.mjs | 11 +- scripts/ci-test-shard.mjs | 33 +++-- scripts/ensure-test-artifacts.mjs | 4 - scripts/lib/content-hash.mjs | 137 +++++++++++++----- scripts/lib/test-inventory-spec.json | 15 +- scripts/test-changed.mjs | 59 ++++++-- 8 files changed, 235 insertions(+), 104 deletions(-) diff --git a/scripts/__tests__/content-hash.test.mjs b/scripts/__tests__/content-hash.test.mjs index ebe673d694..1ae4d9348a 100644 --- a/scripts/__tests__/content-hash.test.mjs +++ b/scripts/__tests__/content-hash.test.mjs @@ -9,7 +9,7 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { computeContentHash } from "../lib/content-hash.mjs"; +import { computeContentHash, createRepoContentSnapshot } from "../lib/content-hash.mjs"; /** * Build a fake git runner from a description of the tree. @@ -21,16 +21,25 @@ import { computeContentHash } from "../lib/content-hash.mjs"; */ function fakeGit(tree) { const { tracked = {}, dirty = [], untracked = [] } = tree; + // Honor the `-- ` filter the way real git does (exact file or dir + // prefix); commands without a path filter return the whole tree. + const selected = (args, file) => { + const dashIdx = args.indexOf("--"); + if (dashIdx === -1) return true; + const inputs = args.slice(dashIdx + 1); + return inputs.some((input) => file === input || file.startsWith(`${input}/`)); + }; return (args) => { if (args[0] === "ls-files") { return Object.entries(tracked) + .filter(([file]) => selected(args, file)) .map(([file, sha]) => `100644 ${sha} 0\t${file}`) .join("\n"); } if (args[0] === "status") { const lines = []; - for (const file of dirty) lines.push(` M ${file}`); - for (const file of untracked) lines.push(`?? ${file}`); + for (const file of dirty) if (selected(args, file)) lines.push(` M ${file}`); + for (const file of untracked) if (selected(args, file)) lines.push(`?? ${file}`); return lines.join("\n"); } return null; @@ -151,3 +160,44 @@ test("versionPrefix busts the hash so a format bump invalidates all entries", () const v2 = computeContentHash({ ...base, versionPrefix: "v2", gitFn: git, readFn: readBytes({}) }); assert.notEqual(v1, v2); }); + +test("snapshot path produces identical hashes to the spawn path and spawns no git", () => { + const tree = { + tracked: { + "packages/core/src/a.ts": "aaa", + "packages/core/src/b.ts": "bbb", + "packages/engine/src/c.ts": "ccc", + "pnpm-lock.yaml": "lll", + }, + dirty: ["packages/core/src/b.ts"], + untracked: ["packages/engine/src/new.ts"], + }; + const readFn = readBytes({ + "packages/core/src/b.ts": "B-ON-DISK", + "packages/engine/src/new.ts": "NEW-ON-DISK", + }); + + const snapshot = createRepoContentSnapshot({ rootDir: base.rootDir, gitFn: fakeGit(tree) }); + + for (const inputPaths of [["packages/core"], ["packages/engine"], ["pnpm-lock.yaml"], ["packages/core", "pnpm-lock.yaml"]]) { + const viaSpawn = computeContentHash({ ...base, inputPaths, gitFn: fakeGit(tree), readFn }); + let spawnCalls = 0; + const viaSnapshot = computeContentHash({ + ...base, + inputPaths, + gitFn: () => { + spawnCalls += 1; + return null; + }, + readFn, + snapshot, + }); + assert.equal(viaSnapshot, viaSpawn, `hash mismatch for ${inputPaths.join(",")}`); + assert.equal(spawnCalls, 0, "snapshot path must not invoke git"); + } + + // Prefix selection must not match sibling dirs sharing a name prefix. + const coreOnly = computeContentHash({ ...base, inputPaths: ["packages/core"], readFn, snapshot }); + const engineOnly = computeContentHash({ ...base, inputPaths: ["packages/engine"], readFn, snapshot }); + assert.notEqual(coreOnly, engineOnly); +}); diff --git a/scripts/__tests__/ensure-test-artifacts.test.mjs b/scripts/__tests__/ensure-test-artifacts.test.mjs index 33371f29c3..0ab7f4a529 100644 --- a/scripts/__tests__/ensure-test-artifacts.test.mjs +++ b/scripts/__tests__/ensure-test-artifacts.test.mjs @@ -4,7 +4,6 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { - detectMissingArtifacts, detectMissingOrStaleArtifacts, ensureTestArtifacts, isStale, @@ -27,7 +26,7 @@ function fakeGitForEngine(blobSha) { } test("detectMissingArtifacts returns missing package list", () => { - const missing = detectMissingArtifacts("/repo", () => false); + const missing = detectMissingOrStaleArtifacts("/repo", () => false); assert.equal(missing.length, REQUIRED_BUILD_PACKAGES.length); assert.equal(missing[0].name, "@fusion/core"); }); @@ -83,7 +82,7 @@ test("ensureTestArtifacts builds only missing packages", () => { }); test("detectMissingArtifacts flags @fusion/dashboard when dist/index.js is missing", () => { - const missing = detectMissingArtifacts("/repo", (fullPath) => !fullPath.endsWith("packages/dashboard/dist/index.js")); + const missing = detectMissingOrStaleArtifacts("/repo", (fullPath) => !fullPath.endsWith("packages/dashboard/dist/index.js")); const names = missing.map((pkg) => pkg.name); assert.ok(names.includes("@fusion/dashboard")); @@ -104,7 +103,7 @@ test("ensureTestArtifacts rebuilds @fusion/dashboard when its dist is missing", }); test("detectMissingArtifacts flags @fusion/engine when dist/index.js is missing", () => { - const missing = detectMissingArtifacts("/repo", (fullPath) => !fullPath.endsWith("packages/engine/dist/index.js")); + const missing = detectMissingOrStaleArtifacts("/repo", (fullPath) => !fullPath.endsWith("packages/engine/dist/index.js")); const names = missing.map((pkg) => pkg.name); assert.ok(names.includes("@fusion/engine")); @@ -125,7 +124,7 @@ test("ensureTestArtifacts rebuilds @fusion/engine when dist is missing", () => { }); test("detectMissingArtifacts flags dependency-graph when dist/dashboard-view.js is missing", () => { - const missing = detectMissingArtifacts( + const missing = detectMissingOrStaleArtifacts( "/repo", (fullPath) => !fullPath.endsWith("plugins/fusion-plugin-dependency-graph/dist/dashboard-view.js"), ); @@ -149,7 +148,7 @@ test("ensureTestArtifacts rebuilds dependency-graph for incomplete dist artifact }); test("detectMissingArtifacts flags hermes when dist/index.js exists but dist/cli-spawn.js is missing", () => { - const missing = detectMissingArtifacts("/repo", (fullPath) => !fullPath.endsWith("dist/cli-spawn.js")); + const missing = detectMissingOrStaleArtifacts("/repo", (fullPath) => !fullPath.endsWith("dist/cli-spawn.js")); const names = missing.map((pkg) => pkg.name); assert.ok(names.includes("@fusion-plugin-examples/hermes-runtime")); @@ -170,7 +169,7 @@ test("ensureTestArtifacts rebuilds hermes for incomplete dist artifacts", () => }); test("detectMissingArtifacts flags openclaw when dist/index.js exists but transitive files are missing", () => { - const missing = detectMissingArtifacts( + const missing = detectMissingOrStaleArtifacts( "/repo", (fullPath) => fullPath.endsWith("plugins/fusion-plugin-openclaw-runtime/dist/index.js"), ); @@ -296,17 +295,6 @@ test("detectMissingOrStaleArtifacts merges missing and stale results without dup assert.equal(new Set(names).size, names.length); }); -test("detectMissingArtifacts alias returns same value as detectMissingOrStaleArtifacts", () => { - const { statFn, readdirFn } = createStaleFs("fusion-plugin-hermes-runtime", { - artifactMtime: 1000, - sourceMtime: 3000, - }); - - const aliasResult = detectMissingArtifacts("/repo", () => true, statFn, readdirFn); - const directResult = detectMissingOrStaleArtifacts("/repo", () => true, statFn, readdirFn); - - assert.deepEqual(aliasResult.map((pkg) => pkg.name), directResult.map((pkg) => pkg.name)); -}); test("ensureTestArtifacts invokes rebuild command for stale package", () => { const { statFn, readdirFn } = createStaleFs("fusion-plugin-hermes-runtime", { diff --git a/scripts/check-test-inventory.mjs b/scripts/check-test-inventory.mjs index dae4a88818..193aff28d1 100644 --- a/scripts/check-test-inventory.mjs +++ b/scripts/check-test-inventory.mjs @@ -31,7 +31,7 @@ */ import { spawnSync } from "node:child_process"; -import { readFileSync, writeFileSync, existsSync, readdirSync, statSync } from "node:fs"; +import { readFileSync, writeFileSync, existsSync, readdirSync } from "node:fs"; import { dirname, join, resolve, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; @@ -240,10 +240,13 @@ export function validateDashboardCurated({ includedFiles, allTestFiles, skipList function listExecutedDashboardQualityFiles({ repoRoot = REPO_ROOT, listFn = runVitestList } = {}) { const { packages } = loadSpec(); const dashboard = packages.find((p) => p.name === "@fusion/dashboard"); - if (!dashboard || !Array.isArray(dashboard.curatedProjects)) { - throw new Error('spec must define @fusion/dashboard with a "curatedProjects" array'); + // `curatedProjects` defaults to `projects` — list it explicitly only when the + // coverage set genuinely diverges from what --capture enumerates. + const curatedProjects = dashboard?.curatedProjects ?? dashboard?.projects; + if (!dashboard || !Array.isArray(curatedProjects)) { + throw new Error('spec must define @fusion/dashboard with "curatedProjects" or "projects"'); } - const rows = listFn(dashboard.dir, dashboard.curatedProjects, { repoRoot }); + const rows = listFn(dashboard.dir, curatedProjects, { repoRoot }); return new Set(rows.map((row) => toRepoRelative(row.file, repoRoot))); } diff --git a/scripts/ci-test-shard.mjs b/scripts/ci-test-shard.mjs index cd1f5be100..94660a4b1e 100644 --- a/scripts/ci-test-shard.mjs +++ b/scripts/ci-test-shard.mjs @@ -69,13 +69,27 @@ export function parseShardArgs(argv = process.argv.slice(2), env = process.env) return { shard, total }; } -export function countPackageTestFiles(packageDir, { projectRoot = process.cwd() } = {}) { +/** + * List a package's test files (repo-relative to the package dir). Single source + * of truth for the test-file glob + dist exclusion so counting, duration + * weighting, and the cold-start probe can't drift apart. + * + * @param {string} packageDir + * @param {{ projectRoot?: string, extraExclude?: (p: string) => boolean }} [options] + * @returns {string[]} + */ +export function listPackageTestFiles(packageDir, { projectRoot = process.cwd(), extraExclude } = {}) { const packageRoot = path.join(projectRoot, packageDir); return globSync("**/__tests__/**/*.test.{ts,tsx,mjs}", { cwd: packageRoot, nodir: true, - exclude: (p) => p.startsWith("dist/") || p.includes("/dist/"), - }).length; + exclude: (p) => + p.startsWith("dist/") || p.includes("/dist/") || (extraExclude ? extraExclude(p) : false), + }); +} + +export function countPackageTestFiles(packageDir, options = {}) { + return listPackageTestFiles(packageDir, options).length; } /** @@ -511,11 +525,7 @@ export function sumFileDurations(files, fileDurations) { */ export function computePackageDurationWeight(pkg, timings, options = {}) { const projectRoot = options.projectRoot ?? process.cwd(); - const files = globSync("**/__tests__/**/*.test.{ts,tsx,mjs}", { - cwd: path.join(projectRoot, pkg.dir), - nodir: true, - exclude: (p) => p.startsWith("dist/") || p.includes("/dist/"), - }).map((f) => `${pkg.dir}/${f}`); + const files = listPackageTestFiles(pkg.dir, { projectRoot }).map((f) => `${pkg.dir}/${f}`); const fallbackPerFile = timings.medianPerFileMs > 0 ? timings.medianPerFileMs : DURATION_BUCKET_MS; const { durationMs, timedCount, untimedCount } = sumFileDurations(files, timings.fileDurations); @@ -995,10 +1005,9 @@ export function runColdStartProbe(packageName, options = {}) { // Pick the cheapest (smallest) test file as the probe target unless given. let testFile = options.testFile ?? null; if (!testFile) { - const candidates = globSync("**/__tests__/**/*.test.{ts,tsx,mjs}", { - cwd: path.join(projectRoot, pkg.dir), - nodir: true, - exclude: (p) => p.startsWith("dist/") || p.includes("/dist/") || /\.slow\./.test(p), + const candidates = listPackageTestFiles(pkg.dir, { + projectRoot, + extraExclude: (p) => /\.slow\./.test(p), }); testFile = candidates.sort((a, b) => a.length - b.length)[0] ?? null; } diff --git a/scripts/ensure-test-artifacts.mjs b/scripts/ensure-test-artifacts.mjs index ea6c5231f0..2a6d16c336 100644 --- a/scripts/ensure-test-artifacts.mjs +++ b/scripts/ensure-test-artifacts.mjs @@ -229,10 +229,6 @@ export function detectMissingOrStaleArtifacts( }); } -export function detectMissingArtifacts(rootDir = process.cwd(), existsFn = existsSync, statFn = statSync, readdirFn = readdirSync) { - return detectMissingOrStaleArtifacts(rootDir, existsFn, statFn, readdirFn); -} - function classifyArtifactIssues(pkgEntry, rootDir, existsFn, statFn, readdirFn) { const missingPaths = pkgEntry.requiredArtifacts.filter((artifactPath) => !existsFn(path.join(rootDir, artifactPath))); if (missingPaths.length > 0) { diff --git a/scripts/lib/content-hash.mjs b/scripts/lib/content-hash.mjs index 0f9190c051..80669056a9 100644 --- a/scripts/lib/content-hash.mjs +++ b/scripts/lib/content-hash.mjs @@ -17,7 +17,7 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { readFileSync, statSync } from "node:fs"; +import { readFileSync } from "node:fs"; import path from "node:path"; /** @@ -61,42 +61,14 @@ function parseLsFiles(lsOut) { } /** - * Compute a content hash over the given repo-relative input paths. + * Parse `git status --porcelain -uall` output into dirty/untracked path sets. * - * Each path may be a file or a directory; git expands directories to their - * tracked files. Dirty (modified-tracked) and untracked-not-ignored files have - * their working-tree bytes hashed so the hash reflects real on-disk content, - * never a stale index blob SHA. - * - * @param {object} options - * @param {string} options.rootDir Repo root (cwd for git). - * @param {string[]} options.inputPaths Repo-relative files/dirs to hash. - * @param {string} [options.versionPrefix] Constant mixed in to bust on format change. - * @param {(args: string[], cwd: string) => string|null} [options.gitFn] Injectable git. - * @param {(absPath: string) => Buffer|string} [options.readFn] Injectable file reader. - * @returns {string} 64-char hex SHA-256. + * @param {string|null} statusOut + * @returns {{ dirtyPaths: Set, untrackedPaths: Set }} */ -export function computeContentHash({ - rootDir, - inputPaths, - versionPrefix = "ch-v1", - gitFn = defaultGitRunner, - readFn = (absPath) => readFileSync(absPath), -}) { - const hash = createHash("sha256"); - hash.update(versionPrefix); - hash.update("\0"); - - // Tracked files (index blob SHAs) for every input path. - const tracked = parseLsFiles(gitFn(["ls-files", "-s", "--", ...inputPaths], rootDir)); - const trackedByPath = new Map(tracked.map((entry) => [entry.filePath, entry.blobSha])); - - // Working-tree status: which tracked files are modified, which are untracked. - // `git status --porcelain -uall -- ` reports both. Untracked entries - // are prefixed with `??`; modified-tracked with ` M`/`M `/etc. +function parseStatus(statusOut) { const dirtyPaths = new Set(); const untrackedPaths = new Set(); - const statusOut = gitFn(["status", "--porcelain", "-uall", "--", ...inputPaths], rootDir); if (statusOut) { for (const rawLine of statusOut.split("\n")) { if (!rawLine) continue; @@ -118,6 +90,102 @@ export function computeContentHash({ } } } + return { dirtyPaths, untrackedPaths }; +} + +/** + * Snapshot the WHOLE repo's tracked blob SHAs and working-tree status with two + * git spawns total, so many computeContentHash calls in one run can filter by + * path prefix in JS instead of each spawning its own scoped `git ls-files` + + * `git status` pair (~2 spawns x N packages otherwise — the dominant fixed cost + * of the cache-check pass). + * + * The snapshot reflects the working tree at creation time; callers must not + * reuse it across operations that modify the tree. + * + * @param {object} options + * @param {string} options.rootDir + * @param {(args: string[], cwd: string) => string|null} [options.gitFn] + * @returns {{ trackedByPath: Map, dirtyPaths: Set, untrackedPaths: Set }} + */ +export function createRepoContentSnapshot({ rootDir, gitFn = defaultGitRunner }) { + const tracked = parseLsFiles(gitFn(["ls-files", "-s"], rootDir)); + const trackedByPath = new Map(tracked.map((entry) => [entry.filePath, entry.blobSha])); + const { dirtyPaths, untrackedPaths } = parseStatus( + gitFn(["status", "--porcelain", "-uall"], rootDir), + ); + return { trackedByPath, dirtyPaths, untrackedPaths }; +} + +/** + * True when repo-relative `filePath` is selected by one of `inputPaths` (each + * an exact file path or a directory prefix). + * + * @param {string} filePath + * @param {string[]} inputPaths + * @returns {boolean} + */ +function matchesInputPaths(filePath, inputPaths) { + for (const input of inputPaths) { + if (filePath === input || filePath.startsWith(`${input}/`)) return true; + } + return false; +} + +/** + * Compute a content hash over the given repo-relative input paths. + * + * Each path may be a file or a directory; git expands directories to their + * tracked files. Dirty (modified-tracked) and untracked-not-ignored files have + * their working-tree bytes hashed so the hash reflects real on-disk content, + * never a stale index blob SHA. + * + * @param {object} options + * @param {string} options.rootDir Repo root (cwd for git). + * @param {string[]} options.inputPaths Repo-relative files/dirs to hash. + * @param {string} [options.versionPrefix] Constant mixed in to bust on format change. + * @param {(args: string[], cwd: string) => string|null} [options.gitFn] Injectable git. + * @param {(absPath: string) => Buffer|string} [options.readFn] Injectable file reader. + * @param {ReturnType} [options.snapshot] + * Optional repo-wide snapshot; when given, no git is spawned — entries + * are selected from the snapshot by path prefix. Hash output is + * identical to the spawn path for the same tree state. + * @returns {string} 64-char hex SHA-256. + */ +export function computeContentHash({ + rootDir, + inputPaths, + versionPrefix = "ch-v1", + gitFn = defaultGitRunner, + readFn = (absPath) => readFileSync(absPath), + snapshot, +}) { + const hash = createHash("sha256"); + hash.update(versionPrefix); + hash.update("\0"); + + let trackedByPath; + let dirtyPaths; + let untrackedPaths; + if (snapshot) { + // Select from the repo-wide snapshot by prefix — zero git spawns. + trackedByPath = new Map(); + for (const [filePath, blobSha] of snapshot.trackedByPath) { + if (matchesInputPaths(filePath, inputPaths)) trackedByPath.set(filePath, blobSha); + } + dirtyPaths = new Set([...snapshot.dirtyPaths].filter((p) => matchesInputPaths(p, inputPaths))); + untrackedPaths = new Set( + [...snapshot.untrackedPaths].filter((p) => matchesInputPaths(p, inputPaths)), + ); + } else { + // Tracked files (index blob SHAs) for every input path. + const tracked = parseLsFiles(gitFn(["ls-files", "-s", "--", ...inputPaths], rootDir)); + trackedByPath = new Map(tracked.map((entry) => [entry.filePath, entry.blobSha])); + // Working-tree status: which tracked files are modified, which are untracked. + ({ dirtyPaths, untrackedPaths } = parseStatus( + gitFn(["status", "--porcelain", "-uall", "--", ...inputPaths], rootDir), + )); + } // Build the full path list: every tracked file plus every untracked file. const allPaths = new Set([...trackedByPath.keys(), ...untrackedPaths]); @@ -172,6 +240,3 @@ export function readJsonCache(filePath, fallback) { export function fusionCacheDir(rootDir) { return path.join(rootDir, "node_modules", ".cache", "fusion"); } - -/** Re-export statSync passthrough so callers can stub uniformly if needed. */ -export { statSync }; diff --git a/scripts/lib/test-inventory-spec.json b/scripts/lib/test-inventory-spec.json index ee3b524d94..043ca85a26 100644 --- a/scripts/lib/test-inventory-spec.json +++ b/scripts/lib/test-inventory-spec.json @@ -1,5 +1,5 @@ { - "$comment": "Capture spec for scripts/check-test-inventory.mjs (plan U2). Each package lists the vitest project names to enumerate via `vitest list --json`. For @fusion/dashboard, `curatedProjects` is the set of executed quality+backfill projects the curated-gate guard checks coverage against; `projects` is what --capture enumerates. Omitting `projects` captures the default (all) projects.", + "$comment": "Capture spec for scripts/check-test-inventory.mjs (plan U2). Each package lists the vitest project names to enumerate via `vitest list --json`. For @fusion/dashboard the curated-gate guard checks coverage against `curatedProjects`, which DEFAULTS to `projects` — only list it when the coverage set genuinely diverges from what --capture enumerates. Omitting `projects` captures the default (all) projects.", "packages": [ { "name": "@fusion/core", @@ -30,19 +30,6 @@ "dashboard-app-quality-backfill", "dashboard-api-quality", "dashboard-api-quality-backfill" - ], - "curatedProjects": [ - "dashboard-app-quality-foundation-api", - "dashboard-app-quality-foundation-ui", - "dashboard-app-quality-foundation-hooks-utils", - "dashboard-app-quality-components-a", - "dashboard-app-quality-components-b", - "dashboard-app-quality-app", - "dashboard-app-quality-chat", - "dashboard-app-quality-settings", - "dashboard-app-quality-backfill", - "dashboard-api-quality", - "dashboard-api-quality-backfill" ] } ] diff --git a/scripts/test-changed.mjs b/scripts/test-changed.mjs index c1a0b9bd31..192d192961 100644 --- a/scripts/test-changed.mjs +++ b/scripts/test-changed.mjs @@ -9,7 +9,7 @@ import { cpus, tmpdir } from "node:os"; import { createRequire } from "node:module"; import { ensureTestArtifacts } from "./ensure-test-artifacts.mjs"; import { isSkillSyncCheckCached } from "./sync-fusion-skill-tools.mjs"; -import { computeContentHash } from "./lib/content-hash.mjs"; +import { computeContentHash, createRepoContentSnapshot } from "./lib/content-hash.mjs"; const currentFilePath = fileURLToPath(import.meta.url); const scriptDir = path.dirname(currentFilePath); @@ -112,6 +112,10 @@ const HASH_VERSION_PREFIX = "v2"; * (mobile, droid-cli, pi-*, and every plugin/example). Dep-aware hashing * alone would miss those, so we fold the tree in globally — the simplest * provably-correct choice (mirrors the tsconfig.base.json treatment). + * + * NOTE: this list intentionally overlaps `shouldForceFullSuite`'s + * `fullSuitePaths` (which decides full-suite mode, a different axis than cache + * busting). When adding a new shared root config input, consider both lists. */ const SHARED_HASH_INPUT_PATHS = [ "pnpm-lock.yaml", @@ -425,6 +429,9 @@ function isTestIrrelevantRootPath(file) { } export function shouldForceFullSuite(changedFiles) { + // NOTE: overlaps SHARED_HASH_INPUT_PATHS by intent (different axis: this list + // forces full-suite mode; that one busts every package's cache hash). When + // adding a new shared root config input, consider both lists. const fullSuitePaths = [ "package.json", "pnpm-lock.yaml", @@ -619,7 +626,7 @@ function adaptGitFnForContentHash(gitFn) { * @param {Map} [memo] Per-call memo keyed by packageDir. * @returns {string} 64-char hex SHA-256 */ -export function computeOwnHash(packageDir, gitFn = gitOutput, memo) { +export function computeOwnHash(packageDir, gitFn = gitOutput, memo, snapshot) { if (memo?.has(packageDir)) return memo.get(packageDir); const ownHash = computeContentHash({ @@ -627,6 +634,7 @@ export function computeOwnHash(packageDir, gitFn = gitOutput, memo) { inputPaths: [packageDir], versionPrefix: `${HASH_VERSION_PREFIX}:own`, gitFn: adaptGitFnForContentHash(gitFn), + snapshot, }); memo?.set(packageDir, ownHash); @@ -642,7 +650,7 @@ export function computeOwnHash(packageDir, gitFn = gitOutput, memo) { * @param {Map} [memo] * @returns {string} */ -function computeSharedInputsHash(gitFn = gitOutput, memo) { +function computeSharedInputsHash(gitFn = gitOutput, memo, snapshot) { const memoKey = "\0shared-inputs\0"; if (memo?.has(memoKey)) return memo.get(memoKey); @@ -651,6 +659,7 @@ function computeSharedInputsHash(gitFn = gitOutput, memo) { inputPaths: SHARED_HASH_INPUT_PATHS, versionPrefix: `${HASH_VERSION_PREFIX}:shared`, gitFn: adaptGitFnForContentHash(gitFn), + snapshot, }); memo?.set(memoKey, sharedHash); @@ -681,10 +690,13 @@ function computeSharedInputsHash(gitFn = gitOutput, memo) { * @param {Map} [options.forwardDependencyMap] name → [dep names]. * @param {Map} [options.packageDirByName] name → relative dir. * @param {Map} [options.memo] Per-run own-hash memo (perf). + * @param {object} [options.snapshot] Repo-wide content snapshot (from + * createRepoContentSnapshot — 2 git spawns total) shared across all hash + * computations in a run; without it each own-hash pays its own spawns. * @returns {string} 64-char hex SHA-256 */ export function computePackageHash(packageDir, gitFn = gitOutput, options = {}) { - const { packageName, forwardDependencyMap, packageDirByName, memo = new Map() } = options; + const { packageName, forwardDependencyMap, packageDirByName, memo = new Map(), snapshot } = options; const hash = createHash("sha256"); hash.update(HASH_VERSION_PREFIX); @@ -692,12 +704,12 @@ export function computePackageHash(packageDir, gitFn = gitOutput, options = {}) // Shared inputs (lockfile, base tsconfig, shared __test-utils__ tree). hash.update("shared="); - hash.update(computeSharedInputsHash(gitFn, memo)); + hash.update(computeSharedInputsHash(gitFn, memo, snapshot)); hash.update("\0"); // This package's own dirty-aware content. hash.update("own="); - hash.update(computeOwnHash(packageDir, gitFn, memo)); + hash.update(computeOwnHash(packageDir, gitFn, memo, snapshot)); hash.update("\0"); // Transitive workspace dependencies' own hashes (sorted by name for stability). @@ -709,7 +721,7 @@ export function computePackageHash(packageDir, gitFn = gitOutput, options = {}) hash.update("dep:"); hash.update(depName); hash.update("="); - hash.update(computeOwnHash(depDir, gitFn, memo)); + hash.update(computeOwnHash(depDir, gitFn, memo, snapshot)); hash.update("\0"); } } @@ -768,6 +780,11 @@ export function applyCacheToPlan(plan, options = {}) { writeCacheFn, packageDirByName = new Map(), forwardDependencyMap = new Map(), + // Shared per-RUN memo + repo snapshot: main() passes the same pair to + // recordCachePass so the record pass re-spawns zero git and re-hashes + // nothing (test runs don't modify hashed source). + memo = new Map(), + snapshot, } = options; // Full suite runs always bypass cache (full means full). @@ -781,9 +798,6 @@ export function applyCacheToPlan(plan, options = {}) { const cachedPackages = []; const activePackages = []; - // Shared per-call memo so each package/dependency own-hash is computed once, - // keeping dep-aware hashing O(packages) rather than O(packages^2). - const memo = new Map(); for (const pkg of plan.packages ?? []) { const pkgDir = packageDirByName.get(pkg) ?? `packages/${pkg.replace(/^@[^/]+\//, "")}`; @@ -792,6 +806,7 @@ export function applyCacheToPlan(plan, options = {}) { forwardDependencyMap, packageDirByName, memo, + snapshot, }); const entry = cache.entries[pkg]; @@ -827,6 +842,10 @@ export function recordCachePass(packages, packageDirByName, options = {}) { readCacheFn, writeCacheFn, forwardDependencyMap = new Map(), + // When main() passes the memo/snapshot already populated by + // applyCacheToPlan, every hash below is a memo hit — zero git spawns. + memo = new Map(), + snapshot, } = options; if (noCache || packages.length === 0) return; @@ -834,8 +853,6 @@ export function recordCachePass(packages, packageDirByName, options = {}) { const filePath = cacheFilePath(); const cache = readCacheFn ? readCacheFn() : readCache(filePath); const now = new Date().toISOString(); - // Shared per-call memo (see applyCacheToPlan): keep own-hashing O(packages). - const memo = new Map(); for (const pkg of packages) { const pkgDir = packageDirByName.get(pkg) ?? `packages/${pkg.replace(/^@[^/]+\//, "")}`; @@ -844,6 +861,7 @@ export function recordCachePass(packages, packageDirByName, options = {}) { forwardDependencyMap, packageDirByName, memo, + snapshot, }); cache.entries[pkg] = { hash, passedAt: now, command: "test" }; } @@ -1100,11 +1118,21 @@ export function main(argv = process.argv.slice(2)) { // actually needs running before we spend setup time. let cachedPackages = []; let activePackages = plan.packages ?? []; + // One repo-wide content snapshot (2 git spawns) + one own-hash memo for the + // entire run: applyCacheToPlan populates them, recordCachePass reuses them, + // so per-package git spawns and re-hashing happen exactly once per run. + const hashMemo = new Map(); + let hashSnapshot; if (plan.mode === "changed") { + if (!(noCache || forceFullSuite)) { + hashSnapshot = createRepoContentSnapshot({ rootDir }); + } ({ cachedPackages, activePackages } = applyCacheToPlan(plan, { noCache: noCache || forceFullSuite, packageDirByName, forwardDependencyMap, + memo: hashMemo, + snapshot: hashSnapshot, })); } @@ -1180,7 +1208,12 @@ export function main(argv = process.argv.slice(2)) { }); // Tests passed — record in cache (never cache failures; process.exit on failure above). - recordCachePass(activePackages, packageDirByName, { noCache, forwardDependencyMap }); + recordCachePass(activePackages, packageDirByName, { + noCache, + forwardDependencyMap, + memo: hashMemo, + snapshot: hashSnapshot, + }); } finally { cleanupIsolatedHome(); }