diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 987f54b820..4468206a2a 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -83,6 +83,43 @@ jobs: - name: Setup Node.js and pnpm uses: ./.github/actions/setup-node-pnpm + # Dist-artifact cache (L1): ensureTestArtifacts otherwise rebuilds dist/ + # for 8 packages (~71s) on every shard because CI starts with no dist. + # Key on a stable, pre-build, git-based hash of ALL build packages' source + # inputs. Exact-match only — NO restore-keys: a partial/stale dist hit is + # the exact failure mode this repo has been bitten by (FN-4232/FN-4605), + # and ensureTestArtifacts still validates/rebuilds anything missing-or-stale + # after restore, so a miss is safe but a wrong-content hit would not be. + # NEVER add node_modules here (breaks Windows pnpm junctions elsewhere). + - name: Compute dist source hash + id: dist-hash + run: echo "hash=$(node scripts/ensure-test-artifacts.mjs --print-source-hash)" >> "$GITHUB_OUTPUT" + + - name: Cache built dist artifacts + id: dist-cache + uses: actions/cache@v4 + with: + path: | + packages/core/dist + packages/dashboard/dist + packages/engine/dist + packages/plugin-sdk/dist + plugins/fusion-plugin-dependency-graph/dist + plugins/fusion-plugin-hermes-runtime/dist + plugins/fusion-plugin-openclaw-runtime/dist + plugins/fusion-plugin-paperclip-runtime/dist + key: dist-${{ runner.os }}-${{ steps.dist-hash.outputs.hash }} + + # On a cache HIT, restored dist files carry their save-time mtimes while + # checkout rewrites src mtimes to "now" (src newer than dist), which would + # make ensureTestArtifacts' mtime fallback rebuild everything and defeat + # the cache. Seed the per-package content-hash cache so its content-hash + # short-circuit fires instead. ensureTestArtifacts still runs (inside + # test:ci:shard) and rebuilds anything genuinely missing/changed. + - name: Seed artifact hash-cache on cache hit + if: steps.dist-cache.outputs.cache-hit == 'true' + run: node scripts/ensure-test-artifacts.mjs --seed-artifact-cache + - name: Test (deterministic shard) run: pnpm test:ci:shard --shard ${{ matrix.shard }} --total 4 @@ -121,6 +158,33 @@ jobs: - name: Setup Node.js and pnpm uses: ./.github/actions/setup-node-pnpm + # Same dist-artifact cache as test-shards (L1): the curated-gate guard runs + # `vitest list`, whose config resolution can touch built dist, so it also + # pays the cold-dist rebuild. Exact-match key on the pre-build source hash; + # NO restore-keys (stale dist is the failure mode), NO node_modules. + - name: Compute dist source hash + id: dist-hash + run: echo "hash=$(node scripts/ensure-test-artifacts.mjs --print-source-hash)" >> "$GITHUB_OUTPUT" + + - name: Cache built dist artifacts + id: dist-cache + uses: actions/cache@v4 + with: + path: | + packages/core/dist + packages/dashboard/dist + packages/engine/dist + packages/plugin-sdk/dist + plugins/fusion-plugin-dependency-graph/dist + plugins/fusion-plugin-hermes-runtime/dist + plugins/fusion-plugin-openclaw-runtime/dist + plugins/fusion-plugin-paperclip-runtime/dist + key: dist-${{ runner.os }}-${{ steps.dist-hash.outputs.hash }} + + - name: Seed artifact hash-cache on cache hit + if: steps.dist-cache.outputs.cache-hit == 'true' + run: node scripts/ensure-test-artifacts.mjs --seed-artifact-cache + - name: Assert every dashboard test file is gated or skip-listed run: node scripts/check-test-inventory.mjs --dashboard-curated diff --git a/scripts/__tests__/ensure-test-artifacts.test.mjs b/scripts/__tests__/ensure-test-artifacts.test.mjs index 0ab7f4a529..384132f84f 100644 --- a/scripts/__tests__/ensure-test-artifacts.test.mjs +++ b/scripts/__tests__/ensure-test-artifacts.test.mjs @@ -1,13 +1,16 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { + computeCombinedSourceHash, detectMissingOrStaleArtifacts, ensureTestArtifacts, isStale, + packageSourceInputs, REQUIRED_BUILD_PACKAGES, + seedArtifactCache, } from "../ensure-test-artifacts.mjs"; const ENGINE_ENTRY = REQUIRED_BUILD_PACKAGES.find((pkg) => pkg.name === "@fusion/engine"); @@ -498,3 +501,108 @@ function sourceHashFor(gitFn) { gitFn, }); } + +// --------------------------------------------------------------------------- +// CI dist-artifact cache: combined source hash (cache key) + seed mode. +// --------------------------------------------------------------------------- + +const ALL_SOURCE_INPUTS = [ + ...new Set(REQUIRED_BUILD_PACKAGES.flatMap((pkg) => packageSourceInputs(pkg))), +]; + +test("packageSourceInputs covers every build package (no empty source sets)", () => { + for (const pkg of REQUIRED_BUILD_PACKAGES) { + assert.ok( + packageSourceInputs(pkg).length > 0, + `${pkg.name} must contribute at least one source input to the combined hash`, + ); + } +}); + +/** + * A whole-repo git stub: every source input dir reports a single tracked file + * with the given per-path blob sha (defaults to a stable derived value). Lets us + * drive the combined hash deterministically without a real work tree. + */ +function fakeGitForAllSources(blobFor = (filePath) => `blob:${filePath}`) { + return (args) => { + if (args[0] === "rev-parse") return "true"; + if (args[0] === "ls-files") { + const lines = ALL_SOURCE_INPUTS.map((dir) => { + const filePath = `${dir}/index.ts`; + return `100644 ${blobFor(filePath)} 0\t${filePath}`; + }); + return lines.join("\n"); + } + if (args[0] === "status") return ""; // clean + return null; + }; +} + +test("computeCombinedSourceHash: same tree -> identical hash (deterministic)", () => { + const git = fakeGitForAllSources(); + const a = computeCombinedSourceHash("/repo", git); + const b = computeCombinedSourceHash("/repo", git); + assert.equal(typeof a, "string"); + assert.equal(a.length, 64); + assert.equal(a, b); +}); + +test("computeCombinedSourceHash: any source change -> different hash", () => { + const base = computeCombinedSourceHash("/repo", fakeGitForAllSources()); + // Flip the blob sha for engine src only; the combined hash must change. + const mutated = computeCombinedSourceHash( + "/repo", + fakeGitForAllSources((filePath) => + filePath.startsWith("packages/engine/src") ? "blob:CHANGED" : `blob:${filePath}`, + ), + ); + assert.notEqual(base, mutated); +}); + +test("computeCombinedSourceHash: returns null outside a git work tree (no unstable key)", () => { + const noGit = (args) => (args[0] === "rev-parse" ? "false" : null); + assert.equal(computeCombinedSourceHash("/repo", noGit), null); +}); + +test("seedArtifactCache: records hashes for staleable packages when all artifacts exist", () => { + const root = mkdtempSync(path.join(tmpdir(), "fn-dist-seed-")); + try { + writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages:\n - 'packages/*'\n"); + // All artifacts present. + const seeded = seedArtifactCache(root, () => true, fakeGitForAllSources()); + // recordArtifactBuild only writes entries for packages with source globs + // (engine + the 4 plugins); the mtime-immune core/dashboard/plugin-sdk are + // returned as "present" but contribute no cache entry. + assert.ok(seeded.includes("@fusion/engine")); + assert.ok(seeded.includes("@fusion-plugin-examples/hermes-runtime")); + + const cache = JSON.parse( + readFileSync(path.join(root, "node_modules", ".cache", "fusion", "artifact-cache.json"), "utf8"), + ); + assert.ok(cache.entries["@fusion/engine"]?.sourceHash); + assert.ok(cache.entries["@fusion-plugin-examples/hermes-runtime"]?.sourceHash); + // No-glob packages must not get a (meaningless) entry. + assert.equal(cache.entries["@fusion/core"], undefined); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("seedArtifactCache: does NOT record a package whose artifacts are missing", () => { + const root = mkdtempSync(path.join(tmpdir(), "fn-dist-seed-miss-")); + try { + writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages:\n - 'packages/*'\n"); + // Engine dist is missing; everything else present. + const existsFn = (p) => !p.endsWith("packages/engine/dist/index.js"); + const seeded = seedArtifactCache(root, existsFn, fakeGitForAllSources()); + assert.ok(!seeded.includes("@fusion/engine"), "missing-artifact package must not be seeded"); + + const cache = JSON.parse( + readFileSync(path.join(root, "node_modules", ".cache", "fusion", "artifact-cache.json"), "utf8"), + ); + assert.equal(cache.entries["@fusion/engine"], undefined); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/ensure-test-artifacts.mjs b/scripts/ensure-test-artifacts.mjs index 2a6d16c336..2e8342a928 100644 --- a/scripts/ensure-test-artifacts.mjs +++ b/scripts/ensure-test-artifacts.mjs @@ -11,14 +11,27 @@ import { } from "./lib/content-hash.mjs"; export const REQUIRED_BUILD_PACKAGES = [ - { name: "@fusion/core", requiredArtifacts: ["packages/core/dist/index.js"] }, - { name: "@fusion/dashboard", requiredArtifacts: ["packages/dashboard/dist/index.js"] }, + { + name: "@fusion/core", + requiredArtifacts: ["packages/core/dist/index.js"], + sourceInputs: ["packages/core/src"], + }, + { + name: "@fusion/dashboard", + requiredArtifacts: ["packages/dashboard/dist/index.js"], + // Dashboard's `vite build && tsc` reads both app/ and src/. + sourceInputs: ["packages/dashboard/app", "packages/dashboard/src"], + }, { name: "@fusion/engine", requiredArtifacts: ["packages/engine/dist/index.js"], staleAgainstGlobs: [{ sourcePath: "packages/engine/src" }], }, - { name: "@fusion/plugin-sdk", requiredArtifacts: ["packages/plugin-sdk/dist/index.js"] }, + { + name: "@fusion/plugin-sdk", + requiredArtifacts: ["packages/plugin-sdk/dist/index.js"], + sourceInputs: ["packages/plugin-sdk/src"], + }, { name: "@fusion-plugin-examples/dependency-graph", requiredArtifacts: [ @@ -75,6 +88,53 @@ export const REQUIRED_BUILD_PACKAGES = [ const ARTIFACT_CACHE_VERSION = 1; +/** + * Repo-relative source input paths for a build package. Prefers the explicit + * `sourceInputs` list (covers packages with no mtime-staleness globs, e.g. + * @fusion/core), and falls back to the `staleAgainstGlobs` source paths so the + * engine + plugin entries keep a single source of truth. + * + * @param {object} pkgEntry + * @returns {string[]} + */ +export function packageSourceInputs(pkgEntry) { + if (Array.isArray(pkgEntry?.sourceInputs) && pkgEntry.sourceInputs.length > 0) { + return [...pkgEntry.sourceInputs]; + } + if (pkgEntry?.staleAgainstGlobs?.length) { + return pkgEntry.staleAgainstGlobs.map((glob) => glob.sourcePath); + } + return []; +} + +/** + * Stable, git-based combined source hash over ALL build packages' source + * inputs. Computable BEFORE any build (it only reads git blob SHAs / working + * tree bytes, never dist), and branch-switch stable because it defers to git + * content rather than file mtimes. Used as the CI dist-cache key. + * + * Returns null when git is unavailable (no stable key → caller must not cache). + * + * @param {string} rootDir + * @param {(args: string[], cwd: string) => string|null} [gitFn] + * @returns {string|null} + */ +export function computeCombinedSourceHash(rootDir = process.cwd(), gitFn = defaultGitRunner) { + const probe = gitFn(["rev-parse", "--is-inside-work-tree"], rootDir); + if (probe !== "true") return null; + // Sorted, de-duplicated union of every package's source inputs so the order in + // REQUIRED_BUILD_PACKAGES can't perturb the hash. + const inputPaths = [ + ...new Set(REQUIRED_BUILD_PACKAGES.flatMap((pkg) => packageSourceInputs(pkg))), + ].sort((a, b) => a.localeCompare(b)); + return computeContentHash({ + rootDir, + inputPaths, + versionPrefix: `artifact-combined-v${ARTIFACT_CACHE_VERSION}`, + gitFn, + }); +} + function artifactCachePath(rootDir) { return path.join(fusionCacheDir(rootDir), "artifact-cache.json"); } @@ -376,6 +436,48 @@ export function ensureTestArtifacts( return names; } -if (import.meta.url === `file://${process.argv[1]}`) { - ensureTestArtifacts(); +/** + * Seed the per-package content-hash cache for every build package whose dist + * artifacts are ALL present, recording the current source hash as the "built + * baseline". Intended to run right after a CI dist-cache HIT: the restored dist + * carries the saved (older) mtime while checkout rewrites src mtimes to "now", + * so without a seeded hash-cache `isStale`'s mtime fallback would rebuild + * everything and defeat the cache. Seeding adopts the restored content as fresh + * so the content-hash short-circuit fires instead. + * + * Only seeds packages with all artifacts present (never masks a genuinely + * missing/partial dist). Returns the list of package names seeded. + * + * @param {string} rootDir + * @param {(p: string) => boolean} [existsFn] + * @param {(args: string[], cwd: string) => string|null} [gitFn] + * @returns {string[]} + */ +export function seedArtifactCache(rootDir = process.cwd(), existsFn = existsSync, gitFn = defaultGitRunner) { + const resolvedRootDir = resolveWorkspaceRoot(rootDir); + const present = REQUIRED_BUILD_PACKAGES.filter((pkg) => + pkg.requiredArtifacts.every((artifactPath) => existsFn(path.join(resolvedRootDir, artifactPath))), + ); + // recordArtifactBuild itself no-ops packages without source globs (the + // mtime-immune @fusion/core/dashboard/plugin-sdk), so only the staleable + // packages actually get an entry — exactly the ones that need the override. + recordArtifactBuild(present, resolvedRootDir, gitFn); + return present.map((pkg) => pkg.name); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const argv = process.argv.slice(2); + if (argv.includes("--print-source-hash")) { + const hash = computeCombinedSourceHash(); + if (hash === null) { + process.stderr.write("[test-bootstrap] cannot compute source hash: not a git work tree\n"); + process.exit(1); + } + process.stdout.write(`${hash}\n`); + } else if (argv.includes("--seed-artifact-cache")) { + const seeded = seedArtifactCache(); + process.stderr.write(`[test-bootstrap] seeded artifact hash-cache for: ${seeded.join(", ") || "(none)"}\n`); + } else { + ensureTestArtifacts(); + } }