perf(ci): cache built dist artifacts keyed by source content hash
Every shard + the curated-guard job paid ~71s rebuilding 8 packages' dist from scratch. actions/cache now restores dist on exact content-hash match (--print-source-hash; branch-switch stable, pure git-based), with a --seed-artifact-cache step on cache-hit that defeats the mtime trap (restored dist looks older than checkout-time src mtimes). No restore-keys partial fallback: stale dist is a known failure mode here. node_modules is never cached (Windows junction policy). ensure-test-artifacts still runs as the authority and rebuilds anything genuinely missing or changed.
This commit is contained in:
gsxdsm
64
.github/workflows/pr-checks.yml
vendored
64
.github/workflows/pr-checks.yml
vendored
@@ -83,6 +83,43 @@ jobs:
|
||||
- name: Setup Node.js and pnpm
|
||||
uses: ./.github/actions/setup-node-pnpm
|
||||
|
||||
# Dist-artifact cache (L1): ensureTestArtifacts otherwise rebuilds dist/
|
||||
# for 8 packages (~71s) on every shard because CI starts with no dist.
|
||||
# Key on a stable, pre-build, git-based hash of ALL build packages' source
|
||||
# inputs. Exact-match only — NO restore-keys: a partial/stale dist hit is
|
||||
# the exact failure mode this repo has been bitten by (FN-4232/FN-4605),
|
||||
# and ensureTestArtifacts still validates/rebuilds anything missing-or-stale
|
||||
# after restore, so a miss is safe but a wrong-content hit would not be.
|
||||
# NEVER add node_modules here (breaks Windows pnpm junctions elsewhere).
|
||||
- name: Compute dist source hash
|
||||
id: dist-hash
|
||||
run: echo "hash=$(node scripts/ensure-test-artifacts.mjs --print-source-hash)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache built dist artifacts
|
||||
id: dist-cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
packages/core/dist
|
||||
packages/dashboard/dist
|
||||
packages/engine/dist
|
||||
packages/plugin-sdk/dist
|
||||
plugins/fusion-plugin-dependency-graph/dist
|
||||
plugins/fusion-plugin-hermes-runtime/dist
|
||||
plugins/fusion-plugin-openclaw-runtime/dist
|
||||
plugins/fusion-plugin-paperclip-runtime/dist
|
||||
key: dist-${{ runner.os }}-${{ steps.dist-hash.outputs.hash }}
|
||||
|
||||
# On a cache HIT, restored dist files carry their save-time mtimes while
|
||||
# checkout rewrites src mtimes to "now" (src newer than dist), which would
|
||||
# make ensureTestArtifacts' mtime fallback rebuild everything and defeat
|
||||
# the cache. Seed the per-package content-hash cache so its content-hash
|
||||
# short-circuit fires instead. ensureTestArtifacts still runs (inside
|
||||
# test:ci:shard) and rebuilds anything genuinely missing/changed.
|
||||
- name: Seed artifact hash-cache on cache hit
|
||||
if: steps.dist-cache.outputs.cache-hit == 'true'
|
||||
run: node scripts/ensure-test-artifacts.mjs --seed-artifact-cache
|
||||
|
||||
- name: Test (deterministic shard)
|
||||
run: pnpm test:ci:shard --shard ${{ matrix.shard }} --total 4
|
||||
|
||||
@@ -121,6 +158,33 @@ jobs:
|
||||
- name: Setup Node.js and pnpm
|
||||
uses: ./.github/actions/setup-node-pnpm
|
||||
|
||||
# Same dist-artifact cache as test-shards (L1): the curated-gate guard runs
|
||||
# `vitest list`, whose config resolution can touch built dist, so it also
|
||||
# pays the cold-dist rebuild. Exact-match key on the pre-build source hash;
|
||||
# NO restore-keys (stale dist is the failure mode), NO node_modules.
|
||||
- name: Compute dist source hash
|
||||
id: dist-hash
|
||||
run: echo "hash=$(node scripts/ensure-test-artifacts.mjs --print-source-hash)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache built dist artifacts
|
||||
id: dist-cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
packages/core/dist
|
||||
packages/dashboard/dist
|
||||
packages/engine/dist
|
||||
packages/plugin-sdk/dist
|
||||
plugins/fusion-plugin-dependency-graph/dist
|
||||
plugins/fusion-plugin-hermes-runtime/dist
|
||||
plugins/fusion-plugin-openclaw-runtime/dist
|
||||
plugins/fusion-plugin-paperclip-runtime/dist
|
||||
key: dist-${{ runner.os }}-${{ steps.dist-hash.outputs.hash }}
|
||||
|
||||
- name: Seed artifact hash-cache on cache hit
|
||||
if: steps.dist-cache.outputs.cache-hit == 'true'
|
||||
run: node scripts/ensure-test-artifacts.mjs --seed-artifact-cache
|
||||
|
||||
- name: Assert every dashboard test file is gated or skip-listed
|
||||
run: node scripts/check-test-inventory.mjs --dashboard-curated
|
||||
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
computeCombinedSourceHash,
|
||||
detectMissingOrStaleArtifacts,
|
||||
ensureTestArtifacts,
|
||||
isStale,
|
||||
packageSourceInputs,
|
||||
REQUIRED_BUILD_PACKAGES,
|
||||
seedArtifactCache,
|
||||
} from "../ensure-test-artifacts.mjs";
|
||||
|
||||
const ENGINE_ENTRY = REQUIRED_BUILD_PACKAGES.find((pkg) => pkg.name === "@fusion/engine");
|
||||
@@ -498,3 +501,108 @@ function sourceHashFor(gitFn) {
|
||||
gitFn,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CI dist-artifact cache: combined source hash (cache key) + seed mode.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ALL_SOURCE_INPUTS = [
|
||||
...new Set(REQUIRED_BUILD_PACKAGES.flatMap((pkg) => packageSourceInputs(pkg))),
|
||||
];
|
||||
|
||||
test("packageSourceInputs covers every build package (no empty source sets)", () => {
|
||||
for (const pkg of REQUIRED_BUILD_PACKAGES) {
|
||||
assert.ok(
|
||||
packageSourceInputs(pkg).length > 0,
|
||||
`${pkg.name} must contribute at least one source input to the combined hash`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* A whole-repo git stub: every source input dir reports a single tracked file
|
||||
* with the given per-path blob sha (defaults to a stable derived value). Lets us
|
||||
* drive the combined hash deterministically without a real work tree.
|
||||
*/
|
||||
function fakeGitForAllSources(blobFor = (filePath) => `blob:${filePath}`) {
|
||||
return (args) => {
|
||||
if (args[0] === "rev-parse") return "true";
|
||||
if (args[0] === "ls-files") {
|
||||
const lines = ALL_SOURCE_INPUTS.map((dir) => {
|
||||
const filePath = `${dir}/index.ts`;
|
||||
return `100644 ${blobFor(filePath)} 0\t${filePath}`;
|
||||
});
|
||||
return lines.join("\n");
|
||||
}
|
||||
if (args[0] === "status") return ""; // clean
|
||||
return null;
|
||||
};
|
||||
}
|
||||
|
||||
test("computeCombinedSourceHash: same tree -> identical hash (deterministic)", () => {
|
||||
const git = fakeGitForAllSources();
|
||||
const a = computeCombinedSourceHash("/repo", git);
|
||||
const b = computeCombinedSourceHash("/repo", git);
|
||||
assert.equal(typeof a, "string");
|
||||
assert.equal(a.length, 64);
|
||||
assert.equal(a, b);
|
||||
});
|
||||
|
||||
test("computeCombinedSourceHash: any source change -> different hash", () => {
|
||||
const base = computeCombinedSourceHash("/repo", fakeGitForAllSources());
|
||||
// Flip the blob sha for engine src only; the combined hash must change.
|
||||
const mutated = computeCombinedSourceHash(
|
||||
"/repo",
|
||||
fakeGitForAllSources((filePath) =>
|
||||
filePath.startsWith("packages/engine/src") ? "blob:CHANGED" : `blob:${filePath}`,
|
||||
),
|
||||
);
|
||||
assert.notEqual(base, mutated);
|
||||
});
|
||||
|
||||
test("computeCombinedSourceHash: returns null outside a git work tree (no unstable key)", () => {
|
||||
const noGit = (args) => (args[0] === "rev-parse" ? "false" : null);
|
||||
assert.equal(computeCombinedSourceHash("/repo", noGit), null);
|
||||
});
|
||||
|
||||
test("seedArtifactCache: records hashes for staleable packages when all artifacts exist", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "fn-dist-seed-"));
|
||||
try {
|
||||
writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages:\n - 'packages/*'\n");
|
||||
// All artifacts present.
|
||||
const seeded = seedArtifactCache(root, () => true, fakeGitForAllSources());
|
||||
// recordArtifactBuild only writes entries for packages with source globs
|
||||
// (engine + the 4 plugins); the mtime-immune core/dashboard/plugin-sdk are
|
||||
// returned as "present" but contribute no cache entry.
|
||||
assert.ok(seeded.includes("@fusion/engine"));
|
||||
assert.ok(seeded.includes("@fusion-plugin-examples/hermes-runtime"));
|
||||
|
||||
const cache = JSON.parse(
|
||||
readFileSync(path.join(root, "node_modules", ".cache", "fusion", "artifact-cache.json"), "utf8"),
|
||||
);
|
||||
assert.ok(cache.entries["@fusion/engine"]?.sourceHash);
|
||||
assert.ok(cache.entries["@fusion-plugin-examples/hermes-runtime"]?.sourceHash);
|
||||
// No-glob packages must not get a (meaningless) entry.
|
||||
assert.equal(cache.entries["@fusion/core"], undefined);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("seedArtifactCache: does NOT record a package whose artifacts are missing", () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "fn-dist-seed-miss-"));
|
||||
try {
|
||||
writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages:\n - 'packages/*'\n");
|
||||
// Engine dist is missing; everything else present.
|
||||
const existsFn = (p) => !p.endsWith("packages/engine/dist/index.js");
|
||||
const seeded = seedArtifactCache(root, existsFn, fakeGitForAllSources());
|
||||
assert.ok(!seeded.includes("@fusion/engine"), "missing-artifact package must not be seeded");
|
||||
|
||||
const cache = JSON.parse(
|
||||
readFileSync(path.join(root, "node_modules", ".cache", "fusion", "artifact-cache.json"), "utf8"),
|
||||
);
|
||||
assert.equal(cache.entries["@fusion/engine"], undefined);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -11,14 +11,27 @@ import {
|
||||
} from "./lib/content-hash.mjs";
|
||||
|
||||
export const REQUIRED_BUILD_PACKAGES = [
|
||||
{ name: "@fusion/core", requiredArtifacts: ["packages/core/dist/index.js"] },
|
||||
{ name: "@fusion/dashboard", requiredArtifacts: ["packages/dashboard/dist/index.js"] },
|
||||
{
|
||||
name: "@fusion/core",
|
||||
requiredArtifacts: ["packages/core/dist/index.js"],
|
||||
sourceInputs: ["packages/core/src"],
|
||||
},
|
||||
{
|
||||
name: "@fusion/dashboard",
|
||||
requiredArtifacts: ["packages/dashboard/dist/index.js"],
|
||||
// Dashboard's `vite build && tsc` reads both app/ and src/.
|
||||
sourceInputs: ["packages/dashboard/app", "packages/dashboard/src"],
|
||||
},
|
||||
{
|
||||
name: "@fusion/engine",
|
||||
requiredArtifacts: ["packages/engine/dist/index.js"],
|
||||
staleAgainstGlobs: [{ sourcePath: "packages/engine/src" }],
|
||||
},
|
||||
{ name: "@fusion/plugin-sdk", requiredArtifacts: ["packages/plugin-sdk/dist/index.js"] },
|
||||
{
|
||||
name: "@fusion/plugin-sdk",
|
||||
requiredArtifacts: ["packages/plugin-sdk/dist/index.js"],
|
||||
sourceInputs: ["packages/plugin-sdk/src"],
|
||||
},
|
||||
{
|
||||
name: "@fusion-plugin-examples/dependency-graph",
|
||||
requiredArtifacts: [
|
||||
@@ -75,6 +88,53 @@ export const REQUIRED_BUILD_PACKAGES = [
|
||||
|
||||
const ARTIFACT_CACHE_VERSION = 1;
|
||||
|
||||
/**
|
||||
* Repo-relative source input paths for a build package. Prefers the explicit
|
||||
* `sourceInputs` list (covers packages with no mtime-staleness globs, e.g.
|
||||
* @fusion/core), and falls back to the `staleAgainstGlobs` source paths so the
|
||||
* engine + plugin entries keep a single source of truth.
|
||||
*
|
||||
* @param {object} pkgEntry
|
||||
* @returns {string[]}
|
||||
*/
|
||||
export function packageSourceInputs(pkgEntry) {
|
||||
if (Array.isArray(pkgEntry?.sourceInputs) && pkgEntry.sourceInputs.length > 0) {
|
||||
return [...pkgEntry.sourceInputs];
|
||||
}
|
||||
if (pkgEntry?.staleAgainstGlobs?.length) {
|
||||
return pkgEntry.staleAgainstGlobs.map((glob) => glob.sourcePath);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable, git-based combined source hash over ALL build packages' source
|
||||
* inputs. Computable BEFORE any build (it only reads git blob SHAs / working
|
||||
* tree bytes, never dist), and branch-switch stable because it defers to git
|
||||
* content rather than file mtimes. Used as the CI dist-cache key.
|
||||
*
|
||||
* Returns null when git is unavailable (no stable key → caller must not cache).
|
||||
*
|
||||
* @param {string} rootDir
|
||||
* @param {(args: string[], cwd: string) => string|null} [gitFn]
|
||||
* @returns {string|null}
|
||||
*/
|
||||
export function computeCombinedSourceHash(rootDir = process.cwd(), gitFn = defaultGitRunner) {
|
||||
const probe = gitFn(["rev-parse", "--is-inside-work-tree"], rootDir);
|
||||
if (probe !== "true") return null;
|
||||
// Sorted, de-duplicated union of every package's source inputs so the order in
|
||||
// REQUIRED_BUILD_PACKAGES can't perturb the hash.
|
||||
const inputPaths = [
|
||||
...new Set(REQUIRED_BUILD_PACKAGES.flatMap((pkg) => packageSourceInputs(pkg))),
|
||||
].sort((a, b) => a.localeCompare(b));
|
||||
return computeContentHash({
|
||||
rootDir,
|
||||
inputPaths,
|
||||
versionPrefix: `artifact-combined-v${ARTIFACT_CACHE_VERSION}`,
|
||||
gitFn,
|
||||
});
|
||||
}
|
||||
|
||||
function artifactCachePath(rootDir) {
|
||||
return path.join(fusionCacheDir(rootDir), "artifact-cache.json");
|
||||
}
|
||||
@@ -376,6 +436,48 @@ export function ensureTestArtifacts(
|
||||
return names;
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
ensureTestArtifacts();
|
||||
/**
|
||||
* Seed the per-package content-hash cache for every build package whose dist
|
||||
* artifacts are ALL present, recording the current source hash as the "built
|
||||
* baseline". Intended to run right after a CI dist-cache HIT: the restored dist
|
||||
* carries the saved (older) mtime while checkout rewrites src mtimes to "now",
|
||||
* so without a seeded hash-cache `isStale`'s mtime fallback would rebuild
|
||||
* everything and defeat the cache. Seeding adopts the restored content as fresh
|
||||
* so the content-hash short-circuit fires instead.
|
||||
*
|
||||
* Only seeds packages with all artifacts present (never masks a genuinely
|
||||
* missing/partial dist). Returns the list of package names seeded.
|
||||
*
|
||||
* @param {string} rootDir
|
||||
* @param {(p: string) => boolean} [existsFn]
|
||||
* @param {(args: string[], cwd: string) => string|null} [gitFn]
|
||||
* @returns {string[]}
|
||||
*/
|
||||
export function seedArtifactCache(rootDir = process.cwd(), existsFn = existsSync, gitFn = defaultGitRunner) {
|
||||
const resolvedRootDir = resolveWorkspaceRoot(rootDir);
|
||||
const present = REQUIRED_BUILD_PACKAGES.filter((pkg) =>
|
||||
pkg.requiredArtifacts.every((artifactPath) => existsFn(path.join(resolvedRootDir, artifactPath))),
|
||||
);
|
||||
// recordArtifactBuild itself no-ops packages without source globs (the
|
||||
// mtime-immune @fusion/core/dashboard/plugin-sdk), so only the staleable
|
||||
// packages actually get an entry — exactly the ones that need the override.
|
||||
recordArtifactBuild(present, resolvedRootDir, gitFn);
|
||||
return present.map((pkg) => pkg.name);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv.includes("--print-source-hash")) {
|
||||
const hash = computeCombinedSourceHash();
|
||||
if (hash === null) {
|
||||
process.stderr.write("[test-bootstrap] cannot compute source hash: not a git work tree\n");
|
||||
process.exit(1);
|
||||
}
|
||||
process.stdout.write(`${hash}\n`);
|
||||
} else if (argv.includes("--seed-artifact-cache")) {
|
||||
const seeded = seedArtifactCache();
|
||||
process.stderr.write(`[test-bootstrap] seeded artifact hash-cache for: ${seeded.join(", ") || "(none)"}\n`);
|
||||
} else {
|
||||
ensureTestArtifacts();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user