feat(FN-4126): complete Step 1 — define shard entry model

Fusion-Task-Id: FN-4126
Fusion-Task-Lineage: ed1e7e10-43d3-4347-933b-0dc9e8cd345a

scripts/ci-test-shard.mjs

@@ -1,5 +1,16 @@
 #!/usr/bin/env node
 
+/**
+ * CI shard planner with virtual package slices.
+ *
+ * Packages are weighted by discovered test-file count. Oversized packages are
+ * rewritten into virtual shard entries `{ name, shardIndex, shardCount }` so
+ * one package can execute across multiple CI shards via `vitest --shard`.
+ * The planner then greedily bin-packs weighted entries into the lightest shard
+ * while keeping slices of the same package on different shards whenever
+ * possible.
+ */
+
 import { spawnSync } from "node:child_process";
 import { globSync } from "node:fs";
 import { cpus } from "node:os";
@@ -65,77 +76,117 @@ export function countPackageTestFiles(packageDir, { projectRoot = process.cwd()
 }
 
 /**
- * Expand oversized packages into virtual entries that carry vitest --shard info.
- * A package whose testFileCount exceeds splitThreshold is divided into
- * ceil(testFileCount / splitThreshold) virtual entries, each with roughly
- * equal file counts and a vitestShardIndex/vitestShardCount pair.
- *
- * @param {Array<{name:string, testFileCount:number}>} packages
- * @param {number} splitThreshold - maximum weight before splitting (default: Infinity = no split)
- * @returns {Array<{name:string, weight:number, vitestShardIndex?:number, vitestShardCount?:number}>}
+ * @typedef {{ name: string, shardIndex?: number, shardCount?: number }} ShardEntry
  */
-export function expandVirtualPackages(packages, splitThreshold = Infinity) {
+
+/**
+ * @typedef {ShardEntry & { weight: number }} WeightedShardEntry
+ */
+
+/**
+ * @param {Array<{name:string, testFileCount:number}>} packages
+ * @param {number} total
+ * @param {{ threshold?: number }} [options]
+ * @returns {WeightedShardEntry[]}
+ */
+export function computeSplitPlan(packages, total, options = {}) {
+  const threshold = options.threshold ?? 0.5;
+  const totalWeight = packages.reduce((sum, p) => sum + p.testFileCount, 0);
+  const perShardBudget = total > 0 ? totalWeight / total : 0;
+  const splitLimit = perShardBudget * threshold;
+
   const result = [];
   for (const pkg of packages) {
-    if (pkg.testFileCount <= splitThreshold || splitThreshold <= 0) {
+    const shouldConsiderSplit =
+      total > 1 &&
+      pkg.testFileCount > 0 &&
+      perShardBudget > 0 &&
+      pkg.testFileCount > splitLimit;
+    const sliceCount = shouldConsiderSplit
+      ? Math.min(total, Math.ceil(pkg.testFileCount / perShardBudget))
+      : 1;
+
+    if (sliceCount < 2) {
       result.push({ name: pkg.name, weight: pkg.testFileCount });
       continue;
     }
-    const count = Math.ceil(pkg.testFileCount / splitThreshold);
-    const baseWeight = Math.floor(pkg.testFileCount / count);
-    const remainder = pkg.testFileCount % count;
-    for (let i = 1; i <= count; i += 1) {
-      const weight = baseWeight + (i <= remainder ? 1 : 0);
+
+    const sliceWeight = Math.ceil(pkg.testFileCount / sliceCount);
+    for (let i = 1; i <= sliceCount; i += 1) {
       result.push({
         name: pkg.name,
-        weight,
-        vitestShardIndex: i,
-        vitestShardCount: count,
+        weight: sliceWeight,
+        shardIndex: i,
+        shardCount: sliceCount,
       });
     }
   }
+
   return result;
 }
 
 /**
- * Plan shard assignments using greedy bin-packing.
- * Packages exceeding the average weight per shard are automatically split
- * into virtual entries that carry vitest --shard info for intra-package
- * parallelism.
- *
- * Each returned entry is { name, weight, vitestShardIndex?, vitestShardCount? }.
- * Plain entries (no shard fields) run the full package test suite.
- * Virtual entries run `vitest --shard index/count` within the package.
+ * @param {Array<{name:string, testFileCount:number}>} packages
+ * @param {number} total
+ * @param {{ threshold?: number }} [options]
+ * @returns {ShardEntry[][]}
  */
-export function planShardAssignments(packages, total) {
-  const totalWeight = packages.reduce((sum, p) => sum + p.testFileCount, 0);
-  const splitThreshold = totalWeight > 0 ? Math.ceil(totalWeight / total) : Infinity;
-  const virtualPackages = expandVirtualPackages(packages, splitThreshold);
-
+export function planShardAssignments(packages, total, options = {}) {
+  const splitPlan = computeSplitPlan(packages, total, options);
   const shardAssignments = Array.from({ length: total }, () => []);
   const shardWeights = Array.from({ length: total }, () => 0);
-  const sorted = [...virtualPackages].sort((a, b) => {
+  const sorted = [...splitPlan].sort((a, b) => {
     if (b.weight !== a.weight) return b.weight - a.weight;
-    return a.name.localeCompare(a.name);
+    const byName = a.name.localeCompare(b.name);
+    if (byName !== 0) return byName;
+    return (a.shardIndex ?? 0) - (b.shardIndex ?? 0);
   });
 
   for (const entry of sorted) {
-    let targetIndex = 0;
-    for (let index = 1; index < total; index += 1) {
+    const eligibleIndices = [];
+    for (let index = 0; index < total; index += 1) {
+      const alreadyHasSlice =
+        entry.shardCount &&
+        shardAssignments[index].some((assigned) => assigned.name === entry.name && assigned.shardCount);
+      if (!alreadyHasSlice) {
+        eligibleIndices.push(index);
+      }
+    }
+
+    const candidates = eligibleIndices.length > 0 ? eligibleIndices : Array.from({ length: total }, (_, i) => i);
+    if (eligibleIndices.length === 0 && entry.shardCount) {
+      console.warn(
+        `[ci-test-shard] unable to isolate split slices for ${entry.name}; placing multiple slices in one shard`,
+      );
+    }
+
+    let targetIndex = candidates[0] ?? 0;
+    for (const index of candidates) {
       if (shardWeights[index] < shardWeights[targetIndex]) {
         targetIndex = index;
       }
     }
 
-    shardAssignments[targetIndex].push(entry);
+    shardAssignments[targetIndex].push(entry.shardCount ? {
+      name: entry.name,
+      shardIndex: entry.shardIndex,
+      shardCount: entry.shardCount,
+    } : { name: entry.name });
     shardWeights[targetIndex] += entry.weight;
   }
 
   return shardAssignments;
 }
 
-export function selectShardPackages(packages, shard, total) {
-  return planShardAssignments(packages, total)[shard - 1] || [];
+/**
+ * @param {Array<{name:string, testFileCount:number}>} packages
+ * @param {number} shard
+ * @param {number} total
+ * @param {{ threshold?: number }} [options]
+ * @returns {ShardEntry[]}
+ */
+export function selectShardPackages(packages, shard, total, options = {}) {
+  return planShardAssignments(packages, total, options)[shard - 1] || [];
 }
 
 export function listWorkspaceTestPackages({ projectRoot = process.cwd() } = {}) {
@@ -149,8 +200,8 @@ export function listWorkspaceTestPackages({ projectRoot = process.cwd() } = {})
 }
 
 function entryLabel(entry) {
-  if (entry.vitestShardCount) {
-    return `${entry.name} [${entry.vitestShardIndex}/${entry.vitestShardCount}]`;
+  if (entry.shardCount) {
+    return `${entry.name} [${entry.shardIndex}/${entry.shardCount}]`;
   }
   return entry.name;
 }
@@ -178,8 +229,8 @@ export function main(argv = process.argv.slice(2), env = process.env) {
 
   // Group entries: plain packages run together in one pnpm invocation;
   // virtual (sharded) entries each get their own vitest --shard invocation.
-  const plain = shardEntries.filter((e) => !e.vitestShardCount);
-  const virtual = shardEntries.filter((e) => e.vitestShardCount);
+  const plain = shardEntries.filter((e) => !e.shardCount);
+  const virtual = shardEntries.filter((e) => e.shardCount);
 
   if (plain.length > 0) {
     const filters = plain.flatMap((e) => ["--filter", e.name]);
@@ -188,13 +239,11 @@ export function main(argv = process.argv.slice(2), env = process.env) {
 
   for (const entry of virtual) {
     console.log(
-      `[ci-test-shard] running ${entry.name} --shard ${entry.vitestShardIndex}/${entry.vitestShardCount}`,
-    );
-    run(
-      "pnpm",
-      ["--filter", entry.name, "exec", "vitest", "run", "--shard", `${entry.vitestShardIndex}/${entry.vitestShardCount}`],
-      { env: shardEnv },
+      `[ci-test-shard] shard ${shard}/${total}: running ${entry.name} --shard ${entry.shardIndex}/${entry.shardCount}`,
     );
+    run("pnpm", ["--filter", entry.name, "test", "--", "--shard", `${entry.shardIndex}/${entry.shardCount}`], {
+      env: shardEnv,
+    });
   }
 }