diff --git a/.changeset/fast-local-workspace-build.md b/.changeset/fast-local-workspace-build.md new file mode 100644 index 0000000000..f34aa72cf7 --- /dev/null +++ b/.changeset/fast-local-workspace-build.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Make local `pnpm build` skip unchanged packages and use fast CLI packaging by default. +category: performance +dev: Content-hash skip cache now covers non-plugin packages; CLI packaging stages desktop/plugins/DTS only with FUSION_CLI_FULL_PACKAGE=1 or CI (`pnpm build:full`). Added maxConcurrentVerifications (default 1) and tsc incremental builds. diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 6a133e8a2d..085bb6cbf3 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -411,6 +411,7 @@ Security-sensitive file-browser escape hatches are project-only. `allowAbsoluteF | `globalPauseReason` | `string` | `undefined` | Optional reason for `globalPause` (`"rate-limit"` for automatic pauses, `"manual"` for user-triggered pauses). Cleared on unpause. | | `enginePaused` | `boolean` | `false` | Soft pause: stop dispatching new work while letting active sessions finish. While paused (including shared pause windows with `globalPause`), stuck-task polling/timers are suspended so paused wall-clock time does not count against `taskStuckTimeoutMs`. Clearing pause state resumes runtime scheduling and gives tracked active sessions a fresh stuck-task grace window before normal detection resumes; when `autoMerge` is enabled, eligible `in-review` tasks are re-swept into the auto-merge queue (paused/blocked/failed review tasks remain skipped). | | `maxConcurrent` | `number` | `2` | Max concurrent task-lane AI agents (planning, executor, merge). Editable from Settings and the Command Center Overview controls dashboard. | +| `maxConcurrentVerifications` | `number` | `1` | Max concurrent verification subprocesses (`fn_run_verification`, merge test/build commands) process-wide. Caps stacked monorepo typecheck/build so concurrent tasks do not peg host CPU. Range **1–8** (clamped at runtime and in Settings). Editable from Settings → Scheduling. Each project engine registers its cap; the effective process limit is the **minimum** of registered project caps. | | `maxTriageConcurrent` | `number` | `2` | Max concurrent planning agents. Editable from Settings and the Command Center Overview controls dashboard. | | `globalMaxConcurrent` | `number` | `4` | System-wide max concurrent agents across all projects. | | `maxWorktrees` | `number` | `4` | Max git worktrees. Editable from Settings and the Command Center Overview controls dashboard. | diff --git a/package.json b/package.json index 31584ae876..ee615fbf07 100644 --- a/package.json +++ b/package.json @@ -31,10 +31,12 @@ "sync:fusion-skill": "node scripts/sync-fusion-skill-tools.mjs", "sync:fusion-skill:check": "node scripts/sync-fusion-skill-tools.mjs --check", "build": "node scripts/build-workspace.mjs", + "build:full": "node scripts/build-workspace.mjs --full", + "build:force": "node scripts/build-workspace.mjs --force", "build:all": "pnpm -r build", - "verify:workspace": "pnpm lint && pnpm test:full && pnpm build", - "build:exe": "pnpm build && pnpm --filter @runfusion/fusion build:exe", - "build:exe:all": "pnpm build && pnpm --filter @runfusion/fusion build:exe:all", + "verify:workspace": "pnpm lint && pnpm test:full && pnpm build:full", + "build:exe": "pnpm build:full && pnpm --filter @runfusion/fusion build:exe", + "build:exe:all": "pnpm build:full && pnpm --filter @runfusion/fusion build:exe:all", "test": "node scripts/test-changed.mjs", "verify:fast": "node scripts/verify-fast.mjs", "test:scripts": "node scripts/run-script-tests.mjs", diff --git a/packages/cli/package.json b/packages/cli/package.json index 4978ea4924..f100797d63 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -47,6 +47,7 @@ "prepack": "node ./scripts/prepare-publish-manifest.mjs prepack", "postpack": "node ./scripts/prepare-publish-manifest.mjs postpack", "build": "tsup", + "build:package": "cross-env FUSION_CLI_FULL_PACKAGE=1 tsup", "build:exe": "bun run build.ts", "build:exe:all": "bun run build.ts --all", "typecheck": "tsc --noEmit", diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.ts index 0f41b46d8c..56acd123c8 100644 --- a/packages/cli/tsup.config.ts +++ b/packages/cli/tsup.config.ts @@ -8,6 +8,21 @@ import { ALL_STAGED_BUNDLED_IDS, RUNTIME_PLUGIN_IDS } from "./src/plugins/staged export { ALL_STAGED_BUNDLED_IDS }; +/* +FNXC:CliPackaging 2026-07-15-03:25: +Local `pnpm build` was spending ~17s on plugin-sdk DTS plus multi-plugin esbuild and optional desktop rebuild on every CLI build. Full packaging (desktop runtime, all staged plugins, self-contained DTS) is only required for publish/CI/release. Default local builds emit bin.js/extension.js + PG migrations; set FUSION_CLI_FULL_PACKAGE=1 (or run under CI=true) for the complete package surface. +*/ +export function wantsFullCliPackage(env: NodeJS.ProcessEnv = process.env): boolean { + const explicit = env.FUSION_CLI_FULL_PACKAGE; + if (explicit === "0" || explicit === "false") return false; + if (explicit === "1" || explicit === "true") return true; + if (env.CI === "true" || env.CI === "1") return true; + if (env.npm_lifecycle_event === "prepack") return true; + return false; +} + +const fullCliPackage = wantsFullCliPackage(); + const RUNTIME_PLUGINS_WITH_MCP_SCHEMA_SERVER = new Set([ "fusion-plugin-openclaw-runtime", "fusion-plugin-droid-runtime", @@ -360,6 +375,31 @@ const cliBuildConfig = { `WARNING: PostgreSQL migrations source not found at ${pgMigrationsSrc}; DATABASE_URL boot will fail to apply schema migrations.`, ); } + + /* + FNXC:CliPackaging 2026-07-15-03:25: + Fast local packaging: migrations + optional dashboard client copy only. Skip desktop ensure-build, multi-plugin esbuild staging, and assertAllStagedBundledPluginsLoadable — those dominate CPU/wall time and are only needed for published artifacts. Prior staged dist/plugins/desktop is left in place if present so a previous full build remains usable. + */ + if (!fullCliPackage) { + if (existsSync(dashboardClientSrc)) { + if (existsSync(dashboardClientDest)) { + rmSync(dashboardClientDest, { recursive: true, force: true }); + } + cpSync(dashboardClientSrc, dashboardClientDest, { recursive: true }); + console.log("Copied dashboard client assets to dist/client/ (fast package mode)"); + } else if (!existsSync(join(dashboardClientDest, "index.html"))) { + mkdirSync(dashboardClientDest, { recursive: true }); + writeFileSync(join(dashboardClientDest, "index.html"), dashboardClientStub, "utf-8"); + console.warn( + `WARNING: Dashboard client assets not found at ${dashboardClientSrc}. Generated minimal stub (fast package mode).`, + ); + } + console.log( + "CLI fast package mode: skipped desktop ensure-build, bundled-plugin staging, and full plugin-sdk DTS. Set FUSION_CLI_FULL_PACKAGE=1 or use `pnpm build:full` for release packaging.", + ); + return; + } + if (existsSync(desktopRuntimeDest)) { rmSync(desktopRuntimeDest, { recursive: true, force: true }); } @@ -539,21 +579,27 @@ const pluginSdkBuildConfig = { platform: "node", target: "node22", tsconfig: join(__dirname, "..", "plugin-sdk", "tsconfig.json"), - dts: { - /* - * FNXC:PluginSDK 2026-06-13-12:00: - * FN-6409 requires the published @runfusion/fusion/plugin-sdk declaration entry to be self-contained. External plugin authors cannot resolve private @fusion/core types from scaffolded projects, so leaving @fusion/* imports in dist/plugin-sdk/index.d.ts makes tsc fail with TS2307 before ctx parameters can typecheck. - */ - resolve: [/^@fusion\//], - compilerOptions: { - rootDir: join(__dirname, ".."), - baseUrl: ".", - paths: { - "@fusion/core": ["../core/src/index.ts"], - }, - removeComments: true, - }, - }, + /* + * FNXC:CliPackaging 2026-07-15-03:25: + * Self-contained plugin-sdk DTS (~17s) is release/publish surface. Local fast builds skip DTS; full package mode (CI / FUSION_CLI_FULL_PACKAGE) keeps FN-6409 resolve behavior. + */ + dts: fullCliPackage + ? { + /* + * FNXC:PluginSDK 2026-06-13-12:00: + * FN-6409 requires the published @runfusion/fusion/plugin-sdk declaration entry to be self-contained. External plugin authors cannot resolve private @fusion/core types from scaffolded projects, so leaving @fusion/* imports in dist/plugin-sdk/index.d.ts makes tsc fail with TS2307 before ctx parameters can typecheck. + */ + resolve: [/^@fusion\//], + compilerOptions: { + rootDir: join(__dirname, ".."), + baseUrl: ".", + paths: { + "@fusion/core": ["../core/src/index.ts"], + }, + removeComments: true, + }, + } + : false, noExternal: [/^@fusion\//], esbuildOptions(options: { alias?: Record }) { options.alias = { diff --git a/packages/core/src/settings-schema.ts b/packages/core/src/settings-schema.ts index e74f4f49d4..38b053e9a5 100644 --- a/packages/core/src/settings-schema.ts +++ b/packages/core/src/settings-schema.ts @@ -343,6 +343,11 @@ export const DEFAULT_PROJECT_SETTINGS = { enginePaused: false, engineLastActiveAt: undefined, maxConcurrent: 2, + /* + FNXC:VerificationConcurrency 2026-07-15-03:35: + Default one verification at a time process-wide so concurrent tasks cannot each run verify:fast / full builds simultaneously and peg the host. Operators with spare cores may raise this in Scheduling settings (clamped 1–8 at runtime). + */ + maxConcurrentVerifications: 1, maxTriageConcurrent: 2, globalMaxConcurrent: 4, maxWorktrees: 4, diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index c49a1e5703..cb936391b8 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -3863,6 +3863,11 @@ export interface ProjectSettings { /** Maximum number of concurrent AI agents across all activity types * (triage specification, task execution, and merge operations). */ maxConcurrent: number; + /** + * FNXC:VerificationConcurrency 2026-07-15-03:35: + * Max concurrent verification subprocesses (fn_run_verification / merge testCommand builds) across all tasks in this process. Caps stacked monorepo typecheck/build pegging CPU when many tasks are in-progress. Default 1. Raise only on high-core hosts. + */ + maxConcurrentVerifications?: number; /** Maximum number of concurrent triage/specification agents. When undefined, * falls back to maxConcurrent. */ maxTriageConcurrent?: number; diff --git a/packages/dashboard/app/components/SettingsModal.tsx b/packages/dashboard/app/components/SettingsModal.tsx index 34f6cbdec6..92311eff92 100644 --- a/packages/dashboard/app/components/SettingsModal.tsx +++ b/packages/dashboard/app/components/SettingsModal.tsx @@ -1006,6 +1006,7 @@ export function SettingsModal({ const sessionBannersHidden = useSessionBannersHidden(); const [form, setForm] = useState({ maxConcurrent: 2, + maxConcurrentVerifications: 1, maxTriageConcurrent: 2, maxWorktrees: 4, pollIntervalMs: 15000, diff --git a/packages/dashboard/app/components/settings/section-keys.ts b/packages/dashboard/app/components/settings/section-keys.ts index ee8c74a904..b5772c0467 100644 --- a/packages/dashboard/app/components/settings/section-keys.ts +++ b/packages/dashboard/app/components/settings/section-keys.ts @@ -103,6 +103,7 @@ const PROJECT_SECTION_KEYS: Record = { "heartbeatScopeDiscipline", "ignoreHiddenOverlapPaths", "maxConcurrent", + "maxConcurrentVerifications", "maxStuckKills", "maxTriageConcurrent", "overlapIgnorePaths", diff --git a/packages/dashboard/app/components/settings/sections/SchedulingSection.tsx b/packages/dashboard/app/components/settings/sections/SchedulingSection.tsx index efe3c098a2..f0e9836458 100644 --- a/packages/dashboard/app/components/settings/sections/SchedulingSection.tsx +++ b/packages/dashboard/app/components/settings/sections/SchedulingSection.tsx @@ -64,6 +64,20 @@ export function SchedulingSection({ scopeBanner, form, setForm, globalMaxConcurr }}/> {t("settings.scheduling.maxConcurrentTasksHint", "Default: 2.")} +
+ + { + const val = e.target.value; + if (val === "") { + setForm((f) => ({ ...f, maxConcurrentVerifications: undefined } as SettingsFormState)); + return; + } + // FNXC:VerificationConcurrency 2026-07-15-08:20: Clamp to 1–8 on the form path so UI cannot persist values outside the engine hard cap. + const n = Math.min(8, Math.max(1, Math.floor(Number(val)) || 1)); + setForm((f) => ({ ...f, maxConcurrentVerifications: n } as SettingsFormState)); + }}/> + {t("settings.scheduling.maxConcurrentVerificationsHint", "Caps stacked typecheck/build verification across tasks. Default: 1. Range: 1–8.")} +
{ diff --git a/packages/dashboard/app/components/settings/sections/__tests__/settings-default-descriptions.test.tsx b/packages/dashboard/app/components/settings/sections/__tests__/settings-default-descriptions.test.tsx index d4375bb353..21f382612d 100644 --- a/packages/dashboard/app/components/settings/sections/__tests__/settings-default-descriptions.test.tsx +++ b/packages/dashboard/app/components/settings/sections/__tests__/settings-default-descriptions.test.tsx @@ -192,6 +192,7 @@ const SETTING_DESCRIPTION_KEYS: Record = { // SchedulingSection globalMaxConcurrent: "scheduling.maximumConcurrentAgentsAcrossAllProjects", maxConcurrent: "scheduling.maxConcurrentTasksHint", + maxConcurrentVerifications: "scheduling.maxConcurrentVerificationsHint", maxTriageConcurrent: "scheduling.maximumConcurrentPlanningAgents", pollIntervalMs: "scheduling.pollIntervalMsHint", heartbeatScopeDiscipline: "scheduling.strictDefault", diff --git a/packages/engine/src/__tests__/verification-concurrency.test.ts b/packages/engine/src/__tests__/verification-concurrency.test.ts new file mode 100644 index 0000000000..456f6c53fe --- /dev/null +++ b/packages/engine/src/__tests__/verification-concurrency.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it, beforeEach } from "vitest"; +import { + clampMaxConcurrentVerifications, + getMaxConcurrentVerifications, + getVerificationSemaphore, + MAX_CONCURRENT_VERIFICATIONS_HARD_CAP, + registerProjectVerificationLimit, + resetVerificationLimitRegistryForTests, + setMaxConcurrentVerifications, + unregisterProjectVerificationLimit, + withVerificationSlot, +} from "../verification-concurrency.js"; + +describe("verification concurrency", () => { + beforeEach(() => { + resetVerificationLimitRegistryForTests(); + setMaxConcurrentVerifications(1); + // Drain any leaked active count from other tests (should be 0). + getVerificationSemaphore().reconcileActiveCount(0); + }); + + it("defaults to one concurrent verification", () => { + expect(getMaxConcurrentVerifications()).toBe(1); + }); + + it("clamps limit to 1–8", () => { + expect(clampMaxConcurrentVerifications(0)).toBe(1); + expect(clampMaxConcurrentVerifications(-3)).toBe(1); + expect(clampMaxConcurrentVerifications(50)).toBe(MAX_CONCURRENT_VERIFICATIONS_HARD_CAP); + expect(clampMaxConcurrentVerifications(3.9)).toBe(3); + setMaxConcurrentVerifications(99); + expect(getMaxConcurrentVerifications()).toBe(8); + }); + + it("uses the minimum of registered project limits (most restrictive wins)", () => { + registerProjectVerificationLimit("proj-a", 8); + registerProjectVerificationLimit("proj-b", 1); + expect(getMaxConcurrentVerifications()).toBe(1); + unregisterProjectVerificationLimit("proj-b"); + expect(getMaxConcurrentVerifications()).toBe(8); + unregisterProjectVerificationLimit("proj-a"); + setMaxConcurrentVerifications(2); + expect(getMaxConcurrentVerifications()).toBe(2); + }); + + it("serializes overlapping withVerificationSlot callers when limit is 1", async () => { + const order: string[] = []; + let releaseFirst!: () => void; + const firstGate = new Promise((resolve) => { + releaseFirst = resolve; + }); + + const first = withVerificationSlot(async () => { + order.push("first-enter"); + await firstGate; + order.push("first-exit"); + }); + + // Let first acquire the slot. + await Promise.resolve(); + await Promise.resolve(); + + const second = withVerificationSlot(async () => { + order.push("second"); + }); + + // Second must not run while first holds the only slot. + await Promise.resolve(); + await Promise.resolve(); + expect(order).toEqual(["first-enter"]); + + releaseFirst(); + await Promise.all([first, second]); + expect(order).toEqual(["first-enter", "first-exit", "second"]); + }); + + it("allows two concurrent slots when limit is 2", async () => { + setMaxConcurrentVerifications(2); + let concurrent = 0; + let peak = 0; + + await Promise.all( + [1, 2].map(() => + withVerificationSlot(async () => { + concurrent++; + peak = Math.max(peak, concurrent); + await new Promise((r) => setTimeout(r, 20)); + concurrent--; + }), + ), + ); + + expect(peak).toBe(2); + }); + + it("rejects with AbortError when aborted while queued for a slot", async () => { + setMaxConcurrentVerifications(1); + let releaseHolder!: () => void; + const holderGate = new Promise((resolve) => { + releaseHolder = resolve; + }); + + const holder = withVerificationSlot(async () => { + await holderGate; + }); + await Promise.resolve(); + await Promise.resolve(); + + const ac = new AbortController(); + const waiting = withVerificationSlot(async () => "ran", ac.signal); + await Promise.resolve(); + await Promise.resolve(); + ac.abort(); + + await expect(waiting).rejects.toMatchObject({ name: "AbortError" }); + releaseHolder(); + await holder; + }); +}); diff --git a/packages/engine/src/concurrency.ts b/packages/engine/src/concurrency.ts index 5dd791c9ed..a72d2a20fc 100644 --- a/packages/engine/src/concurrency.ts +++ b/packages/engine/src/concurrency.ts @@ -14,10 +14,25 @@ export const PRIORITY_SPECIFY = 0; interface PriorityWaiter { priority: number; resolve: () => void; + /** Optional reject for abortable acquires — not used by the priority drain path. */ + reject?: (err: Error) => void; } export const IDLE_SEMAPHORE_LEAK_REPAIR_MS = 5_000; +function createAbortError(): Error { + if (typeof DOMException === "function") { + try { + return new DOMException("The operation was aborted", "AbortError"); + } catch { + // fall through + } + } + const err = new Error("The operation was aborted"); + err.name = "AbortError"; + return err; +} + /* FNXC:GlobalConcurrencyControls 2026-07-14-18:30: Operators reported live running-agent counts above the global concurrency cap (e.g. 5 running with cap 4). Live utilization counts every top-level slot holder (in-progress, planning triage, active in-review), but the scheduler only preflighted capacity and acquired the shared semaphore later inside the executor — so a card could sit in-progress (and count as running) while triage still saw free semaphore slots and filled the rest of the cap. Pre-held executor slots close that gap: tryAcquire before todo→in-progress, keep the slot until the executor/graph run claims and releases it, and admit triage against max(semaphore.activeCount, live running count). @@ -241,21 +256,47 @@ export class AgentSemaphore { * @param priority - Numeric priority (higher = served first). Defaults to `0` * ({@link PRIORITY_SPECIFY}). Use {@link PRIORITY_MERGE} (`2`) for merge * agents and {@link PRIORITY_EXECUTE} (`1`) for execution agents. + * @param signal - Optional AbortSignal. When aborted while queued, the waiter + * is removed and the promise rejects with an AbortError so cancelled + * verification/merge work does not block the queue forever. */ - acquire(priority: number = 0): Promise { + acquire(priority: number = 0, signal?: AbortSignal): Promise { const limit = this.limit; // Uses the guarded getter (returns min 1) + if (signal?.aborted) { + return Promise.reject(createAbortError()); + } if (this._active < limit) { this._active++; return Promise.resolve(); } - return new Promise((resolve) => { - this._waiters.push({ + return new Promise((resolve, reject) => { + let settled = false; + const waiter: PriorityWaiter = { priority, resolve: () => { + if (settled) return; + settled = true; + cleanup(); this._active++; resolve(); }, - }); + reject: (err: Error) => { + if (settled) return; + settled = true; + cleanup(); + reject(err); + }, + }; + const onAbort = () => { + const idx = this._waiters.indexOf(waiter); + if (idx >= 0) this._waiters.splice(idx, 1); + waiter.reject?.(createAbortError()); + }; + const cleanup = () => { + signal?.removeEventListener("abort", onAbort); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + this._waiters.push(waiter); }); } diff --git a/packages/engine/src/project-engine.ts b/packages/engine/src/project-engine.ts index 6e22473a17..9fb8a19f83 100644 --- a/packages/engine/src/project-engine.ts +++ b/packages/engine/src/project-engine.ts @@ -66,6 +66,10 @@ import { sweepStaleAutostashes, VerificationError } from "./merger.js"; import { runAiMerge, landWorkspaceTask, WorkspacePartialLandError, WorkspaceRepoLandBusyError } from "./merger-ai.js"; import { promoteBranchGroup, type BranchGroupPromotionResult, type CreateGroupPrFn, type SyncGroupPrFn } from "./group-merge-coordinator.js"; import { PRIORITY_MERGE } from "./concurrency.js"; +import { + registerProjectVerificationLimit, + unregisterProjectVerificationLimit, +} from "./verification-concurrency.js"; import { runtimeLog } from "./logger.js"; import type { HeartbeatTriggerScheduler } from "./agent-heartbeat.js"; import { ResearchOrchestrator } from "./research-orchestrator.js"; @@ -971,6 +975,15 @@ export class ProjectEngine { // 5. Wire settings event listeners this.wireSettingsListeners(store); + /* + FNXC:VerificationConcurrency 2026-07-15-08:20: + Apply maxConcurrentVerifications once at start (and on settings:updated) so verification + slots do not re-race last-writer-wins on every fn_run_verification / merge command. + + FNXC:VerificationConcurrency 2026-07-15-09:05: + Register per-project so multi-engine hosts take the MIN of all caps (most restrictive wins). + */ + registerProjectVerificationLimit(this.config.projectId, settings.maxConcurrentVerifications ?? 1); // 6. Wire auto-merge on task:moved and task:updated pause interruptions this.wireAutoMerge(store, cwd); @@ -1005,6 +1018,8 @@ export class ProjectEngine { } this.shuttingDown = true; + // FNXC:VerificationConcurrency 2026-07-15-09:05: Drop this project's cap so it no longer pins process min. + unregisterProjectVerificationLimit(this.config.projectId); // Stop merge retry timer if (this.mergeRetryTimer) { @@ -4855,6 +4870,23 @@ export class ProjectEngine { store.on("settings:updated", onStuckTimeoutChange); this.settingsHandlers.push(onStuckTimeoutChange); + // 7b. Verification concurrency — process-wide slot cap (clamped 1–8, min across projects) + const onVerificationConcurrencyChange = ({ + settings: s, + previous: prev, + }: { + settings: Settings; + previous: Settings; + }) => { + if (s.maxConcurrentVerifications === prev.maxConcurrentVerifications) return; + registerProjectVerificationLimit(this.config.projectId, s.maxConcurrentVerifications ?? 1); + runtimeLog.log( + `maxConcurrentVerifications updated for ${this.config.projectId} to ${s.maxConcurrentVerifications ?? 1}`, + ); + }; + store.on("settings:updated", onVerificationConcurrencyChange); + this.settingsHandlers.push(onVerificationConcurrencyChange); + // 8. Memory maintenance settings change — sync automations const onInsightSettingsChange = async ({ settings: s, diff --git a/packages/engine/src/run-verification-tool.ts b/packages/engine/src/run-verification-tool.ts index 7b39bb2a35..f3fc8f962e 100644 --- a/packages/engine/src/run-verification-tool.ts +++ b/packages/engine/src/run-verification-tool.ts @@ -24,6 +24,7 @@ import { isAbsolute, join, relative } from "node:path"; import { Type, type Static } from "@earendil-works/pi-ai"; import type { ToolDefinition } from "@earendil-works/pi-coding-agent"; import { executorLog } from "./logger.js"; +import { withVerificationSlot } from "./verification-concurrency.js"; // --------------------------------------------------------------------------- // Constants @@ -465,6 +466,10 @@ export interface RunVerificationOptions { expectFailure?: boolean; onHeartbeat: () => void; onLine?: (line: string) => void; + /** When true, skip the process-wide verification slot (tests only). */ + bypassVerificationSlot?: boolean; + /** Optional abort signal — cancels while queued for a verification slot and during the command. */ + signal?: AbortSignal; } /** @@ -472,9 +477,26 @@ export interface RunVerificationOptions { * heartbeats, and hard timeout enforcement. * * Exported so tests can exercise the core logic without a full agent session. + * + * FNXC:VerificationConcurrency 2026-07-15-03:35: + * Acquires a process-wide verification slot so concurrent tasks cannot stack + * multiple monorepo typecheck/build commands and peg the host CPU. + * + * FNXC:VerificationConcurrency 2026-07-15-08:20: + * Limit is process-wide and set from engine settings load/update only (not per call) + * so multi-project verifications cannot race last-writer-wins on the slot cap. */ export async function runVerificationCommand( opts: RunVerificationOptions, +): Promise { + if (opts.bypassVerificationSlot) { + return runVerificationCommandUnlocked(opts); + } + return withVerificationSlot(() => runVerificationCommandUnlocked(opts), opts.signal); +} + +async function runVerificationCommandUnlocked( + opts: RunVerificationOptions, ): Promise { const { command, cwd, timeoutMs, expectFailure = false, onHeartbeat, onLine } = opts; const startMs = Date.now(); @@ -671,7 +693,16 @@ export interface CreateRunVerificationToolOpts { export function createRunVerificationTool( opts: CreateRunVerificationToolOpts, ): ToolDefinition { - const { worktreePath, rootDir, taskId, recordActivity, verificationCommandTimeoutMs, onVerificationStart, onVerificationEnd, log } = opts; + const { + worktreePath, + rootDir, + taskId, + recordActivity, + verificationCommandTimeoutMs, + onVerificationStart, + onVerificationEnd, + log, + } = opts; return { name: "fn_run_verification", diff --git a/packages/engine/src/verification-concurrency.ts b/packages/engine/src/verification-concurrency.ts new file mode 100644 index 0000000000..8f6f843233 --- /dev/null +++ b/packages/engine/src/verification-concurrency.ts @@ -0,0 +1,105 @@ +/* +FNXC:VerificationConcurrency 2026-07-15-03:35: +Multiple in-progress tasks each calling fn_run_verification (often `pnpm verify:fast` / full typecheck+build) pegged CPU by running several monorepo compiles in parallel. Cap concurrent verification subprocesses project-wide so task concurrency can stay higher without stacking heavy builds. Default limit is 1; operators raise maxConcurrentVerifications when the machine has spare cores. + +FNXC:VerificationConcurrency 2026-07-15-08:20: +Greptile P1/P2: (1) clamp 1–8 so programmatic settings cannot open 50 slots; (2) do not re-set the process limit on every verification start (multi-project races last-writer-wins) — wire the limit from engine settings load/update only; (3) honor AbortSignal while queued so cancelled merge/verification does not block the slot queue. + +FNXC:VerificationConcurrency 2026-07-15-09:05: +Greptile P1 multi-project: multiple ProjectEngine instances must not last-write the singleton limit. Register each project's desired cap; the effective process limit is the MIN of registered caps (most restrictive wins) so a project set to 1 cannot be overridden by a peer set to 8. +*/ +import { AgentSemaphore, PRIORITY_EXECUTE } from "./concurrency.js"; + +/** Hard ceiling matching the Scheduling UI max. */ +export const MAX_CONCURRENT_VERIFICATIONS_HARD_CAP = 8; +/** Floor — at least one verification can always run. */ +export const MIN_CONCURRENT_VERIFICATIONS = 1; + +/** projectId -> clamped desired limit for that engine instance */ +const projectLimits = new Map(); +let fallbackLimit = MIN_CONCURRENT_VERIFICATIONS; +const verificationSemaphore = new AgentSemaphore(() => resolveEffectiveLimit()); + +/** + * Clamp a raw setting/API value into the enforced verification concurrency range. + */ +export function clampMaxConcurrentVerifications(next: number): number { + if (!Number.isFinite(next)) return MIN_CONCURRENT_VERIFICATIONS; + return Math.min( + MAX_CONCURRENT_VERIFICATIONS_HARD_CAP, + Math.max(MIN_CONCURRENT_VERIFICATIONS, Math.floor(next)), + ); +} + +function resolveEffectiveLimit(): number { + if (projectLimits.size === 0) return fallbackLimit; + let min = MAX_CONCURRENT_VERIFICATIONS_HARD_CAP; + for (const value of projectLimits.values()) { + if (value < min) min = value; + } + return min; +} + +/** + * Register or update one project's desired verification concurrency. + * Effective process limit = min(registered project caps). + */ +export function registerProjectVerificationLimit(projectId: string, next: number): void { + if (!projectId) return; + projectLimits.set(projectId, clampMaxConcurrentVerifications(next)); +} + +/** + * Drop a project's registration when its engine stops so stale caps do not pin the min forever. + */ +export function unregisterProjectVerificationLimit(projectId: string): void { + if (!projectId) return; + projectLimits.delete(projectId); +} + +/** + * Legacy setter used by tests and single-engine paths without a project id. + * Sets the fallback limit when no projects are registered; when projects are + * registered this is ignored for the effective min (use registerProjectVerificationLimit). + */ +export function setMaxConcurrentVerifications(next: number): void { + fallbackLimit = clampMaxConcurrentVerifications(next); +} + +/** Current effective verification concurrency limit (after clamping / min aggregation). */ +export function getMaxConcurrentVerifications(): number { + return verificationSemaphore.limit; +} + +/** Test helper: clear project registrations. */ +export function resetVerificationLimitRegistryForTests(): void { + projectLimits.clear(); + fallbackLimit = MIN_CONCURRENT_VERIFICATIONS; +} + +/** + * Run `fn` while holding one verification slot. Waiters queue at execute priority. + * When `signal` aborts while queued, the waiter is removed and the promise rejects + * with AbortError so cancelled work does not block the queue. + */ +export async function withVerificationSlot( + fn: () => Promise, + signal?: AbortSignal, +): Promise { + await verificationSemaphore.acquire(PRIORITY_EXECUTE, signal); + try { + if (signal?.aborted) { + const err = new Error("The operation was aborted"); + err.name = "AbortError"; + throw err; + } + return await fn(); + } finally { + verificationSemaphore.release(); + } +} + +/** Test/diagnostic access to the underlying semaphore. */ +export function getVerificationSemaphore(): AgentSemaphore { + return verificationSemaphore; +} diff --git a/packages/engine/src/verification-utils.ts b/packages/engine/src/verification-utils.ts index 4159dc11d1..edd613a02f 100644 --- a/packages/engine/src/verification-utils.ts +++ b/packages/engine/src/verification-utils.ts @@ -5,6 +5,7 @@ import type { TaskStore, AgentRole } from "@fusion/core"; import { resolveSandboxBackend } from "./sandbox/index.js"; import type { SandboxBackend, SandboxRunStreamingOptions, SandboxStreamingResult } from "./sandbox/index.js"; +import { withVerificationSlot } from "./verification-concurrency.js"; // ── Constants ────────────────────────────────────────────────────────── @@ -372,6 +373,46 @@ export async function runVerificationCommand( * state (required for safe concurrent verification — see mission-verification). */ backend?: SandboxBackend, +): Promise { + /* + FNXC:VerificationConcurrency 2026-07-15-03:35: + Merge/mission verification shares the process-wide verification slot with fn_run_verification so stacked monorepo builds cannot run unbounded across concurrent tasks. + + FNXC:VerificationConcurrency 2026-07-15-08:20: + Pass signal into the slot wait so cancelled merge verification leaves the queue. + Limit is process-wide from engine settings wiring, not re-set per call. + */ + return withVerificationSlot( + () => + runVerificationCommandUnlocked( + store, + rootDir, + taskId, + command, + type, + signal, + log, + agentLabel, + extraEnv, + timeoutMsOverride, + backend, + ), + signal, + ); +} + +async function runVerificationCommandUnlocked( + store: TaskStore, + rootDir: string, + taskId: string, + command: string, + type: "test" | "build", + signal: AbortSignal | undefined, + log?: { log: (message: string, ...args: unknown[]) => void; error: (message: string, ...args: unknown[]) => void; warn: (message: string, ...args: unknown[]) => void }, + agentLabel?: string, + extraEnv?: NodeJS.ProcessEnv, + timeoutMsOverride?: number, + backend?: SandboxBackend, ): Promise { const logger = log ?? { log: console.log, error: console.error, warn: console.warn }; const label = (agentLabel ?? "merger") as AgentRole; diff --git a/packages/i18n/locales/en/app.json b/packages/i18n/locales/en/app.json index 9967e96749..fa0ec0555f 100644 --- a/packages/i18n/locales/en/app.json +++ b/packages/i18n/locales/en/app.json @@ -6654,6 +6654,8 @@ "browseWorkspacePath": "Browse workspace path", "overlapPickerNote": "Choose a file to ignore directly, or navigate into a folder and select the current directory.", "maxConcurrentTasksHint": "Default: 2.", + "maxConcurrentVerifications": "Max Concurrent Verifications", + "maxConcurrentVerificationsHint": "Caps stacked typecheck/build verification across tasks. Default: 1. Range: 1–8.", "pollIntervalMsHint": "Default: 15000 (15 seconds)." }, "scope": { diff --git a/packages/i18n/locales/es/app.json b/packages/i18n/locales/es/app.json index d5fd615a46..787793c0b7 100644 --- a/packages/i18n/locales/es/app.json +++ b/packages/i18n/locales/es/app.json @@ -6634,6 +6634,8 @@ "ignoreHiddenDotPathsHelp": "", "ignoreHiddenDotPathsInOverlapChecks": "", "maxConcurrentTasksHint": "", + "maxConcurrentVerifications": "", + "maxConcurrentVerificationsHint": "", "pollIntervalMsHint": "" }, "scope": { diff --git a/packages/i18n/locales/fr/app.json b/packages/i18n/locales/fr/app.json index 046bc0f13d..48bd485e05 100644 --- a/packages/i18n/locales/fr/app.json +++ b/packages/i18n/locales/fr/app.json @@ -6634,6 +6634,8 @@ "ignoreHiddenDotPathsHelp": "", "ignoreHiddenDotPathsInOverlapChecks": "", "maxConcurrentTasksHint": "", + "maxConcurrentVerifications": "", + "maxConcurrentVerificationsHint": "", "pollIntervalMsHint": "" }, "scope": { diff --git a/packages/i18n/locales/ko/app.json b/packages/i18n/locales/ko/app.json index cd0886c740..8335a482e5 100644 --- a/packages/i18n/locales/ko/app.json +++ b/packages/i18n/locales/ko/app.json @@ -6634,6 +6634,8 @@ "ignoreHiddenDotPathsHelp": "", "ignoreHiddenDotPathsInOverlapChecks": "", "maxConcurrentTasksHint": "", + "maxConcurrentVerifications": "", + "maxConcurrentVerificationsHint": "", "pollIntervalMsHint": "" }, "scope": { diff --git a/packages/i18n/locales/zh-CN/app.json b/packages/i18n/locales/zh-CN/app.json index a8b8349092..87e4497033 100644 --- a/packages/i18n/locales/zh-CN/app.json +++ b/packages/i18n/locales/zh-CN/app.json @@ -6634,6 +6634,8 @@ "ignoreHiddenDotPathsHelp": "", "ignoreHiddenDotPathsInOverlapChecks": "", "maxConcurrentTasksHint": "", + "maxConcurrentVerifications": "", + "maxConcurrentVerificationsHint": "", "pollIntervalMsHint": "" }, "scope": { diff --git a/packages/i18n/locales/zh-TW/app.json b/packages/i18n/locales/zh-TW/app.json index 3b83479f56..6a31ce1e00 100644 --- a/packages/i18n/locales/zh-TW/app.json +++ b/packages/i18n/locales/zh-TW/app.json @@ -6634,6 +6634,8 @@ "ignoreHiddenDotPathsHelp": "", "ignoreHiddenDotPathsInOverlapChecks": "", "maxConcurrentTasksHint": "", + "maxConcurrentVerifications": "", + "maxConcurrentVerificationsHint": "", "pollIntervalMsHint": "" }, "scope": { diff --git a/packages/i18n/src/resources.d.ts b/packages/i18n/src/resources.d.ts index 866bf74b21..1197cb5dac 100644 --- a/packages/i18n/src/resources.d.ts +++ b/packages/i18n/src/resources.d.ts @@ -6647,6 +6647,8 @@ export default interface Resources { "lite": "Lite", "maxConcurrentTasks": "Max Concurrent Tasks", "maxConcurrentTasksHint": "Default: 2.", + "maxConcurrentVerifications": "Max Concurrent Verifications", + "maxConcurrentVerificationsHint": "Caps stacked typecheck/build verification across tasks. Default: 1. Range: 1–8.", "maxStuckRetries": "Max Stuck Retries", "maxTriageConcurrent": "Max Triage Concurrent", "maximumAgeInHoursBeforeAPlanIs": "Maximum age in hours before a plan is considered stale. Default: 6 hours.", diff --git a/packages/plugin-sdk/tsconfig.json b/packages/plugin-sdk/tsconfig.json index 88ca8c35c8..d59ec35fac 100644 --- a/packages/plugin-sdk/tsconfig.json +++ b/packages/plugin-sdk/tsconfig.json @@ -15,6 +15,8 @@ "declaration": true, "declarationMap": true, "sourceMap": true, + "incremental": true, + "tsBuildInfoFile": "./dist/.tsbuildinfo", "skipLibCheck": true, "strict": true, "esModuleInterop": true, diff --git a/scripts/__tests__/build-workspace.test.mjs b/scripts/__tests__/build-workspace.test.mjs index 86bfc5cd5d..7111aeb6eb 100644 --- a/scripts/__tests__/build-workspace.test.mjs +++ b/scripts/__tests__/build-workspace.test.mjs @@ -10,9 +10,11 @@ import { PLUGIN_BUILD_GLOBAL_INPUT_PATHS, computePluginSourceHash, discoverWorkspacePackages, + ensureFullPackageCliPlanned, planWorkspaceBuild, readPluginBuildCache, requiredPluginOutputs, + wantsFullCliPackage, } from "../build-workspace.mjs"; function createWorkspace() { @@ -93,23 +95,61 @@ test("discovers workspace packages and classifies plugin directories", () => { }); }); -test("non-plugin build packages stay planned while unchanged plugins with outputs and cache are skipped", () => { +test("unchanged packages with outputs and cache are skipped (plugins and non-plugins)", () => { withWorkspace((root) => { writePluginDist(root); + // Core required outputs from src/index.ts → packages/core/dist/index.js + mkdirSync(path.join(root, "packages/core", "dist"), { recursive: true }); + writeFileSync(path.join(root, "packages/core", "dist", "index.js"), "export const core = 1;\n"); initGit(root); const packages = discoverWorkspacePackages(root); const plugin = packageByName(packages, "@fusion-plugin-examples/alpha"); - const hash = computePluginSourceHash(plugin, root); - const cache = { version: BUILD_CACHE_VERSION, entries: { [plugin.name]: { sourceHash: hash } } }; + const core = packageByName(packages, "@fusion/core"); + const pluginHash = computePluginSourceHash(plugin, root); + const coreHash = computePluginSourceHash(core, root); + const cache = { + version: BUILD_CACHE_VERSION, + entries: { + [plugin.name]: { sourceHash: pluginHash }, + [core.name]: { sourceHash: coreHash }, + }, + }; const plan = planWorkspaceBuild({ rootDir: root, packages, cache }); - assert.deepEqual(plan.plannedPackages.map((pkg) => pkg.name), ["@fusion/core"]); - assert.deepEqual(plan.skippedPlugins.map((pkg) => pkg.name), ["@fusion-plugin-examples/alpha"]); + assert.deepEqual(plan.plannedPackages.map((pkg) => pkg.name), []); + assert.deepEqual( + (plan.skippedPackages ?? plan.skippedPlugins).map((pkg) => pkg.name).sort(), + ["@fusion-plugin-examples/alpha", "@fusion/core"].sort(), + ); assert.deepEqual(plan.excludedPackages.map((pkg) => pkg.name), ["@fusion/desktop"]); }); }); +test("force rebuilds packages even when cache matches", () => { + withWorkspace((root) => { + writePluginDist(root); + mkdirSync(path.join(root, "packages/core", "dist"), { recursive: true }); + writeFileSync(path.join(root, "packages/core", "dist", "index.js"), "export const core = 1;\n"); + initGit(root); + const packages = discoverWorkspacePackages(root); + const plugin = packageByName(packages, "@fusion-plugin-examples/alpha"); + const core = packageByName(packages, "@fusion/core"); + const cache = { + version: BUILD_CACHE_VERSION, + entries: { + [plugin.name]: { sourceHash: computePluginSourceHash(plugin, root) }, + [core.name]: { sourceHash: computePluginSourceHash(core, root) }, + }, + }; + + const plan = planWorkspaceBuild({ rootDir: root, packages, cache, force: true }); + assert.ok(plan.plannedPackages.some((pkg) => pkg.name === "@fusion/core")); + assert.ok(plan.plannedPackages.some((pkg) => pkg.name === "@fusion-plugin-examples/alpha")); + assert.equal(packageByName(plan.plannedPackages, "@fusion/core").buildReason, "force"); + }); +}); + test("plugin packages build when required outputs are missing even with a matching cache", () => { withWorkspace((root) => { initGit(root); @@ -251,3 +291,32 @@ test("root package build script points at the workspace build wrapper", () => { assert.equal(rootPackage.scripts.build, "node scripts/build-workspace.mjs"); }); + +test("full package mode force-includes CLI even when content-hash would skip it", () => { + const skipped = [ + { name: "@runfusion/fusion", isPlugin: false, buildReason: "unchanged", sourceHash: "abc" }, + { name: "@fusion/core", isPlugin: false, buildReason: "unchanged", sourceHash: "def" }, + ]; + const { plannedPackages, skippedPackages } = ensureFullPackageCliPlanned([], skipped, { fullPackage: true }); + assert.equal(plannedPackages.length, 1); + assert.equal(plannedPackages[0].name, "@runfusion/fusion"); + assert.equal(plannedPackages[0].buildReason, "full-package"); + assert.deepEqual(skippedPackages.map((p) => p.name), ["@fusion/core"]); +}); + +test("full package mode is a no-op when CLI already planned", () => { + const planned = [{ name: "@runfusion/fusion", buildReason: "changed-inputs" }]; + const skipped = [{ name: "@fusion/core", buildReason: "unchanged" }]; + const result = ensureFullPackageCliPlanned(planned, skipped, { fullPackage: true }); + assert.equal(result.plannedPackages.length, 1); + assert.equal(result.plannedPackages[0].buildReason, "changed-inputs"); +}); + +test("wantsFullCliPackage matches CLI packaging env rules", () => { + assert.equal(wantsFullCliPackage({}, { fullFlag: false }), false); + assert.equal(wantsFullCliPackage({}, { fullFlag: true }), true); + assert.equal(wantsFullCliPackage({ CI: "true" }, { fullFlag: false }), true); + assert.equal(wantsFullCliPackage({ FUSION_CLI_FULL_PACKAGE: "1" }, { fullFlag: false }), true); + assert.equal(wantsFullCliPackage({ FUSION_CLI_FULL_PACKAGE: "0", CI: "true" }, { fullFlag: true }), false); + assert.equal(wantsFullCliPackage({ npm_lifecycle_event: "prepack" }, { fullFlag: false }), true); +}); diff --git a/scripts/build-workspace.mjs b/scripts/build-workspace.mjs index 3143afc765..7e5cf44112 100644 --- a/scripts/build-workspace.mjs +++ b/scripts/build-workspace.mjs @@ -2,6 +2,9 @@ /* FNXC:WorkspaceBuild 2026-06-30-00:00: Root builds may skip unchanged plugin workspaces to keep local and CI feedback fast, but only after required dist outputs exist and a content hash proves plugin package inputs match the last successful plugin build. Non-plugin packages still build every run so the root command preserves the pre-existing recursive build contract outside plugins. + +FNXC:WorkspaceBuild 2026-07-15-03:20: +Root `pnpm build` was pegging CPU for ~2 minutes even when nothing changed: non-plugin packages always rebuilt, CLI packaging always staged desktop + 15 plugins + DTS, and tsc had no incremental cache. Extend the content-hash skip cache to ALL workspace packages (not just plugins), support `--force` / `--full`, and default CLI packaging to a fast local mode (full package on CI or FUSION_CLI_FULL_PACKAGE=1). */ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; @@ -17,10 +20,14 @@ import { readJsonCache, } from "./lib/content-hash.mjs"; -export const BUILD_CACHE_VERSION = 1; +/* +FNXC:WorkspaceBuild 2026-07-15-03:20: +Bump cache version when the skip contract expands from plugins-only to all packages so stale plugin-only entries cannot incorrectly interact with the broader skip set. File name stays plugin-build-cache.json for path stability under .fusion/cache. +*/ +export const BUILD_CACHE_VERSION = 2; export const BUILD_CACHE_FILE = "plugin-build-cache.json"; export const ROOT_BUILD_EXCLUDED_PACKAGES = new Set(["@fusion/desktop", "@fusion/mobile"]); -export const PLUGIN_BUILD_GLOBAL_INPUT_PATHS = [ +export const PACKAGE_BUILD_GLOBAL_INPUT_PATHS = [ "package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml", @@ -30,6 +37,8 @@ export const PLUGIN_BUILD_GLOBAL_INPUT_PATHS = [ "scripts/build-workspace.mjs", "scripts/lib/content-hash.mjs", ]; +/** @deprecated Use PACKAGE_BUILD_GLOBAL_INPUT_PATHS — kept for existing tests. */ +export const PLUGIN_BUILD_GLOBAL_INPUT_PATHS = PACKAGE_BUILD_GLOBAL_INPUT_PATHS; const scriptDir = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(scriptDir, ".."); @@ -127,8 +136,8 @@ export function discoverWorkspacePackages(rootDir, patterns = readWorkspacePacka const packagesByName = new Map(packages.map((pkg) => [pkg.name, pkg])); for (const pkg of packages) { - if (!pkg.isPlugin) continue; - pkg.inputPaths = collectPluginHashInputPaths(pkg, packagesByName); + // FNXC:WorkspaceBuild 2026-07-15-03:20: Hash inputs for every package (plugins and non-plugins) so core/engine/dashboard/cli can skip when unchanged. + pkg.inputPaths = collectPackageHashInputPaths(pkg, packagesByName); } return packages; @@ -141,10 +150,10 @@ function declaredDependencyNames(manifest) { } /** - * Resolve a plugin's content-hash input directories. Include local workspace + * Resolve a package's content-hash input directories. Include local workspace * dependency directories and root build config/tooling files as invalidators so - * skipping a plugin cannot hide a compile break against changed shared package - * types, TypeScript settings, pnpm resolution, or build wrapper behavior. + * skipping cannot hide a compile break against changed shared package types, + * TypeScript settings, pnpm resolution, or build wrapper behavior. * * FNXC:WorkspaceBuild 2026-06-30-00:00: * Plugin skip decisions must include declared local workspace dependencies and @@ -152,12 +161,16 @@ function declaredDependencyNames(manifest) { * directory, because root pnpm builds previously recompiled plugins after shared * package API/type changes and root TypeScript/build-tooling changes. * + * FNXC:WorkspaceBuild 2026-07-15-03:20: + * Same contract now applies to non-plugin packages so unchanged core/engine/ + * dashboard/cli skip the multi-minute full rebuild. + * * @param {object} pkg * @param {Map} packagesByName * @returns {string[]} */ -export function collectPluginHashInputPaths(pkg, packagesByName) { - const inputPaths = new Set([...PLUGIN_BUILD_GLOBAL_INPUT_PATHS, pkg.dir]); +export function collectPackageHashInputPaths(pkg, packagesByName) { + const inputPaths = new Set([...PACKAGE_BUILD_GLOBAL_INPUT_PATHS, pkg.dir]); const seen = new Set(); const visit = (current) => { if (seen.has(current.name)) return; @@ -173,6 +186,11 @@ export function collectPluginHashInputPaths(pkg, packagesByName) { return [...inputPaths].sort((a, b) => a.localeCompare(b)); } +/** @deprecated Use collectPackageHashInputPaths */ +export function collectPluginHashInputPaths(pkg, packagesByName) { + return collectPackageHashInputPaths(pkg, packagesByName); +} + /** * Plugin workspaces live under plugins/ (including plugins/examples/). This * directory classification keeps future plugin packages covered by the skip @@ -243,16 +261,35 @@ function collectDistEntrypoints(manifest, outputPaths = new Set()) { */ export function requiredPluginOutputs(rootDir, dir, manifest) { const outputs = collectDistEntrypoints(manifest, collectDistExports(manifest.exports)); - const sourceFiles = fg.sync(["src/**/*.{ts,tsx,mts,cts}"], { - cwd: path.join(rootDir, dir), - onlyFiles: true, - unique: true, - ignore: ["**/*.d.ts", "**/*.test.*", "**/__tests__/**", "**/node_modules/**", "**/dist/**"], - }); - for (const sourceFile of sourceFiles) { - outputs.add(sourceFile.replace(/^src\//, "dist/").replace(/\.[cm]?[tj]sx?$/, ".js")); + const buildScript = typeof manifest.scripts?.build === "string" ? manifest.scripts.build : ""; + /* + FNXC:WorkspaceBuild 2026-07-15-03:50: + Bundlers (tsup/esbuild without tsc) emit entry bundles only — never per-source dist mirrors. + Mapping src/** → dist/** for @runfusion/fusion made every warm build report missing-output and + force a full CLI rebuild. Only tsc-style packages require per-file dist outputs. + */ + const isBundledPackage = + /\b(tsup|esbuild)\b/.test(buildScript) && !/\btsc\b/.test(buildScript); + + if (!isBundledPackage) { + const sourceFiles = fg.sync(["src/**/*.{ts,tsx,mts,cts}"], { + cwd: path.join(rootDir, dir), + onlyFiles: true, + unique: true, + ignore: [ + "**/*.d.ts", + "**/*.test.*", + "**/__tests__/**", + "**/__test-utils__/**", + "**/node_modules/**", + "**/dist/**", + ], + }); + for (const sourceFile of sourceFiles) { + outputs.add(sourceFile.replace(/^src\//, "dist/").replace(/\.[cm]?[tj]sx?$/, ".js")); + } } - if (typeof manifest.scripts?.build === "string" && manifest.scripts.build.includes("copy-css")) { + if (buildScript.includes("copy-css")) { const cssFiles = fg.sync(["src/**/*.css"], { cwd: path.join(rootDir, dir), onlyFiles: true, @@ -263,12 +300,27 @@ export function requiredPluginOutputs(rootDir, dir, manifest) { outputs.add(cssFile.replace(/^src\//, "dist/")); } } + /* + FNXC:WorkspaceBuild 2026-07-15-03:20: + Dashboard client is produced by Vite into dist/client (from app/), not by mapping package src to dist. + Require the client index so a warm tsc-only dist cannot skip a missing UI build. + */ + if (/\bvite\b/.test(buildScript)) { + outputs.add("dist/client/index.html"); + } + if (manifest.name === "@runfusion/fusion") { + outputs.add("dist/bin.js"); + outputs.add("dist/extension.js"); + } if (outputs.size === 0) outputs.add("dist/index.js"); return [...outputs].sort((a, b) => a.localeCompare(b)).map((output) => path.posix.join(dir, output)); } +/** Alias — required outputs apply to every workspace package, not only plugins. */ +export const requiredPackageOutputs = requiredPluginOutputs; + /** - * Compute a plugin package input hash using the shared git-backed content hash. + * Compute a package input hash using the shared git-backed content hash. * Returns null when git is unavailable; callers must build rather than skip in * that case. * @@ -279,20 +331,25 @@ export function requiredPluginOutputs(rootDir, dir, manifest) { * @param {ReturnType} [options.snapshot] * @returns {string|null} */ -export function computePluginSourceHash(pkg, rootDir, { gitFn = defaultGitRunner, snapshot } = {}) { +export function computePackageSourceHash(pkg, rootDir, { gitFn = defaultGitRunner, snapshot } = {}) { const probe = gitFn(["rev-parse", "--is-inside-work-tree"], rootDir); if (probe !== "true") return null; return computeContentHash({ rootDir, inputPaths: pkg.inputPaths?.length ? pkg.inputPaths : [pkg.dir], - versionPrefix: `plugin-build-v${BUILD_CACHE_VERSION}`, + versionPrefix: `package-build-v${BUILD_CACHE_VERSION}`, gitFn, snapshot, }); } +/** @deprecated Use computePackageSourceHash */ +export function computePluginSourceHash(pkg, rootDir, options) { + return computePackageSourceHash(pkg, rootDir, options); +} + /** - * Explain whether a plugin package must be built. A skip requires every required + * Explain whether a package must be built. A skip requires every required * output to exist plus a matching successful-build source hash. * * @param {object} pkg @@ -302,11 +359,13 @@ export function computePluginSourceHash(pkg, rootDir, { gitFn = defaultGitRunner * @param {(p: string) => boolean} [options.existsFn] * @param {(args: string[], cwd: string) => string|null} [options.gitFn] * @param {ReturnType} [options.snapshot] + * @param {boolean} [options.force] * @returns {{ shouldBuild: boolean, reason: string, sourceHash: string|null, missingOutputs: string[] }} */ -export function evaluatePluginBuild(pkg, { rootDir, cache, existsFn = existsSync, gitFn = defaultGitRunner, snapshot } = {}) { +export function evaluatePackageBuild(pkg, { rootDir, cache, existsFn = existsSync, gitFn = defaultGitRunner, snapshot, force = false } = {}) { const missingOutputs = pkg.requiredOutputs.filter((output) => !existsFn(path.join(rootDir, output))); - const sourceHash = computePluginSourceHash(pkg, rootDir, { gitFn, snapshot }); + const sourceHash = computePackageSourceHash(pkg, rootDir, { gitFn, snapshot }); + if (force) return { shouldBuild: true, reason: "force", sourceHash, missingOutputs }; if (missingOutputs.length > 0) return { shouldBuild: true, reason: "missing-output", sourceHash, missingOutputs }; if (sourceHash === null) return { shouldBuild: true, reason: "no-git-hash", sourceHash, missingOutputs }; const entry = cache?.entries?.[pkg.name]; @@ -315,10 +374,15 @@ export function evaluatePluginBuild(pkg, { rootDir, cache, existsFn = existsSync return { shouldBuild: false, reason: "unchanged", sourceHash, missingOutputs }; } +/** @deprecated Use evaluatePackageBuild */ +export function evaluatePluginBuild(pkg, options) { + return evaluatePackageBuild(pkg, options); +} + /** - * Plan the root build. Non-plugin build packages are always planned; plugin - * packages are planned only when the safe content-hash cache says they changed - * or their required outputs/cache entry are missing. + * Plan the root build. Every buildable workspace package (plugins and non-plugins) + * is planned only when the content-hash cache says inputs changed or required + * outputs/cache entries are missing. Desktop/mobile stay excluded. * * @param {object} options * @param {string} [options.rootDir] @@ -327,13 +391,14 @@ export function evaluatePluginBuild(pkg, { rootDir, cache, existsFn = existsSync * @param {(p: string) => boolean} [options.existsFn] * @param {(args: string[], cwd: string) => string|null} [options.gitFn] * @param {ReturnType} [options.snapshot] - * @returns {{ plannedPackages: object[], skippedPlugins: object[], excludedPackages: object[], pluginEvaluations: Map }} + * @param {boolean} [options.force] + * @returns {{ plannedPackages: object[], skippedPackages: object[], skippedPlugins: object[], excludedPackages: object[], packageEvaluations: Map, pluginEvaluations: Map }} */ -export function planWorkspaceBuild({ rootDir = repoRoot, packages = discoverWorkspacePackages(rootDir), cache = readPluginBuildCache(rootDir), existsFn = existsSync, gitFn = defaultGitRunner, snapshot } = {}) { +export function planWorkspaceBuild({ rootDir = repoRoot, packages = discoverWorkspacePackages(rootDir), cache = readPluginBuildCache(rootDir), existsFn = existsSync, gitFn = defaultGitRunner, snapshot, force = false } = {}) { const plannedPackages = []; - const skippedPlugins = []; + const skippedPackages = []; const excludedPackages = []; - const pluginEvaluations = new Map(); + const packageEvaluations = new Map(); for (const pkg of packages) { if (!pkg.hasBuild) continue; @@ -341,20 +406,25 @@ export function planWorkspaceBuild({ rootDir = repoRoot, packages = discoverWork excludedPackages.push(pkg); continue; } - if (!pkg.isPlugin) { - plannedPackages.push({ ...pkg, buildReason: "non-plugin" }); - continue; - } - const evaluation = evaluatePluginBuild(pkg, { rootDir, cache, existsFn, gitFn, snapshot }); - pluginEvaluations.set(pkg.name, evaluation); + const evaluation = evaluatePackageBuild(pkg, { rootDir, cache, existsFn, gitFn, snapshot, force }); + packageEvaluations.set(pkg.name, evaluation); if (evaluation.shouldBuild) { plannedPackages.push({ ...pkg, buildReason: evaluation.reason, sourceHash: evaluation.sourceHash }); } else { - skippedPlugins.push({ ...pkg, buildReason: evaluation.reason, sourceHash: evaluation.sourceHash }); + skippedPackages.push({ ...pkg, buildReason: evaluation.reason, sourceHash: evaluation.sourceHash }); } } - return { plannedPackages, skippedPlugins, excludedPackages, pluginEvaluations }; + // Back-compat: callers/tests that only inspect skippedPlugins keep working. + const skippedPlugins = skippedPackages.filter((pkg) => pkg.isPlugin); + return { + plannedPackages, + skippedPackages, + skippedPlugins, + excludedPackages, + packageEvaluations, + pluginEvaluations: packageEvaluations, + }; } /** @@ -366,7 +436,7 @@ export function planWorkspaceBuild({ rootDir = repoRoot, packages = discoverWork * @param {(command: string, args: string[], options: object) => { status: number|null }} [spawnFn] * @returns {{ status: number, packageNames: string[] }} */ -export function runPlannedBuilds(plannedPackages, rootDir, spawnFn = spawnSync) { +export function runPlannedBuilds(plannedPackages, rootDir, spawnFn = spawnSync, { fullPackage = false, env = process.env } = {}) { if (plannedPackages.length === 0) return { status: 0, packageNames: [] }; const packageNames = plannedPackages.map((pkg) => pkg.name); const args = [...packageNames.flatMap((name) => ["--filter", name]), "build"]; @@ -376,13 +446,27 @@ export function runPlannedBuilds(plannedPackages, rootDir, spawnFn = spawnSync) * shell (ENOENT / EINVAL since CVE-2024-27980). Without shell:true the root build failed * with `spawn pnpm ENOENT` on Windows. The args are workspace filters + package names * (no spaces or shell metacharacters), so shell quoting is safe. + * + * FNXC:WorkspaceBuild 2026-07-15-03:20: + * Propagate FUSION_CLI_FULL_PACKAGE so @runfusion/fusion tsup stages desktop + bundled + * plugins + DTS only when root build was invoked with --full (or CI already set the env). + * Day-to-day local builds skip that multi-minute packaging tail. */ - const result = spawnFn("pnpm", args, { cwd: rootDir, stdio: "inherit", shell: process.platform === "win32" }); + const childEnv = { + ...env, + ...(fullPackage ? { FUSION_CLI_FULL_PACKAGE: "1" } : {}), + }; + const result = spawnFn("pnpm", args, { + cwd: rootDir, + stdio: "inherit", + shell: process.platform === "win32", + env: childEnv, + }); return { status: result.status ?? 1, packageNames }; } /** - * Record hashes for plugins that built successfully. + * Record hashes for packages that built successfully (plugins and non-plugins). * * @param {object[]} builtPackages * @param {object} options @@ -390,12 +474,12 @@ export function runPlannedBuilds(plannedPackages, rootDir, spawnFn = spawnSync) * @param {ReturnType} options.cache * @param {(args: string[], cwd: string) => string|null} [options.gitFn] */ -export function recordSuccessfulPluginBuilds(builtPackages, { rootDir, cache, gitFn = defaultGitRunner } = {}) { +export function recordSuccessfulPackageBuilds(builtPackages, { rootDir, cache, gitFn = defaultGitRunner } = {}) { const nextCache = { version: BUILD_CACHE_VERSION, entries: { ...(cache?.entries ?? {}) } }; let changed = false; const snapshot = createRepoContentSnapshot({ rootDir, gitFn }); - for (const pkg of builtPackages.filter((entry) => entry.isPlugin)) { - const sourceHash = computePluginSourceHash(pkg, rootDir, { gitFn, snapshot }); + for (const pkg of builtPackages) { + const sourceHash = computePackageSourceHash(pkg, rootDir, { gitFn, snapshot }); if (sourceHash === null) continue; nextCache.entries[pkg.name] = { sourceHash, builtAt: new Date().toISOString() }; changed = true; @@ -403,6 +487,11 @@ export function recordSuccessfulPluginBuilds(builtPackages, { rootDir, cache, gi if (changed) writePluginBuildCache(rootDir, nextCache); } +/** @deprecated Use recordSuccessfulPackageBuilds */ +export function recordSuccessfulPluginBuilds(builtPackages, options) { + return recordSuccessfulPackageBuilds(builtPackages, options); +} + function formatPlanLine(pkg) { return `${pkg.name} (${pkg.buildReason})`; } @@ -415,28 +504,106 @@ function formatPlanLine(pkg) { * rebuilding every non-plugin workspace package. Plugins load their built * dist/ at runtime, so a never-rebuilt plugin dist silently runs phantom-old * code — exactly the Grok "messages aren't sending" wrong-CLI-flags failure. + * + * FNXC:WorkspaceBuild 2026-07-15-03:20: + * `--force` rebuilds every package ignoring the skip cache. `--full` sets + * FUSION_CLI_FULL_PACKAGE for the CLI packaging path (desktop + plugins + DTS). */ -export function main({ rootDir = repoRoot, spawnFn = spawnSync, gitFn = defaultGitRunner, pluginsOnly = false } = {}) { - const cache = readPluginBuildCache(rootDir); +/** + * FNXC:WorkspaceBuild 2026-07-15-03:25 / 2026-07-15-09:05: + * Mirror packages/cli wantsFullCliPackage so build-workspace and tsup agree on when + * full CLI packaging runs. CLI enables full via FUSION_CLI_FULL_PACKAGE, CI=true, or prepack; + * root also enables via --full. Explicit FUSION_CLI_FULL_PACKAGE=0/false opts out. + * + * @param {NodeJS.ProcessEnv} [env] + * @param {{ fullFlag?: boolean }} [options] + * @returns {boolean} + */ +export function wantsFullCliPackage(env = process.env, { fullFlag = false } = {}) { + const explicit = env.FUSION_CLI_FULL_PACKAGE; + if (explicit === "0" || explicit === "false") return false; + if (explicit === "1" || explicit === "true") return true; + if (fullFlag) return true; + if (env.CI === "true" || env.CI === "1") return true; + if (env.npm_lifecycle_event === "prepack") return true; + return false; +} + +/** + * FNXC:WorkspaceBuild 2026-07-15-08:15: + * Greptile P1: a warm fast build caches CLI after emitting only bin.js/extension.js. + * Full packaging modes must still run tsup so desktop/plugins/DTS stage. Force-include + * @runfusion/fusion whenever full packaging is active, even if content-hash says skip. + * + * FNXC:WorkspaceBuild 2026-07-15-09:05: + * fullPackage must include env-driven modes (CI / FUSION_CLI_FULL_PACKAGE), not only --full. + */ +export function ensureFullPackageCliPlanned(plannedPackages, skippedPackages, { fullPackage = false } = {}) { + if (!fullPackage) { + return { plannedPackages, skippedPackages }; + } + const cliName = "@runfusion/fusion"; + if (plannedPackages.some((pkg) => pkg.name === cliName)) { + return { plannedPackages, skippedPackages }; + } + const skippedCli = (skippedPackages ?? []).find((pkg) => pkg.name === cliName); + if (!skippedCli) { + return { plannedPackages, skippedPackages }; + } + return { + plannedPackages: [ + ...plannedPackages, + { ...skippedCli, buildReason: "full-package", sourceHash: skippedCli.sourceHash }, + ], + skippedPackages: (skippedPackages ?? []).filter((pkg) => pkg.name !== cliName), + }; +} + +export function main({ + rootDir = repoRoot, + spawnFn = spawnSync, + gitFn = defaultGitRunner, + pluginsOnly = false, + force = false, + fullPackage = false, + env = process.env, +} = {}) { + /* + FNXC:WorkspaceBuild 2026-07-15-09:05: + Align with CLI tsup wantsFullCliPackage: --full OR CI OR FUSION_CLI_FULL_PACKAGE (unless explicitly 0). + */ + const effectiveFullPackage = wantsFullCliPackage(env, { fullFlag: fullPackage }); + const cache = force ? { version: BUILD_CACHE_VERSION, entries: {} } : readPluginBuildCache(rootDir); const snapshot = createRepoContentSnapshot({ rootDir, gitFn }); - const plan = planWorkspaceBuild({ rootDir, cache, gitFn, snapshot }); - const plannedPackages = pluginsOnly ? plan.plannedPackages.filter((pkg) => pkg.isPlugin) : plan.plannedPackages; + const plan = planWorkspaceBuild({ rootDir, cache, gitFn, snapshot, force }); + let plannedPackages = pluginsOnly ? plan.plannedPackages.filter((pkg) => pkg.isPlugin) : plan.plannedPackages; + let skippedPackages = plan.skippedPackages ?? plan.skippedPlugins; + if (!pluginsOnly) { + ({ plannedPackages, skippedPackages } = ensureFullPackageCliPlanned(plannedPackages, skippedPackages, { + fullPackage: effectiveFullPackage, + })); + } const plannedNames = plannedPackages.map(formatPlanLine); - const skippedNames = plan.skippedPlugins.map((pkg) => pkg.name); + const skippedNames = skippedPackages.map((pkg) => pkg.name); const scope = pluginsOnly ? "changed plugins" : "planned builds"; console.log(`[build-workspace] ${scope}: ${plannedNames.join(", ") || "(none)"}`); if (skippedNames.length > 0) { - console.log(`[build-workspace] skipped unchanged plugins: ${skippedNames.join(", ")}`); + console.log(`[build-workspace] skipped unchanged packages: ${skippedNames.join(", ")}`); + } + if (effectiveFullPackage) { + console.log("[build-workspace] full CLI packaging enabled (CI / FUSION_CLI_FULL_PACKAGE / --full)"); } - const result = runPlannedBuilds(plannedPackages, rootDir, spawnFn); + const result = runPlannedBuilds(plannedPackages, rootDir, spawnFn, { fullPackage: effectiveFullPackage, env }); if (result.status !== 0) { process.stderr.write(`[build-workspace] FAILED packages: ${result.packageNames.join(", ") || "(none)"}\n`); return result.status; } - recordSuccessfulPluginBuilds(plannedPackages, { rootDir, cache, gitFn }); + // When force used empty cache for planning, still merge into on-disk cache. + const persistCache = force ? readPluginBuildCache(rootDir) : cache; + recordSuccessfulPackageBuilds(plannedPackages, { rootDir, cache: persistCache, gitFn }); return 0; } @@ -450,6 +617,9 @@ export function main({ rootDir = repoRoot, spawnFn = spawnSync, gitFn = defaultG * file URL of argv[1] so the guard is correct on Windows, macOS, and Linux. */ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - const pluginsOnly = process.argv.slice(2).includes("--plugins-only"); - process.exit(main({ pluginsOnly })); + const args = process.argv.slice(2); + const pluginsOnly = args.includes("--plugins-only"); + const force = args.includes("--force"); + const fullPackage = args.includes("--full"); + process.exit(main({ pluginsOnly, force, fullPackage })); } diff --git a/scripts/spotlight-exclude-fusion-paths.sh b/scripts/spotlight-exclude-fusion-paths.sh new file mode 100755 index 0000000000..b0cbd686a0 --- /dev/null +++ b/scripts/spotlight-exclude-fusion-paths.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Re-apply Spotlight skip markers for Fusion-heavy directories. +# Uses .metadata_never_index (Spotlight does not index dirs containing this file). +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +MARKER=".metadata_never_index" +PATHS=( + "$ROOT/.worktrees" + "$ROOT/.worktrees/.ai-merge" + "$ROOT/node_modules" + "$ROOT/.fusion" + "$ROOT/packages/desktop/deploy" + "$ROOT/packages/desktop/dist" + "$HOME/.fusion" + "$HOME/.fusion/embedded-postgres" + "$HOME/orca/workspaces" + "$HOME/.herdr/worktrees/kb" + "$HOME/.paseo/worktrees" +) +for p in "${PATHS[@]}"; do + [ -d "$p" ] || continue + f="$p/$MARKER" + [ -f "$f" ] || touch "$f" + chflags hidden "$f" 2>/dev/null || true +done +if [ -d "$ROOT/.worktrees" ]; then + for wt in "$ROOT/.worktrees"/*/; do + [ -d "$wt" ] || continue + f="${wt}${MARKER}" + [ -f "$f" ] || touch "$f" + chflags hidden "$f" 2>/dev/null || true + if [ -d "${wt}node_modules" ]; then + nf="${wt}node_modules/${MARKER}" + [ -f "$nf" ] || touch "$nf" + chflags hidden "$nf" 2>/dev/null || true + fi + done +fi +echo "Spotlight skip markers applied under $ROOT and related agent worktree roots." diff --git a/tsconfig.base.json b/tsconfig.base.json index 93e423a0c3..134af9a3b1 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -6,6 +6,15 @@ "declaration": true, "declarationMap": true, "sourceMap": true, + /* + FNXC:WorkspaceBuild 2026-07-15-03:30: + Full program tsc of core/engine (~700-1000 files) dominated root builds. Enable incremental so warm rebuilds only recheck changed files. + + FNXC:WorkspaceBuild 2026-07-15-08:40: + Use ${configDir} for tsBuildInfoFile so each package that extends this base writes its own dist/.tsbuildinfo. A plain relative path would resolve against this base file (repo root) and collide across packages. + */ + "incremental": true, + "tsBuildInfoFile": "${configDir}/dist/.tsbuildinfo", "strict": true, "esModuleInterop": true, "skipLibCheck": true,