fix: stop agents exceeding the global concurrency cap (#2107)
## Summary - Operators could see more agents running than Global Max Concurrent (e.g. 5 running with cap 4: 4 planners + 1 executor). - Scheduler now `tryAcquire`s a shared semaphore slot before todo→in-progress and hands that pre-held slot to the executor/graph run. - Triage admits planners against the live top-level running-agent claim (planning + in-progress + active in-review), not only `semaphore.availableCount`. - Executor claims the pre-held slot for the full run and avoids a second top-level acquire on step/seam re-entry (deadlock under a full cap). ## Test plan - [x] `pnpm --filter @fusion/engine exec vitest run src/__tests__/concurrency.test.ts src/__tests__/triage.test.ts` - [x] Regression: triage leaves room when 1 in-progress agent is live under global cap 4 - [x] Regression: pre-held executor slot register/take/drop handoff - [ ] Manual: set Global Max Concurrent and Max triage concurrent to 4, fill Planning + run 1 In Progress; footer should not show 5 running under a full steady state <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **Bug Fixes** - Improved global concurrency enforcement so the scheduler and executor never start more agents than the configured limit, including tighter top-level “claimed capacity” accounting. - Updated triage admission control to consider global top-level utilization, factoring processing tasks and agents already running to prevent over-admitting planners. - Added safer pre-held concurrency-slot handoff behavior to avoid capacity leaks and drift during graph routing, step execution, and legacy fallback. - Ensured reserved capacity is reliably released on early exits, failed dispatches, and other aborted paths (with idempotent cleanup). - Refreshed concurrency diagnostics to better explain whether throttling is due to project or global limits, with clearer claimed/processing visibility. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
7
.changeset/fix-global-concurrency-over-cap.md
Normal file
7
.changeset/fix-global-concurrency-over-cap.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
summary: Stop more agents from running than the global concurrency cap allows.
|
||||||
|
category: fix
|
||||||
|
dev: Scheduler tryAcquires a shared slot before todo→in-progress and hands it to the executor/graph; triage admits planners against live running-agent claim (planning + in-progress + active in-review), not only semaphore.availableCount.
|
||||||
@@ -6,8 +6,14 @@ import {
|
|||||||
PRIORITY_MERGE,
|
PRIORITY_MERGE,
|
||||||
PRIORITY_EXECUTE,
|
PRIORITY_EXECUTE,
|
||||||
PRIORITY_SPECIFY,
|
PRIORITY_SPECIFY,
|
||||||
|
clearPreHeldExecutorSlotsForTests,
|
||||||
|
computeTopLevelConcurrencyClaimed,
|
||||||
|
dropPreHeldExecutorSlot,
|
||||||
|
hasPreHeldExecutorSlot,
|
||||||
persistedTopLevelAgentSlots,
|
persistedTopLevelAgentSlots,
|
||||||
recoverIdleSemaphoreLeakCandidate,
|
recoverIdleSemaphoreLeakCandidate,
|
||||||
|
registerPreHeldExecutorSlot,
|
||||||
|
takePreHeldExecutorSlot,
|
||||||
} from "../concurrency.js";
|
} from "../concurrency.js";
|
||||||
|
|
||||||
describe("ScopedAgentSemaphore", () => {
|
describe("ScopedAgentSemaphore", () => {
|
||||||
@@ -405,6 +411,109 @@ describe("AgentSemaphore", () => {
|
|||||||
expect(persistedTopLevelAgentSlots(tasks)).toBe(7);
|
expect(persistedTopLevelAgentSlots(tasks)).toBe(7);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("claims top-level concurrency as max(live running agents, semaphore active, pending specify)", () => {
|
||||||
|
const tasks = [
|
||||||
|
{ column: "in-progress" },
|
||||||
|
{ column: "triage", status: "planning", paused: false },
|
||||||
|
{ column: "triage", status: "planning", paused: false },
|
||||||
|
{ column: "triage", status: "planning", paused: false },
|
||||||
|
{ column: "triage", status: "planning", paused: false },
|
||||||
|
{ column: "todo" },
|
||||||
|
] as Task[];
|
||||||
|
|
||||||
|
// 4 planning + 1 in-progress = 5 live holders (the reported over-cap symptom).
|
||||||
|
expect(computeTopLevelConcurrencyClaimed({ tasks })).toBe(5);
|
||||||
|
// Prefer the larger of live holders and in-memory activeCount.
|
||||||
|
expect(computeTopLevelConcurrencyClaimed({ tasks, semaphoreActiveCount: 2 })).toBe(5);
|
||||||
|
expect(computeTopLevelConcurrencyClaimed({ tasks: [], semaphoreActiveCount: 3, pendingSpecifyCount: 2 })).toBe(3);
|
||||||
|
expect(computeTopLevelConcurrencyClaimed({ tasks: [], semaphoreActiveCount: 1, pendingSpecifyCount: 2 })).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("hands off pre-held executor slots without double-registering", () => {
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
const sem = new AgentSemaphore(2);
|
||||||
|
expect(sem.tryAcquire()).toBe(true);
|
||||||
|
registerPreHeldExecutorSlot("FN-1");
|
||||||
|
expect(hasPreHeldExecutorSlot("FN-1")).toBe(true);
|
||||||
|
|
||||||
|
expect(takePreHeldExecutorSlot("FN-1")).toBe(true);
|
||||||
|
expect(hasPreHeldExecutorSlot("FN-1")).toBe(false);
|
||||||
|
expect(takePreHeldExecutorSlot("FN-1")).toBe(false);
|
||||||
|
|
||||||
|
// Failed dispatch path releases both the registry entry and the semaphore slot.
|
||||||
|
expect(sem.tryAcquire()).toBe(true);
|
||||||
|
registerPreHeldExecutorSlot("FN-2");
|
||||||
|
dropPreHeldExecutorSlot("FN-2", sem);
|
||||||
|
expect(hasPreHeldExecutorSlot("FN-2")).toBe(false);
|
||||||
|
expect(sem.activeCount).toBe(1);
|
||||||
|
sem.release();
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
});
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-15-02:55:
|
||||||
|
Graph fallback re-registers a scheduler pre-held slot for the legacy execute path. That registration must always be either take()n (legacy agent work under runWithExecutorSemaphore) or drop()ped on early exits. A re-register without a subsequent take/drop is the permanent global-capacity leak Greptile P1 (PR #2107) caught: activeCount stays inflated while the task is no longer holding work.
|
||||||
|
*/
|
||||||
|
it("treats graph→legacy re-register as a leak unless take or drop follows", () => {
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
const sem = new AgentSemaphore(1);
|
||||||
|
expect(sem.tryAcquire()).toBe(true);
|
||||||
|
// Scheduler reserved the slot before todo→in-progress.
|
||||||
|
registerPreHeldExecutorSlot("FN-LEGACY-HANDOFF");
|
||||||
|
|
||||||
|
// Graph claims then re-registers for legacy (transferPreHeldToLegacy).
|
||||||
|
expect(takePreHeldExecutorSlot("FN-LEGACY-HANDOFF")).toBe(true);
|
||||||
|
registerPreHeldExecutorSlot("FN-LEGACY-HANDOFF");
|
||||||
|
expect(hasPreHeldExecutorSlot("FN-LEGACY-HANDOFF")).toBe(true);
|
||||||
|
expect(sem.activeCount).toBe(1);
|
||||||
|
|
||||||
|
// Authoritative / work-engine / heartbeat-defer early returns must drop, not leave the registration.
|
||||||
|
dropPreHeldExecutorSlot("FN-LEGACY-HANDOFF", sem);
|
||||||
|
expect(hasPreHeldExecutorSlot("FN-LEGACY-HANDOFF")).toBe(false);
|
||||||
|
expect(sem.activeCount).toBe(0);
|
||||||
|
|
||||||
|
// Happy path: re-register then take + release (runWithExecutorSemaphore contract).
|
||||||
|
expect(sem.tryAcquire()).toBe(true);
|
||||||
|
registerPreHeldExecutorSlot("FN-LEGACY-TAKE");
|
||||||
|
expect(takePreHeldExecutorSlot("FN-LEGACY-TAKE")).toBe(true);
|
||||||
|
expect(hasPreHeldExecutorSlot("FN-LEGACY-TAKE")).toBe(false);
|
||||||
|
sem.release();
|
||||||
|
expect(sem.activeCount).toBe(0);
|
||||||
|
// Second drop after take is a no-op — safe for execute()'s outer finally belt-and-suspenders.
|
||||||
|
dropPreHeldExecutorSlot("FN-LEGACY-TAKE", sem);
|
||||||
|
expect(sem.activeCount).toBe(0);
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
});
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-15-03:50:
|
||||||
|
Scheduler hold/release: tryAcquire + register before the column move; if the move fails the
|
||||||
|
prep release() lambda must dropPreHeldExecutorSlot so activeCount returns to zero. Without
|
||||||
|
that cleanup a failed dispatch permanently shrinks global capacity.
|
||||||
|
*/
|
||||||
|
it("releases pre-held semaphore when scheduler dispatch prep release() runs (move failure)", () => {
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
const sem = new AgentSemaphore(2);
|
||||||
|
expect(sem.tryAcquire()).toBe(true);
|
||||||
|
registerPreHeldExecutorSlot("FN-MOVE-FAIL");
|
||||||
|
expect(hasPreHeldExecutorSlot("FN-MOVE-FAIL")).toBe(true);
|
||||||
|
expect(sem.activeCount).toBe(1);
|
||||||
|
|
||||||
|
// Mirrors scheduler.ts prep.release() after a failed/aborted hold release.
|
||||||
|
let released = false;
|
||||||
|
const release = () => {
|
||||||
|
if (released) return;
|
||||||
|
released = true;
|
||||||
|
dropPreHeldExecutorSlot("FN-MOVE-FAIL", sem);
|
||||||
|
};
|
||||||
|
release();
|
||||||
|
release(); // idempotent
|
||||||
|
|
||||||
|
expect(hasPreHeldExecutorSlot("FN-MOVE-FAIL")).toBe(false);
|
||||||
|
expect(sem.activeCount).toBe(0);
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
});
|
||||||
|
|
||||||
it("recovers idle semaphore leaks only after a stable persisted-idle window", async () => {
|
it("recovers idle semaphore leaks only after a stable persisted-idle window", async () => {
|
||||||
const sem = new AgentSemaphore(2);
|
const sem = new AgentSemaphore(2);
|
||||||
await sem.acquire();
|
await sem.acquire();
|
||||||
|
|||||||
@@ -0,0 +1,201 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { TaskDetail } from "@fusion/core";
|
||||||
|
import "./executor-test-helpers.js";
|
||||||
|
import {
|
||||||
|
AgentSemaphore,
|
||||||
|
clearPreHeldExecutorSlotsForTests,
|
||||||
|
hasPreHeldExecutorSlot,
|
||||||
|
registerPreHeldExecutorSlot,
|
||||||
|
} from "../concurrency.js";
|
||||||
|
import { TaskExecutor } from "../executor.js";
|
||||||
|
import { executingTaskLock } from "../active-session-registry.js";
|
||||||
|
import { createMockStore, resetExecutorMocks } from "./executor-test-helpers.js";
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-15-02:55:
|
||||||
|
Regression for Greptile P1 on PR #2107 (legacy handoff leaks reserved slot). When
|
||||||
|
maybeExecuteWorkflowGraph falls back it re-registers any scheduler pre-held slot for
|
||||||
|
the legacy execute path. execute() must drop that registration on every early return
|
||||||
|
that never reaches runWithExecutorSemaphore.take — authoritative dispatch accept,
|
||||||
|
workflow work-engine claim, and heartbeat defer — or the shared semaphore permanently
|
||||||
|
shrinks global capacity. Surface enumeration covers all three named leak paths.
|
||||||
|
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-15-03:10:
|
||||||
|
Also cover authoritative dispatch rejection: a thrown callback exits execute() before the
|
||||||
|
accept-path drop and before the main try/finally, so the re-registered slot would leak
|
||||||
|
unless drop runs in the catch-before-rethrow path.
|
||||||
|
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-15-03:50:
|
||||||
|
execute() now wraps executeCore in try/finally that always dropPreHeldExecutorSlot, so
|
||||||
|
even paths that omit an explicit drop still free capacity on exit.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const now = "2026-07-15T00:00:00.000Z";
|
||||||
|
|
||||||
|
function task(overrides: Partial<TaskDetail> = {}): TaskDetail {
|
||||||
|
return {
|
||||||
|
id: "FN-PREHELD-HANDOFF",
|
||||||
|
title: "Pre-held legacy handoff",
|
||||||
|
description: "Graph fallback re-registers a pre-held concurrency slot",
|
||||||
|
column: "in-progress",
|
||||||
|
dependencies: [],
|
||||||
|
// No enabledWorkflowSteps: minimal mock store lacks workflow-selection API and
|
||||||
|
// falls back to legacy without fail-closing (transferPreHeldToLegacy=true).
|
||||||
|
steps: [{ name: "Implement", status: "pending" }],
|
||||||
|
currentStep: 0,
|
||||||
|
log: [],
|
||||||
|
branch: "fusion/fn-preheld-handoff",
|
||||||
|
baseBranch: "main",
|
||||||
|
worktree: "/tmp/fusion-fn-preheld-handoff",
|
||||||
|
status: null,
|
||||||
|
error: null,
|
||||||
|
paused: false,
|
||||||
|
userPaused: false,
|
||||||
|
autoMerge: true,
|
||||||
|
mergeRetries: 0,
|
||||||
|
createdAt: now,
|
||||||
|
updatedAt: now,
|
||||||
|
...overrides,
|
||||||
|
} as TaskDetail;
|
||||||
|
}
|
||||||
|
|
||||||
|
function settings(overrides: Record<string, unknown> = {}) {
|
||||||
|
return {
|
||||||
|
autoMerge: true,
|
||||||
|
maxAutoMergeRetries: 3,
|
||||||
|
maxConcurrent: 2,
|
||||||
|
maxWorktrees: 4,
|
||||||
|
pollIntervalMs: 15000,
|
||||||
|
ephemeralAgentsEnabled: true,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Simulate the hold/release sweep: tryAcquire + register before execute. */
|
||||||
|
function reservePreHeld(taskId: string, sem: AgentSemaphore): void {
|
||||||
|
expect(sem.tryAcquire()).toBe(true);
|
||||||
|
registerPreHeldExecutorSlot(taskId);
|
||||||
|
expect(hasPreHeldExecutorSlot(taskId)).toBe(true);
|
||||||
|
expect(sem.activeCount).toBe(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
executingTaskLock._clearForTest();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("executor pre-held legacy handoff (graph fallback)", () => {
|
||||||
|
it("drops the re-registered slot when authoritative dispatch owns the task", async () => {
|
||||||
|
resetExecutorMocks();
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
executingTaskLock._clearForTest();
|
||||||
|
|
||||||
|
const sem = new AgentSemaphore(2);
|
||||||
|
const live = task();
|
||||||
|
const store = createMockStore();
|
||||||
|
store.getTask.mockResolvedValue(live);
|
||||||
|
store.getSettings.mockResolvedValue(settings());
|
||||||
|
// Ensure graph takes the minimal-store fallback path (no selection API).
|
||||||
|
delete (store as { getTaskWorkflowSelection?: unknown }).getTaskWorkflowSelection;
|
||||||
|
delete (store as { getTaskWorkflowSelectionAsync?: unknown }).getTaskWorkflowSelectionAsync;
|
||||||
|
|
||||||
|
reservePreHeld(live.id, sem);
|
||||||
|
|
||||||
|
const workflowAuthoritativeDispatch = vi.fn().mockResolvedValue(true);
|
||||||
|
const executor = new TaskExecutor(store, "/tmp/test", {
|
||||||
|
semaphore: sem,
|
||||||
|
workflowAuthoritativeDispatch,
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
await executor.execute(live);
|
||||||
|
|
||||||
|
expect(workflowAuthoritativeDispatch).toHaveBeenCalledTimes(1);
|
||||||
|
expect(hasPreHeldExecutorSlot(live.id)).toBe(false);
|
||||||
|
expect(sem.activeCount).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops the re-registered slot when authoritative dispatch rejects", async () => {
|
||||||
|
resetExecutorMocks();
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
executingTaskLock._clearForTest();
|
||||||
|
|
||||||
|
const sem = new AgentSemaphore(2);
|
||||||
|
const live = task();
|
||||||
|
const store = createMockStore();
|
||||||
|
store.getTask.mockResolvedValue(live);
|
||||||
|
store.getSettings.mockResolvedValue(settings());
|
||||||
|
delete (store as { getTaskWorkflowSelection?: unknown }).getTaskWorkflowSelection;
|
||||||
|
delete (store as { getTaskWorkflowSelectionAsync?: unknown }).getTaskWorkflowSelectionAsync;
|
||||||
|
|
||||||
|
reservePreHeld(live.id, sem);
|
||||||
|
|
||||||
|
const dispatchError = new Error("authoritative dispatch failed");
|
||||||
|
const workflowAuthoritativeDispatch = vi.fn().mockRejectedValue(dispatchError);
|
||||||
|
const executor = new TaskExecutor(store, "/tmp/test", {
|
||||||
|
semaphore: sem,
|
||||||
|
workflowAuthoritativeDispatch,
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
await expect(executor.execute(live)).rejects.toThrow("authoritative dispatch failed");
|
||||||
|
|
||||||
|
expect(workflowAuthoritativeDispatch).toHaveBeenCalledTimes(1);
|
||||||
|
expect(hasPreHeldExecutorSlot(live.id)).toBe(false);
|
||||||
|
expect(sem.activeCount).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops the re-registered slot when the workflow work engine claims execution", async () => {
|
||||||
|
resetExecutorMocks();
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
executingTaskLock._clearForTest();
|
||||||
|
|
||||||
|
const sem = new AgentSemaphore(2);
|
||||||
|
const live = task();
|
||||||
|
const store = createMockStore();
|
||||||
|
store.getTask.mockResolvedValue(live);
|
||||||
|
store.getSettings.mockResolvedValue(settings());
|
||||||
|
delete (store as { getTaskWorkflowSelection?: unknown }).getTaskWorkflowSelection;
|
||||||
|
delete (store as { getTaskWorkflowSelectionAsync?: unknown }).getTaskWorkflowSelectionAsync;
|
||||||
|
|
||||||
|
reservePreHeld(live.id, sem);
|
||||||
|
|
||||||
|
const executor = new TaskExecutor(store, "/tmp/test", {
|
||||||
|
semaphore: sem,
|
||||||
|
workflowAuthoritativeDispatch: vi.fn().mockResolvedValue(false),
|
||||||
|
} as any);
|
||||||
|
vi.spyOn(executor as any, "maybeDispatchWorkflowWorkEngine").mockResolvedValue(true);
|
||||||
|
|
||||||
|
await executor.execute(live);
|
||||||
|
|
||||||
|
expect(hasPreHeldExecutorSlot(live.id)).toBe(false);
|
||||||
|
expect(sem.activeCount).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops the re-registered slot when heartbeat deferral skips legacy execution", async () => {
|
||||||
|
resetExecutorMocks();
|
||||||
|
clearPreHeldExecutorSlotsForTests();
|
||||||
|
executingTaskLock._clearForTest();
|
||||||
|
|
||||||
|
const sem = new AgentSemaphore(2);
|
||||||
|
const live = task({ assignedAgentId: "agent-serial" });
|
||||||
|
const store = createMockStore();
|
||||||
|
store.getTask.mockResolvedValue(live);
|
||||||
|
store.getSettings.mockResolvedValue(settings());
|
||||||
|
delete (store as { getTaskWorkflowSelection?: unknown }).getTaskWorkflowSelection;
|
||||||
|
delete (store as { getTaskWorkflowSelectionAsync?: unknown }).getTaskWorkflowSelectionAsync;
|
||||||
|
|
||||||
|
reservePreHeld(live.id, sem);
|
||||||
|
|
||||||
|
const executor = new TaskExecutor(store, "/tmp/test", {
|
||||||
|
semaphore: sem,
|
||||||
|
workflowAuthoritativeDispatch: vi.fn().mockResolvedValue(false),
|
||||||
|
} as any);
|
||||||
|
vi.spyOn(executor as any, "maybeDispatchWorkflowWorkEngine").mockResolvedValue(false);
|
||||||
|
vi.spyOn(executor as any, "resolveEffectivePrincipalId").mockReturnValue("agent-serial");
|
||||||
|
vi.spyOn(executor as any, "shouldDeferForHeartbeat").mockResolvedValue(true);
|
||||||
|
|
||||||
|
await executor.execute(live);
|
||||||
|
|
||||||
|
expect(hasPreHeldExecutorSlot(live.id)).toBe(false);
|
||||||
|
expect(sem.activeCount).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -2259,6 +2259,51 @@ describe("TriageProcessor", () => {
|
|||||||
expect(specifySpy).toHaveBeenCalledWith(expect.objectContaining({ id: "FN-200" }));
|
expect(specifySpy).toHaveBeenCalledWith(expect.objectContaining({ id: "FN-200" }));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-14-18:30:
|
||||||
|
When an in-progress executor already counts toward the live running-agent total, triage must leave room under the global cap instead of filling maxTriageConcurrent purely from semaphore.availableCount.
|
||||||
|
*/
|
||||||
|
it("leaves global concurrency room for live in-progress agents when admitting planners", async () => {
|
||||||
|
const tasks: Task[] = [
|
||||||
|
createTriageTask({ id: "FN-300", priority: "urgent" }),
|
||||||
|
createTriageTask({ id: "FN-301", priority: "urgent" }),
|
||||||
|
createTriageTask({ id: "FN-302", priority: "urgent" }),
|
||||||
|
createTriageTask({ id: "FN-303", priority: "urgent" }),
|
||||||
|
{
|
||||||
|
...createTriageTask({ id: "FN-EXEC", priority: "normal" }),
|
||||||
|
column: "in-progress",
|
||||||
|
status: null,
|
||||||
|
} as Task,
|
||||||
|
];
|
||||||
|
|
||||||
|
const triageStore = createMockStore({
|
||||||
|
listTasks: vi.fn().mockResolvedValue(tasks),
|
||||||
|
getSettings: vi.fn().mockResolvedValue({
|
||||||
|
maxConcurrent: 4,
|
||||||
|
maxTriageConcurrent: 4,
|
||||||
|
pollIntervalMs: 10_000,
|
||||||
|
groupOverlappingFiles: false,
|
||||||
|
autoMerge: true,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const semaphore = {
|
||||||
|
availableCount: 4,
|
||||||
|
activeCount: 0,
|
||||||
|
limit: 4,
|
||||||
|
snapshot: vi.fn(() => ({ activeCount: 0, waitingCount: 0, availableCount: 4, limit: 4 })),
|
||||||
|
};
|
||||||
|
const triageProcessor = new TriageProcessor(triageStore, rootDir, { semaphore: semaphore as any });
|
||||||
|
const specifySpy = vi
|
||||||
|
.spyOn(triageProcessor, "specifyTask")
|
||||||
|
.mockResolvedValue(undefined);
|
||||||
|
|
||||||
|
(triageProcessor as any).running = true;
|
||||||
|
await (triageProcessor as any).poll();
|
||||||
|
|
||||||
|
// Global cap 4 with 1 in-progress holder → at most 3 new planners.
|
||||||
|
expect(specifySpy).toHaveBeenCalledTimes(3);
|
||||||
|
});
|
||||||
|
|
||||||
/*
|
/*
|
||||||
FNXC:PlanReview 2026-06-29-15:42:
|
FNXC:PlanReview 2026-06-29-15:42:
|
||||||
Polling must honor Plan Review retry backoff as a dispatch boundary: future `nextRecoveryAt` rows stay parked, elapsed reviewer-outage rows bypass the planner, and ordinary null/needs-replan rows still launch planning.
|
Polling must honor Plan Review retry backoff as a dispatch boundary: future `nextRecoveryAt` rows stay parked, elapsed reviewer-outage rows bypass the planner, and ordinary null/needs-replan rows still launch planning.
|
||||||
|
|||||||
@@ -18,6 +18,47 @@ interface PriorityWaiter {
|
|||||||
|
|
||||||
export const IDLE_SEMAPHORE_LEAK_REPAIR_MS = 5_000;
|
export const IDLE_SEMAPHORE_LEAK_REPAIR_MS = 5_000;
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-14-18:30:
|
||||||
|
Operators reported live running-agent counts above the global concurrency cap (e.g. 5 running with cap 4). Live utilization counts every top-level slot holder (in-progress, planning triage, active in-review), but the scheduler only preflighted capacity and acquired the shared semaphore later inside the executor — so a card could sit in-progress (and count as running) while triage still saw free semaphore slots and filled the rest of the cap. Pre-held executor slots close that gap: tryAcquire before todo→in-progress, keep the slot until the executor/graph run claims and releases it, and admit triage against max(semaphore.activeCount, live running count).
|
||||||
|
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-15-03:50:
|
||||||
|
Hard invariant: registerPreHeldExecutorSlot may only run immediately after a successful semaphore.tryAcquire() for that same task, and every registration must later be either take()d (caller releases the semaphore) or drop()d (releases the semaphore). The Set is process-local soft state decoupled from activeCount except via this discipline — acquire-without-register or register-without-acquire desyncs capacity accounting.
|
||||||
|
*/
|
||||||
|
const preHeldExecutorSlots = new Set<string>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Register a semaphore slot that was **just** acquired via `tryAcquire` for a task about to enter in-progress.
|
||||||
|
* Must not be called without a matching prior acquire; pair with take() or drop().
|
||||||
|
*/
|
||||||
|
export function registerPreHeldExecutorSlot(taskId: string): void {
|
||||||
|
preHeldExecutorSlots.add(taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transfer ownership of a pre-held executor slot to the caller.
|
||||||
|
* Returns true when a slot was registered; the caller MUST release the underlying semaphore in its finally path.
|
||||||
|
*/
|
||||||
|
export function takePreHeldExecutorSlot(taskId: string): boolean {
|
||||||
|
return preHeldExecutorSlots.delete(taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drop a pre-held slot without transferring ownership (failed reserve / cancelled dispatch). Optionally releases the semaphore. */
|
||||||
|
export function dropPreHeldExecutorSlot(taskId: string, semaphore?: { release(): void }): void {
|
||||||
|
if (!preHeldExecutorSlots.delete(taskId)) return;
|
||||||
|
semaphore?.release();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test/helper: whether a task currently has an unclaimed pre-held executor slot. */
|
||||||
|
export function hasPreHeldExecutorSlot(taskId: string): boolean {
|
||||||
|
return preHeldExecutorSlots.has(taskId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test helper: clear all pre-held registrations without releasing semaphore slots. */
|
||||||
|
export function clearPreHeldExecutorSlotsForTests(): void {
|
||||||
|
preHeldExecutorSlots.clear();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* FNXC:GlobalConcurrencyControls 2026-06-27-00:00:
|
* FNXC:GlobalConcurrencyControls 2026-06-27-00:00:
|
||||||
* Persisted semaphore repair must use the same top-level slot predicate as dashboard and CLI live counts, including active in-review agents, so read-layer utilization and engine recovery cannot drift.
|
* Persisted semaphore repair must use the same top-level slot predicate as dashboard and CLI live counts, including active in-review agents, so read-layer utilization and engine recovery cannot drift.
|
||||||
@@ -26,6 +67,22 @@ export function persistedTopLevelAgentSlots(tasks: Task[]): number {
|
|||||||
return countRunningAgentTasks(tasks);
|
return countRunningAgentTasks(tasks);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* FNXC:GlobalConcurrencyControls 2026-07-14-18:30:
|
||||||
|
* Admission control for new top-level agents must use the same running-agent predicate the dashboard shows next to the global/project caps. Prefer the larger of live task-based holders and in-memory semaphore activeCount so neither under-counts the other during the brief window between column/status writes and acquire/release.
|
||||||
|
*/
|
||||||
|
export function computeTopLevelConcurrencyClaimed(params: {
|
||||||
|
tasks: readonly Task[];
|
||||||
|
semaphoreActiveCount?: number;
|
||||||
|
/** specifyTask calls that have entered `processing` but not yet written status:"planning". */
|
||||||
|
pendingSpecifyCount?: number;
|
||||||
|
}): number {
|
||||||
|
const persisted = countRunningAgentTasks(params.tasks);
|
||||||
|
const pending = Math.max(0, Math.floor(params.pendingSpecifyCount ?? 0));
|
||||||
|
const active = Math.max(0, Math.floor(params.semaphoreActiveCount ?? 0));
|
||||||
|
return Math.max(active, persisted + pending);
|
||||||
|
}
|
||||||
|
|
||||||
export interface IdleSemaphoreLeakRecoveryResult {
|
export interface IdleSemaphoreLeakRecoveryResult {
|
||||||
candidateSinceMs: number | null;
|
candidateSinceMs: number | null;
|
||||||
reconciliation?: { before: number; after: number; changed: boolean };
|
reconciliation?: { before: number; after: number; changed: boolean };
|
||||||
|
|||||||
@@ -101,7 +101,13 @@ import { buildUserCommentsPromptSection, selectUserCommentsForAgentContext } fro
|
|||||||
import { resolveSandboxBackend } from "./sandbox/index.js";
|
import { resolveSandboxBackend } from "./sandbox/index.js";
|
||||||
import type { SandboxBackend } from "./sandbox/types.js";
|
import type { SandboxBackend } from "./sandbox/types.js";
|
||||||
import { ModelRegistry, SessionManager, type ToolDefinition, type AgentSession } from "@earendil-works/pi-coding-agent";
|
import { ModelRegistry, SessionManager, type ToolDefinition, type AgentSession } from "@earendil-works/pi-coding-agent";
|
||||||
import { PRIORITY_EXECUTE, type AgentSemaphore } from "./concurrency.js";
|
import {
|
||||||
|
PRIORITY_EXECUTE,
|
||||||
|
dropPreHeldExecutorSlot,
|
||||||
|
registerPreHeldExecutorSlot,
|
||||||
|
takePreHeldExecutorSlot,
|
||||||
|
type AgentSemaphore,
|
||||||
|
} from "./concurrency.js";
|
||||||
// FNXC:Workspace 2026-06-21-15:00: F5/F8 — wire in the previously dead workspace-path helpers.
|
// FNXC:Workspace 2026-06-21-15:00: F5/F8 — wire in the previously dead workspace-path helpers.
|
||||||
// `normalizeRepoRelPath` is the single shared scope-path normalizer (F8); `deriveRepoScopeSubset`
|
// `normalizeRepoRelPath` is the single shared scope-path normalizer (F8); `deriveRepoScopeSubset`
|
||||||
// maps the task's repo-prefixed declared File Scope to a repo-LOCAL subset so the per-repo scope-leak
|
// maps the task's repo-prefixed declared File Scope to a repo-LOCAL subset so the per-repo scope-leak
|
||||||
@@ -2998,6 +3004,42 @@ export class TaskExecutor {
|
|||||||
return assertMcpResolutionSucceeded(resolved);
|
return assertMcpResolutionSucceeded(resolved);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tasks whose graph run already owns a top-level concurrency slot (scheduler pre-held handoff).
|
||||||
|
* Seam re-entry under that graph must not acquire a second slot.
|
||||||
|
*/
|
||||||
|
private outerConcurrencyClaims = new Set<string>();
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-14-18:30:
|
||||||
|
Prefer a scheduler pre-held global slot when present so the hold/release tryAcquire and the executor share one top-level claim. Without this handoff the executor would acquire a second slot (or leave a gap if the pre-held slot were dropped) and live running counts could drift above the global cap again. While this outer claim is active, seam/step sessions must not acquire again — a second top-level acquire under a full global cap deadlocks (parent holds the last slot, child waits forever).
|
||||||
|
*/
|
||||||
|
private async runWithExecutorSemaphore<T>(taskId: string, work: () => Promise<T>): Promise<T> {
|
||||||
|
const sem = this.options.semaphore;
|
||||||
|
if (!sem) return work();
|
||||||
|
if (this.outerConcurrencyClaims.has(taskId)) {
|
||||||
|
return work();
|
||||||
|
}
|
||||||
|
|
||||||
|
const runUnderOuterClaim = async (): Promise<T> => {
|
||||||
|
this.outerConcurrencyClaims.add(taskId);
|
||||||
|
try {
|
||||||
|
return await work();
|
||||||
|
} finally {
|
||||||
|
this.outerConcurrencyClaims.delete(taskId);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (takePreHeldExecutorSlot(taskId)) {
|
||||||
|
try {
|
||||||
|
return await runUnderOuterClaim();
|
||||||
|
} finally {
|
||||||
|
sem.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sem.run(runUnderOuterClaim, PRIORITY_EXECUTE);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* FNXC:PlannerOversight 2026-07-13-23:05:
|
* FNXC:PlannerOversight 2026-07-13-23:05:
|
||||||
* Wire session-advisor live log flush after ProjectEngine starts (options are
|
* Wire session-advisor live log flush after ProjectEngine starts (options are
|
||||||
@@ -5134,6 +5176,16 @@ export class TaskExecutor {
|
|||||||
// the same task cannot both enter graph routing (mirrors executingTaskLock).
|
// the same task cannot both enter graph routing (mirrors executingTaskLock).
|
||||||
this.graphRouting.add(task.id);
|
this.graphRouting.add(task.id);
|
||||||
let graphAbortController: AbortController | undefined;
|
let graphAbortController: AbortController | undefined;
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-14-18:30:
|
||||||
|
The hold/release sweep may have already tryAcquired a global slot for this card before moving it to in-progress. Claim that pre-held slot for the full graph run so utilization stays honest between workflow nodes and triage cannot overfill the cap while this task is still graph-owned.
|
||||||
|
*/
|
||||||
|
const hadPreHeldExecutorSlot = takePreHeldExecutorSlot(task.id);
|
||||||
|
if (hadPreHeldExecutorSlot) {
|
||||||
|
this.outerConcurrencyClaims.add(task.id);
|
||||||
|
}
|
||||||
|
/** When true, re-register the pre-held slot for the legacy execute path instead of releasing it. */
|
||||||
|
let transferPreHeldToLegacy = false;
|
||||||
try {
|
try {
|
||||||
let settings: Settings;
|
let settings: Settings;
|
||||||
try {
|
try {
|
||||||
@@ -5186,6 +5238,7 @@ export class TaskExecutor {
|
|||||||
});
|
});
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
transferPreHeldToLegacy = true;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
@@ -5432,6 +5485,18 @@ export class TaskExecutor {
|
|||||||
} catch (err) {
|
} catch (err) {
|
||||||
executorLog.warn(`terminateAllChildren failed for graph task ${task.id}: ${err instanceof Error ? err.message : String(err)}`);
|
executorLog.warn(`terminateAllChildren failed for graph task ${task.id}: ${err instanceof Error ? err.message : String(err)}`);
|
||||||
}
|
}
|
||||||
|
if (hadPreHeldExecutorSlot) {
|
||||||
|
this.outerConcurrencyClaims.delete(task.id);
|
||||||
|
if (transferPreHeldToLegacy) {
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-15-02:55:
|
||||||
|
Graph declined ownership and is handing the reserved global slot to the legacy execute path. Re-register only; do not release here. execute() must take this registration via runWithExecutorSemaphore or dropPreHeldExecutorSlot on every early return (authoritative dispatch accept, work-engine claim, heartbeat defer, lock contention, soft-delete, etc.). Leaving the registration live after execute returns permanently reduces global capacity.
|
||||||
|
*/
|
||||||
|
registerPreHeldExecutorSlot(task.id);
|
||||||
|
} else {
|
||||||
|
this.options.semaphore?.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
if (graphAbortController && this.activeWorkflowGraphAbortControllers.get(task.id) === graphAbortController) {
|
if (graphAbortController && this.activeWorkflowGraphAbortControllers.get(task.id) === graphAbortController) {
|
||||||
this.activeWorkflowGraphAbortControllers.delete(task.id);
|
this.activeWorkflowGraphAbortControllers.delete(task.id);
|
||||||
}
|
}
|
||||||
@@ -9744,7 +9809,23 @@ export class TaskExecutor {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-15-03:50:
|
||||||
|
Structural cleanup for scheduler pre-held global slots: every execute() exit path
|
||||||
|
(early return, throw, graph-owned, legacy handoff) must leave no unclaimed registration.
|
||||||
|
take() removes the registration so a successful claim+release is a no-op here; early
|
||||||
|
returns that never take() release the underlying semaphore. New early-return paths
|
||||||
|
cannot reintroduce permanent capacity leaks without bypassing this wrapper.
|
||||||
|
*/
|
||||||
async execute(task: Task): Promise<void> {
|
async execute(task: Task): Promise<void> {
|
||||||
|
try {
|
||||||
|
await this.executeCore(task);
|
||||||
|
} finally {
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async executeCore(task: Task): Promise<void> {
|
||||||
this.completionFinalizedTaskIds.delete(task.id);
|
this.completionFinalizedTaskIds.delete(task.id);
|
||||||
await this.clearStalePauseAbortBeforeDispatch(task);
|
await this.clearStalePauseAbortBeforeDispatch(task);
|
||||||
// Workflow graph interpreter routing (cutover M-C): graph-selected tasks
|
// Workflow graph interpreter routing (cutover M-C): graph-selected tasks
|
||||||
@@ -9758,17 +9839,40 @@ export class TaskExecutor {
|
|||||||
executorLog.log(`execute() called for ${task.id} while graph routing is active — skipping duplicate`);
|
executorLog.log(`execute() called for ${task.id} while graph routing is active — skipping duplicate`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (await this.blockOuterDispatchWhenDependenciesUnmet(task)) return;
|
if (await this.blockOuterDispatchWhenDependenciesUnmet(task)) {
|
||||||
|
// FNXC:GlobalConcurrencyControls 2026-07-14-18:30: release any scheduler pre-held slot when outer dispatch aborts before agent work starts.
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
|
return;
|
||||||
|
}
|
||||||
// FNXC:EphemeralAgents 2026-07-01-00:00: gate ALL workflow dispatch paths
|
// FNXC:EphemeralAgents 2026-07-01-00:00: gate ALL workflow dispatch paths
|
||||||
// (graph/authoritative/work-engine) on ephemeralAgentsEnabled before any of
|
// (graph/authoritative/work-engine) on ephemeralAgentsEnabled before any of
|
||||||
// them can claim the task. Placed inside the outer-dispatch block so seam
|
// them can claim the task. Placed inside the outer-dispatch block so seam
|
||||||
// re-entry (interceptor registered) is unaffected, and ahead of every path
|
// re-entry (interceptor registered) is unaffected, and ahead of every path
|
||||||
// so the single check covers all three entry points.
|
// so the single check covers all three entry points.
|
||||||
if (await this.blockOuterDispatchWhenEphemeralDisabled(task)) return;
|
if (await this.blockOuterDispatchWhenEphemeralDisabled(task)) {
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
|
return;
|
||||||
|
}
|
||||||
const graphOwned = await this.maybeExecuteWorkflowGraph(task);
|
const graphOwned = await this.maybeExecuteWorkflowGraph(task);
|
||||||
if (graphOwned) return;
|
if (graphOwned) return;
|
||||||
const authoritativeOwned = await this.options.workflowAuthoritativeDispatch?.(task);
|
/*
|
||||||
if (authoritativeOwned) return;
|
FNXC:GlobalConcurrencyControls 2026-07-15-02:55:
|
||||||
|
After graph falls back it may have re-registered a scheduler pre-held slot for legacy execute. Any return that does not reach runWithExecutorSemaphore (which take()s the registration) must dropPreHeldExecutorSlot or the shared semaphore stays permanently inflated.
|
||||||
|
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-15-03:10:
|
||||||
|
workflowAuthoritativeDispatch can reject as well as return true. Rejection propagates out of execute() before the explicit drop below and before the main executor try/finally, so the re-registered pre-held registration and underlying semaphore claim would otherwise stay active forever. Drop before rethrowing.
|
||||||
|
*/
|
||||||
|
let authoritativeOwned: boolean | undefined;
|
||||||
|
try {
|
||||||
|
authoritativeOwned = await this.options.workflowAuthoritativeDispatch?.(task);
|
||||||
|
} catch (err) {
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
if (authoritativeOwned) {
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// FN-4811 follow-up (FN-4814/FN-4809/FN-4811 production failure): claim a
|
// FN-4811 follow-up (FN-4814/FN-4809/FN-4811 production failure): claim a
|
||||||
@@ -9783,7 +9887,11 @@ export class TaskExecutor {
|
|||||||
// active-session-registry.ts, a module-level Set.
|
// active-session-registry.ts, a module-level Set.
|
||||||
const claimed = executingTaskLock.tryClaim(task.id);
|
const claimed = executingTaskLock.tryClaim(task.id);
|
||||||
executorLog.log(`execute() called for ${task.id} (claimed=${claimed}, perInstanceExecuting=${this.executing.has(task.id)})`);
|
executorLog.log(`execute() called for ${task.id} (claimed=${claimed}, perInstanceExecuting=${this.executing.has(task.id)})`);
|
||||||
if (!claimed) return;
|
if (!claimed) {
|
||||||
|
// FNXC:GlobalConcurrencyControls 2026-07-15-02:55: graph fallback may have re-registered a pre-held slot; drop it when this process cannot claim the executor lock.
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Maintain the per-instance Set too, for back-compat with all the existing
|
// Maintain the per-instance Set too, for back-compat with all the existing
|
||||||
// `this.executing.has()` checks throughout the file (handler gates,
|
// `this.executing.has()` checks throughout the file (handler gates,
|
||||||
@@ -9795,6 +9903,7 @@ export class TaskExecutor {
|
|||||||
executorLog.warn(`${task.id}: refusing execute — task is soft-deleted`);
|
executorLog.warn(`${task.id}: refusing execute — task is soft-deleted`);
|
||||||
this.executing.delete(task.id);
|
this.executing.delete(task.id);
|
||||||
executingTaskLock.release(task.id);
|
executingTaskLock.release(task.id);
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -9802,6 +9911,8 @@ export class TaskExecutor {
|
|||||||
executorLog.log(`${task.id}: workflow work engine claimed execution`);
|
executorLog.log(`${task.id}: workflow work engine claimed execution`);
|
||||||
this.executing.delete(task.id);
|
this.executing.delete(task.id);
|
||||||
executingTaskLock.release(task.id);
|
executingTaskLock.release(task.id);
|
||||||
|
// FNXC:GlobalConcurrencyControls 2026-07-15-02:55: work-engine ownership never take()s the legacy handoff registration — release the reserved global slot.
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -9819,6 +9930,8 @@ export class TaskExecutor {
|
|||||||
// Release the slot we just claimed — we never actually ran.
|
// Release the slot we just claimed — we never actually ran.
|
||||||
this.executing.delete(task.id);
|
this.executing.delete(task.id);
|
||||||
executingTaskLock.release(task.id);
|
executingTaskLock.release(task.id);
|
||||||
|
// FNXC:GlobalConcurrencyControls 2026-07-15-02:55: heartbeat defer must free any re-registered pre-held global slot so capacity is not stranded until the next dispatch.
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -9883,6 +9996,8 @@ export class TaskExecutor {
|
|||||||
await moveTaskToReplanColumn(this.store, task);
|
await moveTaskToReplanColumn(this.store, task);
|
||||||
await this.store.updateTask(task.id, { status: "needs-replan" });
|
await this.store.updateTask(task.id, { status: "needs-replan" });
|
||||||
await this.store.logEntry(task.id, staleness.reason, undefined, this.getRunContextFor(task.id));
|
await this.store.logEntry(task.id, staleness.reason, undefined, this.getRunContextFor(task.id));
|
||||||
|
// FNXC:GlobalConcurrencyControls 2026-07-15-02:55: replan handoff never starts agent work — free any re-registered pre-held slot before leaving execute().
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -9899,6 +10014,7 @@ export class TaskExecutor {
|
|||||||
if (await this.finalizeMergeConfirmedWorkflowGraphTask(task.id, "execute-preflight")) {
|
if (await this.finalizeMergeConfirmedWorkflowGraphTask(task.id, "execute-preflight")) {
|
||||||
this.executing.delete(task.id);
|
this.executing.delete(task.id);
|
||||||
executingTaskLock.release(task.id);
|
executingTaskLock.release(task.id);
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -10343,7 +10459,8 @@ export class TaskExecutor {
|
|||||||
worktreePath,
|
worktreePath,
|
||||||
rootDir: this.rootDir,
|
rootDir: this.rootDir,
|
||||||
settings,
|
settings,
|
||||||
semaphore: this.options.semaphore,
|
// FNXC:GlobalConcurrencyControls 2026-07-14-18:30: When the graph run already owns a top-level slot (outerConcurrencyClaims), do not pass the semaphore into per-step sessions — each step would acquire a second slot and can deadlock under a full global cap.
|
||||||
|
semaphore: this.outerConcurrencyClaims.has(task.id) ? undefined : this.options.semaphore,
|
||||||
stuckTaskDetector: this.options.stuckTaskDetector,
|
stuckTaskDetector: this.options.stuckTaskDetector,
|
||||||
pluginRunner: this.options.pluginRunner,
|
pluginRunner: this.options.pluginRunner,
|
||||||
runtimeHint: stepSessionRuntimeHint,
|
runtimeHint: stepSessionRuntimeHint,
|
||||||
@@ -10688,11 +10805,7 @@ export class TaskExecutor {
|
|||||||
});
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (this.options.semaphore) {
|
await this.runWithExecutorSemaphore(task.id, retryableStepWork);
|
||||||
await this.options.semaphore.run(retryableStepWork, PRIORITY_EXECUTE);
|
|
||||||
} else {
|
|
||||||
await retryableStepWork();
|
|
||||||
}
|
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err);
|
const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err);
|
||||||
if (this.depAborted.has(task.id)) {
|
if (this.depAborted.has(task.id)) {
|
||||||
@@ -11899,11 +12012,7 @@ export class TaskExecutor {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (this.options.semaphore) {
|
await this.runWithExecutorSemaphore(task.id, retryableWork);
|
||||||
await this.options.semaphore.run(retryableWork, PRIORITY_EXECUTE);
|
|
||||||
} else {
|
|
||||||
await retryableWork();
|
|
||||||
}
|
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err);
|
const { message: errorMessage, detail: errorDetail, stack: errorStack } = formatError(err);
|
||||||
if (this.depAborted.has(task.id)) {
|
if (this.depAborted.has(task.id)) {
|
||||||
@@ -12618,6 +12727,14 @@ export class TaskExecutor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-15-02:55:
|
||||||
|
Belt-and-suspenders for graph→legacy pre-held handoff inside the lock-claimed try:
|
||||||
|
release any still-registered slot before lock/executing cleanup. execute()'s outer
|
||||||
|
finally also drops (no-op once take/drop already cleared the registration).
|
||||||
|
*/
|
||||||
|
dropPreHeldExecutorSlot(task.id, this.options.semaphore);
|
||||||
|
|
||||||
this.executing.delete(task.id);
|
this.executing.delete(task.id);
|
||||||
executingTaskLock.release(task.id);
|
executingTaskLock.release(task.id);
|
||||||
// Clear run context at end of execute() lifecycle
|
// Clear run context at end of execute() lifecycle
|
||||||
|
|||||||
@@ -19,7 +19,13 @@ import {
|
|||||||
import { existsSync } from "node:fs";
|
import { existsSync } from "node:fs";
|
||||||
import { readFile } from "node:fs/promises";
|
import { readFile } from "node:fs/promises";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { recoverIdleSemaphoreLeakCandidate, type AgentSemaphore } from "./concurrency.js";
|
import {
|
||||||
|
computeTopLevelConcurrencyClaimed,
|
||||||
|
dropPreHeldExecutorSlot,
|
||||||
|
recoverIdleSemaphoreLeakCandidate,
|
||||||
|
registerPreHeldExecutorSlot,
|
||||||
|
type AgentSemaphore,
|
||||||
|
} from "./concurrency.js";
|
||||||
import { planTaskWorktreePath, resolveTaskWorkingBranch } from "./worktree-names.js";
|
import { planTaskWorktreePath, resolveTaskWorkingBranch } from "./worktree-names.js";
|
||||||
import { schedulerLog } from "./logger.js";
|
import { schedulerLog } from "./logger.js";
|
||||||
import { type PrMonitor, type PrComment } from "./pr-monitor.js";
|
import { type PrMonitor, type PrComment } from "./pr-monitor.js";
|
||||||
@@ -394,6 +400,12 @@ function computeConcurrencyGateDiagnostic(params: {
|
|||||||
semaphore?: AgentSemaphore;
|
semaphore?: AgentSemaphore;
|
||||||
inProgressTaskIds: string[];
|
inProgressTaskIds: string[];
|
||||||
startedThisTick?: number;
|
startedThisTick?: number;
|
||||||
|
/**
|
||||||
|
* Live top-level running-agent claim (planning + in-progress + active in-review,
|
||||||
|
* optionally merged with semaphore.activeCount). When provided, the shared
|
||||||
|
* semaphore gate uses this instead of only in-progress agentSlots.
|
||||||
|
*/
|
||||||
|
topLevelClaimedSlots?: number;
|
||||||
/** U6: additive per-column capacity gates (flag-ON only). Omitted → the legacy
|
/** U6: additive per-column capacity gates (flag-ON only). Omitted → the legacy
|
||||||
* three-gate report is byte-identical. */
|
* three-gate report is byte-identical. */
|
||||||
perColumnGates?: PerColumnCapacityGate[];
|
perColumnGates?: PerColumnCapacityGate[];
|
||||||
@@ -413,7 +425,13 @@ function computeConcurrencyGateDiagnostic(params: {
|
|||||||
};
|
};
|
||||||
const semaphoreGate = params.semaphore
|
const semaphoreGate = params.semaphore
|
||||||
? (() => {
|
? (() => {
|
||||||
const used = Math.max(0, params.semaphore.activeCount, params.agentSlots) + startedThisTick;
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-14-18:30:
|
||||||
|
Global semaphore pressure must include every live top-level agent holder (planning triage and active in-review), not only in-progress WIP, otherwise the hold/release sweep can admit an executor on top of a full planner fleet and the footer shows running > cap.
|
||||||
|
*/
|
||||||
|
const claimed = params.topLevelClaimedSlots
|
||||||
|
?? Math.max(0, params.semaphore.activeCount, params.agentSlots);
|
||||||
|
const used = Math.max(0, claimed) + startedThisTick;
|
||||||
return {
|
return {
|
||||||
used,
|
used,
|
||||||
limit: params.semaphore.limit,
|
limit: params.semaphore.limit,
|
||||||
@@ -1398,6 +1416,10 @@ export class Scheduler {
|
|||||||
maxWorktrees,
|
maxWorktrees,
|
||||||
semaphore: this.options.semaphore,
|
semaphore: this.options.semaphore,
|
||||||
inProgressTaskIds,
|
inProgressTaskIds,
|
||||||
|
topLevelClaimedSlots: computeTopLevelConcurrencyClaimed({
|
||||||
|
tasks,
|
||||||
|
semaphoreActiveCount: this.options.semaphore?.activeCount,
|
||||||
|
}),
|
||||||
startedThisTick: started,
|
startedThisTick: started,
|
||||||
perColumnGates,
|
perColumnGates,
|
||||||
});
|
});
|
||||||
@@ -2690,6 +2712,11 @@ export class Scheduler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const topLevelClaimedSlots = computeTopLevelConcurrencyClaimed({
|
||||||
|
tasks,
|
||||||
|
// Prior tryAcquire reservations in this sweep already bump activeCount.
|
||||||
|
semaphoreActiveCount: this.options.semaphore?.activeCount,
|
||||||
|
});
|
||||||
const concurrencyDiagnostic = computeConcurrencyGateDiagnostic({
|
const concurrencyDiagnostic = computeConcurrencyGateDiagnostic({
|
||||||
agentSlots: reservedConcurrentSlots,
|
agentSlots: reservedConcurrentSlots,
|
||||||
maxConcurrent,
|
maxConcurrent,
|
||||||
@@ -2697,10 +2724,14 @@ export class Scheduler {
|
|||||||
maxWorktrees,
|
maxWorktrees,
|
||||||
semaphore: this.options.semaphore,
|
semaphore: this.options.semaphore,
|
||||||
inProgressTaskIds,
|
inProgressTaskIds,
|
||||||
|
topLevelClaimedSlots,
|
||||||
});
|
});
|
||||||
/*
|
/*
|
||||||
FNXC:WorkflowScheduling 2026-06-23-20:58:
|
FNXC:WorkflowScheduling 2026-06-23-20:58:
|
||||||
The workflow hold/release sweep is the only todo pickup path, so it must honor the same maxConcurrent, maxWorktrees, and shared semaphore pressure before releasing a task to in-progress. This is deliberately a non-mutating preflight: executor owns the actual semaphore acquire, and the scheduler only prevents capacity-obvious over-release without double-acquiring slots.
|
The workflow hold/release sweep is the only todo pickup path, so it must honor the same maxConcurrent, maxWorktrees, and shared semaphore pressure before releasing a task to in-progress.
|
||||||
|
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-14-18:30:
|
||||||
|
Preflight is no longer non-mutating for the shared semaphore: tryAcquire reserves a real slot before the move so triage cannot fill the global cap while this card is already counted as an in-progress runner. On move failure the reservation is released; on success the pre-held slot is transferred to the executor/graph run.
|
||||||
*/
|
*/
|
||||||
if (concurrencyDiagnostic.available <= 0) {
|
if (concurrencyDiagnostic.available <= 0) {
|
||||||
if (reservedScope) {
|
if (reservedScope) {
|
||||||
@@ -2713,6 +2744,25 @@ export class Scheduler {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const sem = this.options.semaphore;
|
||||||
|
if (sem && !sem.tryAcquire()) {
|
||||||
|
if (reservedScope) {
|
||||||
|
activeScopes.delete(task.id);
|
||||||
|
activeScopeColumns.delete(task.id);
|
||||||
|
}
|
||||||
|
const reason = formatConcurrencyLimitReason({
|
||||||
|
...concurrencyDiagnostic,
|
||||||
|
available: 0,
|
||||||
|
bindingGates: [...new Set([...concurrencyDiagnostic.bindingGates, "semaphore" as const])],
|
||||||
|
});
|
||||||
|
await this.store.updateTask(task.id, { status: "queued" });
|
||||||
|
await this.logDispatchQueuedReason(task.id, reason, formatConcurrencyLimitMemoKey(concurrencyDiagnostic));
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (sem) {
|
||||||
|
registerPreHeldExecutorSlot(task.id);
|
||||||
|
}
|
||||||
|
|
||||||
dispatchPrepByTaskId.set(task.id, {
|
dispatchPrepByTaskId.set(task.id, {
|
||||||
baseBranch: this.resolveBaseBranch(freshTask, tasks),
|
baseBranch: this.resolveBaseBranch(freshTask, tasks),
|
||||||
dispatchStormCount: nextDispatchStormCount,
|
dispatchStormCount: nextDispatchStormCount,
|
||||||
@@ -2736,6 +2786,7 @@ export class Scheduler {
|
|||||||
reservedWorktreeSlots = Math.max(0, reservedWorktreeSlots - 1);
|
reservedWorktreeSlots = Math.max(0, reservedWorktreeSlots - 1);
|
||||||
reservedConcurrentSlots = Math.max(0, reservedConcurrentSlots - 1);
|
reservedConcurrentSlots = Math.max(0, reservedConcurrentSlots - 1);
|
||||||
dispatchPrepByTaskId.delete(task.id);
|
dispatchPrepByTaskId.delete(task.id);
|
||||||
|
dropPreHeldExecutorSlot(task.id, sem);
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -112,7 +112,12 @@ import {
|
|||||||
formatExternalIntegrationEvidenceDiagnostic,
|
formatExternalIntegrationEvidenceDiagnostic,
|
||||||
} from "./spec-validation/external-integration-evidence.js";
|
} from "./spec-validation/external-integration-evidence.js";
|
||||||
import { buildSessionSkillContext } from "./session-skill-context.js";
|
import { buildSessionSkillContext } from "./session-skill-context.js";
|
||||||
import { PRIORITY_SPECIFY, recoverIdleSemaphoreLeakCandidate, type AgentSemaphore } from "./concurrency.js";
|
import {
|
||||||
|
PRIORITY_SPECIFY,
|
||||||
|
computeTopLevelConcurrencyClaimed,
|
||||||
|
recoverIdleSemaphoreLeakCandidate,
|
||||||
|
type AgentSemaphore,
|
||||||
|
} from "./concurrency.js";
|
||||||
import { AgentLogger } from "./agent-logger.js";
|
import { AgentLogger } from "./agent-logger.js";
|
||||||
import {
|
import {
|
||||||
resolveAgentInstructions,
|
resolveAgentInstructions,
|
||||||
@@ -866,6 +871,10 @@ export class TriageProcessor {
|
|||||||
|
|
||||||
// Respect both per-project maxTriageConcurrent and the global semaphore.
|
// Respect both per-project maxTriageConcurrent and the global semaphore.
|
||||||
// Only planning tasks count against the triage limit; execution is governed by maxConcurrent.
|
// Only planning tasks count against the triage limit; execution is governed by maxConcurrent.
|
||||||
|
/*
|
||||||
|
FNXC:GlobalConcurrencyControls 2026-07-14-18:30:
|
||||||
|
Live utilization counts in-progress executors and active planners toward the same global cap. Cap new triage starts by remaining room under that shared claim (not only semaphore.availableCount), so planning cannot fill the entire global max while an in-progress executor is already counted as running.
|
||||||
|
*/
|
||||||
const maxTriageConcurrent = settings.maxTriageConcurrent ?? settings.maxConcurrent ?? 2;
|
const maxTriageConcurrent = settings.maxTriageConcurrent ?? settings.maxConcurrent ?? 2;
|
||||||
const planning = allTasks.filter(
|
const planning = allTasks.filter(
|
||||||
(t) => (t.column === "triage" || t.column === "todo") && t.status === "planning" && !t.paused,
|
(t) => (t.column === "triage" || t.column === "todo") && t.status === "planning" && !t.paused,
|
||||||
@@ -876,7 +885,21 @@ export class TriageProcessor {
|
|||||||
const semaphoreAvailable = this.options.semaphore
|
const semaphoreAvailable = this.options.semaphore
|
||||||
? Math.max(0, this.options.semaphore.availableCount)
|
? Math.max(0, this.options.semaphore.availableCount)
|
||||||
: Infinity;
|
: Infinity;
|
||||||
const maxToStart = Math.min(perProjectAvailable, semaphoreAvailable);
|
// processing entries that have not yet written status:"planning" still claim a future slot.
|
||||||
|
let pendingSpecifyCount = 0;
|
||||||
|
for (const id of this.processing) {
|
||||||
|
const row = allTasks.find((t) => t.id === id);
|
||||||
|
if (!row || row.status !== "planning") pendingSpecifyCount += 1;
|
||||||
|
}
|
||||||
|
const claimed = computeTopLevelConcurrencyClaimed({
|
||||||
|
tasks: allTasks,
|
||||||
|
semaphoreActiveCount: this.options.semaphore?.activeCount,
|
||||||
|
pendingSpecifyCount,
|
||||||
|
});
|
||||||
|
const globalRoom = this.options.semaphore
|
||||||
|
? Math.max(0, this.options.semaphore.limit - claimed)
|
||||||
|
: Infinity;
|
||||||
|
const maxToStart = Math.min(perProjectAvailable, semaphoreAvailable, globalRoom);
|
||||||
|
|
||||||
if (maxToStart <= 0 && triageTasks.length > 0) {
|
if (maxToStart <= 0 && triageTasks.length > 0) {
|
||||||
const semaphoreSnapshot = this.options.semaphore?.snapshot();
|
const semaphoreSnapshot = this.options.semaphore?.snapshot();
|
||||||
@@ -885,10 +908,14 @@ export class TriageProcessor {
|
|||||||
: ", semaphore unavailable";
|
: ", semaphore unavailable";
|
||||||
const processingIds = [...this.processing].slice(0, 5);
|
const processingIds = [...this.processing].slice(0, 5);
|
||||||
const eligibleIds = triageTasks.slice(0, 5).map((t) => t.id);
|
const eligibleIds = triageTasks.slice(0, 5).map((t) => t.id);
|
||||||
const blockedBy = perProjectAvailable <= 0 ? "triage concurrency" : "global semaphore";
|
const blockedBy = perProjectAvailable <= 0
|
||||||
|
? "triage concurrency"
|
||||||
|
: globalRoom <= 0
|
||||||
|
? "global running-agent cap"
|
||||||
|
: "global semaphore";
|
||||||
planLog.log(
|
planLog.log(
|
||||||
`Plan throttled by ${blockedBy}: eligible=${triageTasks.length} [${eligibleIds.join(", ")}], ` +
|
`Plan throttled by ${blockedBy}: eligible=${triageTasks.length} [${eligibleIds.join(", ")}], ` +
|
||||||
`planning=${activeAgents}/${maxTriageConcurrent}, processing=${this.processing.size}` +
|
`planning=${activeAgents}/${maxTriageConcurrent}, claimed=${claimed}, processing=${this.processing.size}` +
|
||||||
`${processingIds.length > 0 ? ` [${processingIds.join(", ")}]` : ""}${semaphoreDetail}`,
|
`${processingIds.length > 0 ? ` [${processingIds.join(", ")}]` : ""}${semaphoreDetail}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user