feat(engine): in-txn capacity enforcement + generalized hold/release sweep with reservation-first ordering (U6)

This commit is contained in:
gsxdsm
2026-06-04 01:22:56 -07:00
parent 6491441102
commit 6a15a13917
10 changed files with 1390 additions and 15 deletions

View File

@@ -12,7 +12,7 @@
// - crash-mid-transition marker recovery (SQLite authoritative)
// - unknown-column rejection
// - guard rejection typed (flag-ON) vs legacy string (flag-OFF)
// - bypassGuards capacity pass-through (documenting; U6 fills enforcement)
// - in-txn capacity enforcement (U6; NEVER bypassable — KTD-10)
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { VALID_TRANSITIONS } from "../types.js";
@@ -199,18 +199,69 @@ describe("transition-parity — store flag-ON scenarios", () => {
expect(after?.worktree).toBe("/tmp/wt/seed-todo");
});
it("bypassGuards capacity pass-through (U4 documenting test): engine move into in-progress is NOT blocked by capacity (U6 fills enforcement)", async () => {
// U4 intentionally leaves the per-(workflow,column) capacity check as a
// pass-through slot; capacity enforcement lands in U6. This test pins the
// U4 contract: no WIP-constrained scenario is enforced yet, and an engine
// move (bypassGuards) into a wip-flagged column commits. It must be UPDATED
// by U6 (capacity is NEVER bypassable, KTD-10) — not silently left green.
it("U6 in-txn capacity: default-workflow in-progress WIP reads through maxConcurrent and rejects the over-limit move", async () => {
// The default workflow's in-progress column has a `wip` trait whose limit
// reads through to settings.maxConcurrent (legacy parity). With limit 1, the
// first move into in-progress commits and a second rejects with the typed
// capacity-exhausted code.
await store.updateSettings({ maxConcurrent: 1 } as Parameters<typeof store.updateSettings>[0]);
const t1 = await seedInColumn("todo");
const t2 = await seedInColumn("todo");
const m1 = await store.moveTask(t1.id, "in-progress", { moveSource: "engine" });
const m2 = await store.moveTask(t2.id, "in-progress", { moveSource: "engine" });
const m1 = await store.moveTask(t1.id, "in-progress", { moveSource: "user" });
expect(m1.column).toBe("in-progress");
expect(m2.column).toBe("in-progress");
let caught: unknown;
try {
await store.moveTask(t2.id, "in-progress", { moveSource: "user" });
} catch (e) {
caught = e;
}
expect(caught).toBeInstanceOf(TransitionRejectionError);
expect((caught as TransitionRejectionError).rejection.code).toBe("capacity-exhausted");
// The rejected card is untouched.
expect((await store.getTask(t2.id))?.column).toBe("todo");
});
it("U6 capacity is NEVER bypassable (KTD-10): an engine/bypassGuards move into a full column still rejects", async () => {
await store.updateSettings({ maxConcurrent: 1 } as Parameters<typeof store.updateSettings>[0]);
const t1 = await seedInColumn("todo");
const t2 = await seedInColumn("todo");
await store.moveTask(t1.id, "in-progress", { moveSource: "user" });
let caught: unknown;
try {
// Engine-sourced + bypassGuards skips trait guards, but capacity is not a
// guard — it must still reject.
await store.moveTask(t2.id, "in-progress", { moveSource: "engine", bypassGuards: true });
} catch (e) {
caught = e;
}
expect(caught).toBeInstanceOf(TransitionRejectionError);
expect((caught as TransitionRejectionError).rejection.code).toBe("capacity-exhausted");
});
it("U6 capacity counts cards mid-transitionPending (they hold their slot from commit time)", async () => {
await store.updateSettings({ maxConcurrent: 1 } as Parameters<typeof store.updateSettings>[0]);
const t1 = await seedInColumn("todo");
const t2 = await seedInColumn("todo");
await store.moveTask(t1.id, "in-progress", { moveSource: "user" });
// Simulate a crash before t1's marker clears: it is still mid-transition into
// in-progress, holding its slot. (Its column already equals in-progress, so
// this also independently holds the slot; this asserts the marker path does
// not under-count or double-count.)
const db = (store as unknown as { db: { prepare: (s: string) => { run: (...a: unknown[]) => unknown } } }).db;
db.prepare("UPDATE tasks SET transitionPending = ? WHERE id = ?").run(
JSON.stringify({ toColumn: "in-progress", hooksRemaining: ["default-workflow:postCommit"], startedAt: Date.now() }),
t1.id,
);
let caught: unknown;
try {
await store.moveTask(t2.id, "in-progress", { moveSource: "user" });
} catch (e) {
caught = e;
}
expect(caught).toBeInstanceOf(TransitionRejectionError);
expect((caught as TransitionRejectionError).rejection.code).toBe("capacity-exhausted");
});
});

View File

@@ -69,7 +69,7 @@ export interface DefaultWorkflowMoveContext {
task: Task;
fromColumn: string;
toColumn: string;
moveSource: "user" | "engine";
moveSource: "user" | "engine" | "scheduler";
/** True when guards + abort-on-exit are bypassed (engine/recovery, KTD-9). */
bypassGuards: boolean;
movedAt: string;

View File

@@ -132,6 +132,9 @@ export {
} from "./workflow-transitions.js";
export type { ColumnAdjacency } from "./workflow-transitions.js";
export { isWorkflowColumnsEnabled } from "./workflow-columns-settings.js";
// ── U6: workflow capacity (WIP) resolution shared by store + sweep ───────────
export { resolveColumnCapacity } from "./workflow-capacity.js";
export type { ColumnCapacity } from "./workflow-capacity.js";
// ── U5: workflow lifecycle reconciliation (switch / edit / delete) ───────────
export {
OccupiedColumnsError,

View File

@@ -10,6 +10,7 @@ import { DEFAULT_PROJECT_SETTINGS } from "./settings-schema.js";
import { parseWorkflowIr, serializeWorkflowIr } from "./workflow-ir.js";
import { isWorkflowColumnsEnabled } from "./workflow-columns-settings.js";
import { resolveAllowedColumns, workflowHasColumn } from "./workflow-transitions.js";
import { resolveColumnCapacity } from "./workflow-capacity.js";
import {
OccupiedColumnsError,
assertRehomeTargetValid,
@@ -676,7 +677,7 @@ function deepMergeWithNullDelete(
export interface TaskStoreEvents {
"task:created": [task: Task];
"task:moved": [data: { task: Task; from: Column; to: Column; source: "user" | "engine" }];
"task:moved": [data: { task: Task; from: Column; to: Column; source: "user" | "engine" | "scheduler" }];
"task:updated": [task: Task];
"task:deleted": [task: Task, meta?: { githubIssueAction?: GithubIssueAction }];
"task:merged": [result: MergeResult];
@@ -1102,7 +1103,7 @@ interface MoveTaskOptions {
preserveWorktree?: boolean;
preserveStatus?: boolean;
allocateWorktree?: (reservedNames: Set<string>) => string | null;
moveSource?: "user" | "engine";
moveSource?: "user" | "engine" | "scheduler";
skipMergeBlocker?: boolean;
allowDirectInReviewMove?: boolean;
/**
@@ -1138,6 +1139,10 @@ interface MoveTaskInternalOptions {
export class TaskStore extends EventEmitter<TaskStoreEvents> {
private static readonly ACTIVE_TASKS_WHERE = '"deletedAt" IS NULL';
/** U6: sentinel effective-workflow id for default-workflow (null-selection)
* tasks, so they all share one per-column capacity pool (KTD-10). It is not a
* real workflow row id (no `builtin:`/custom collision possible). */
private static readonly DEFAULT_WORKFLOW_POOL_ID = "__default-workflow__";
static async getOrCreateForProject(
projectId?: string,
@@ -5653,14 +5658,16 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
// `getSettingsSync()` row would miss it — read merged settings (global +
// project) via getSettingsFast(). This is an async read taken before the
// lock-sensitive transaction; it does not touch the task lock.
const useWorkflow = isWorkflowColumnsEnabled(await this.getSettingsFast());
const mergedSettingsForMove = await this.getSettingsFast();
const useWorkflow = isWorkflowColumnsEnabled(mergedSettingsForMove);
// bypassGuards (KTD-9): engine-sourced moves + the existing skipMergeBlocker
// call sites map onto it. Capacity (KTD-10) is NEVER bypassed by this — the
// capacity check is not a guard (U6 fills the enforcement; U4 leaves a
// pass-through slot). An explicit option value wins; otherwise derive it.
const bypassGuards =
options?.recoveryRehome === true ||
(options?.bypassGuards ?? (moveSource === "engine" || options?.skipMergeBlocker === true));
(options?.bypassGuards ??
(moveSource === "engine" || moveSource === "scheduler" || options?.skipMergeBlocker === true));
const workflowIr: WorkflowIr | undefined = useWorkflow
? this.resolveTaskWorkflowIrSync(id)
: undefined;
@@ -5974,6 +5981,41 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
return;
}
// ── U6: in-txn capacity enforcement (KTD-10) ──────────────────────────
// WIP limits are trait *config*; enforcement is a substrate capability
// that runs HERE, inside the move transaction, so two holds releasing into
// one slot serialize — exactly one commits, the other rejects and retries
// next sweep. It is NOT a guard: it runs regardless of bypassGuards /
// recoveryRehome / moveSource (engine/recovery/scheduler moves honor it
// too). Only a real column change into a capacity-bearing column is gated;
// same-column no-ops were returned earlier. The count is taken with the
// moving task EXCLUDED and the prospective slot it is about to occupy
// added back implicitly (it must fit alongside existing holders), so a
// full column (occupants == limit) rejects.
if (useWorkflow && workflowIr && fromColumn !== toColumn) {
const capacity = resolveColumnCapacity(workflowIr, toColumn, mergedSettingsForMove);
if (capacity.hasCapacity && Number.isFinite(capacity.limit)) {
const workflowId = this.resolveEffectiveWorkflowIdSync(id);
const occupants = this.countActiveInCapacitySlotSync({
targetColumn: toColumn,
workflowId,
countPending: capacity.countPending,
excludeTaskId: id,
});
if (occupants >= capacity.limit) {
throw new TransitionRejectionError(
makeTransitionRejection(
"capacity-exhausted",
"transition.rejected.capacityExhausted",
true,
`Column '${toColumn}' is at capacity (${occupants}/${capacity.limit})`,
),
`Cannot move ${id} to '${toColumn}': column at capacity (${occupants}/${capacity.limit})`,
);
}
}
}
this.upsertTaskWithFtsRecovery(task);
this.insertRunAuditEventRow({
taskId: id,
@@ -11880,6 +11922,81 @@ ${stepsSection}`;
}
}
/**
* U6 (KTD-10): the *effective workflow id* used to scope the per-(workflow,
* column) capacity count. A task with no selection (or a missing/empty
* selection row) resolves to the built-in default workflow, represented by a
* stable sentinel so all default-workflow tasks share one capacity pool. A
* selected workflow id (builtin or custom) is its own pool. Pure DB read; safe
* inside the move transaction.
*/
private resolveEffectiveWorkflowIdSync(taskId: string): string {
const selection = this.getTaskWorkflowSelection(taskId);
return selection?.workflowId ?? TaskStore.DEFAULT_WORKFLOW_POOL_ID;
}
/**
* U6 (KTD-10): count cards currently occupying a (workflow, column) capacity
* slot, for the in-txn capacity check. Runs INSIDE `moveTaskInternal`'s
* transaction. A slot is held by a card that:
* - has committed its column to `targetColumn` (the steady-state holders), OR
* - (when `countPending`) has a `transitionPending` marker targeting
* `targetColumn` — it reserved the slot at commit time even though its
* post-commit hooks haven't finished yet.
* The moving task itself (`excludeTaskId`) is excluded so a same-column no-op
* or re-entry never counts itself. Only the candidates in the SAME effective
* workflow as the mover count (capacity is per-(workflow, column)). Soft-deleted
* tasks never hold a slot.
*/
private countActiveInCapacitySlotSync(params: {
targetColumn: string;
workflowId: string;
countPending: boolean;
excludeTaskId: string;
}): number {
const { targetColumn, workflowId, countPending, excludeTaskId } = params;
// Candidate rows: in the column now, or (optionally) mid-transition into it.
// LEFT JOIN the selection row so we can scope by effective workflow id in JS.
const rows = this.db
.prepare(
`SELECT t.id AS id, t."column" AS col, t.transitionPending AS tp, s.workflowId AS wid
FROM tasks t
LEFT JOIN task_workflow_selection s ON s.taskId = t.id
WHERE t.deletedAt IS NULL
AND t.id != ?
AND (t."column" = ? OR (t.transitionPending IS NOT NULL AND t.transitionPending != ''))`,
)
.all(excludeTaskId, targetColumn) as Array<{
id: string;
col: string;
tp: string | null;
wid: string | null;
}>;
let count = 0;
for (const row of rows) {
const effectiveWorkflowId = row.wid ?? TaskStore.DEFAULT_WORKFLOW_POOL_ID;
if (effectiveWorkflowId !== workflowId) continue;
if (row.col === targetColumn) {
count += 1;
continue;
}
// Not committed into the column — only counts if it has reserved the slot
// via a transitionPending marker targeting this column AND countPending.
if (!countPending || !row.tp) continue;
let toColumn: string | undefined;
try {
const parsed = JSON.parse(row.tp) as { toColumn?: unknown };
if (typeof parsed.toColumn === "string") toColumn = parsed.toColumn;
} catch {
// Corrupt marker — treat as not holding this slot.
}
if (toColumn === targetColumn) count += 1;
}
return count;
}
getTaskWorkflowSelection(taskId: string): { workflowId: string; stepIds: string[] } | undefined {
const row = this.db
.prepare("SELECT workflowId, stepIds FROM task_workflow_selection WHERE taskId = ?")

View File

@@ -0,0 +1,116 @@
/**
* Workflow capacity resolution (U6, KTD-10, R9 capacity half).
*
* WIP/capacity limits are trait *configuration*; their *enforcement* is a
* substrate capability that runs INSIDE `moveTaskInternal`'s transaction and is
* NEVER bypassable (not a guard — runs regardless of bypassGuards/recoveryRehome
* /moveSource). This module is the pure resolution layer shared by both the
* in-txn check (`store.ts`) and the hold/release sweep (`@fusion/engine`
* `hold-release.ts`): given a workflow IR + a column id + settings it answers
* - does this column have a `wip` (capacity) trait?
* - what is its effective limit (read-through to `settings.maxConcurrent` for
* the default workflow's in-progress column so the legacy knob keeps working
* — U6 scheduler-integration half)?
* - does its config opt into counting mid-`transitionPending` cards?
*
* It performs NO DB access and NO counting — the caller owns the count (the
* store counts in-txn; the sweep counts from a listTasks snapshot). Keeping the
* resolution pure means the two enforcement points can never disagree on what a
* limit *is*, only on the live count, which is exactly the serialization the
* in-txn check arbitrates (two holds, one slot → one wins).
*/
import type { Settings } from "./types.js";
import type { WorkflowIr, WorkflowIrV2, WorkflowIrColumn } from "./workflow-ir-types.js";
import { DEFAULT_WORKFLOW_COLUMN_IDS } from "./workflow-ir.js";
import { getTraitRegistry } from "./trait-registry.js";
/** The default-workflow column whose WIP limit read-through is
* `settings.maxConcurrent` (the legacy "N agents in-progress" gate). */
const DEFAULT_WIP_COLUMN_ID = "in-progress";
/** Resolved capacity configuration for a single column. */
export interface ColumnCapacity {
/** True when the column carries a capacity (`wip`/`countsTowardWip`) trait. */
hasCapacity: boolean;
/** The effective max concurrent cards. `Infinity` means "no finite limit"
* (a capacity trait with no resolvable limit does not gate). */
limit: number;
/** Whether mid-`transitionPending` cards (holding their destination slot from
* commit time) count toward the limit. Defaults true: a card that has
* committed its move into the column holds the slot even before its
* post-commit hooks finish (KTD-10). */
countPending: boolean;
}
const NO_CAPACITY: ColumnCapacity = { hasCapacity: false, limit: Infinity, countPending: true };
function findColumn(ir: WorkflowIr, columnId: string): WorkflowIrColumn | undefined {
const v2 = ir as WorkflowIrV2;
if (!Array.isArray(v2.columns)) return undefined;
return v2.columns.find((c) => c.id === columnId);
}
/** True when the IR's column set is exactly the default-workflow column ids. */
function isDefaultWorkflowColumns(ir: WorkflowIr): boolean {
const v2 = ir as WorkflowIrV2;
if (!Array.isArray(v2.columns)) return false;
const ids = v2.columns.map((c) => c.id);
if (ids.length !== DEFAULT_WORKFLOW_COLUMN_IDS.length) return false;
const set = new Set(ids);
return DEFAULT_WORKFLOW_COLUMN_IDS.every((id) => set.has(id));
}
/**
* Resolve the capacity configuration for `columnId` under `ir`.
*
* Limit resolution order:
* 1. An explicit numeric `limit` in the column's `wip` trait config wins.
* 2. Otherwise, for the DEFAULT workflow's `in-progress` column, read through
* to `settings.maxConcurrent` (default 2) so the legacy knob keeps working
* and flag-ON default-workflow scheduling matches flag-OFF (legacy parity).
* 3. Otherwise the column has a capacity trait but no resolvable finite limit
* → `Infinity` (does not gate; the trait is inert until configured).
*/
export function resolveColumnCapacity(
ir: WorkflowIr,
columnId: string,
settings?: Pick<Settings, "maxConcurrent"> | undefined,
): ColumnCapacity {
const column = findColumn(ir, columnId);
if (!column) return NO_CAPACITY;
const flags = getTraitRegistry().resolveColumnFlags(column);
if (!flags.countsTowardWip) return NO_CAPACITY;
// The capacity trait config (the `wip` trait carries `limit` + `countPending`).
// Find the first trait config whose trait sets countsTowardWip.
let configLimit: number | undefined;
let countPending = true;
for (const ct of column.traits) {
const def = getTraitRegistry().getTrait(ct.trait);
if (!def?.flags.countsTowardWip) continue;
const cfg = ct.config ?? {};
if (typeof cfg.limit === "number" && Number.isFinite(cfg.limit)) {
configLimit = cfg.limit;
}
if (typeof cfg.countPending === "boolean") {
countPending = cfg.countPending;
}
break;
}
let limit: number;
if (configLimit !== undefined) {
limit = configLimit;
} else if (columnId === DEFAULT_WIP_COLUMN_ID && isDefaultWorkflowColumns(ir)) {
// Read-through: legacy maxConcurrent maps onto the default workflow's
// in-progress WIP limit (U6 scheduler integration).
const maxConcurrent = settings?.maxConcurrent;
limit = typeof maxConcurrent === "number" && Number.isFinite(maxConcurrent) ? maxConcurrent : 2;
} else {
limit = Infinity;
}
return { hasCapacity: true, limit, countPending };
}

View File

@@ -0,0 +1,402 @@
// @vitest-environment node
//
// HOLD/RELEASE SWEEP SUITE (U6).
//
// Exercises the generalized scheduler sweep (`hold-release.ts`) against a REAL
// TaskStore so the in-txn capacity check (KTD-10) actually arbitrates races:
// - two holds, one slot → exactly one releases; other retries next sweep
// - timer release fires at its deadline under fake timers (no real sleeps)
// - manual release only on the explicit promote call
// - capacity release respects mid-transitionPending cards (in-txn authority)
// - cross-workflow dependency complete-flag unblocks + dual-accept diff logged
// - sweep release into a full column rejected by the in-txn check despite
// moveSource:"scheduler" bypassing trait guards (capacity is not a guard)
// - reservation-first: semaphore exhausted → no commit, card stays held
// - paused / recovery-backoff tasks skipped exactly as the legacy scheduler
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { execSync } from "node:child_process";
import { TaskStore, type WorkflowIr } from "@fusion/core";
import {
runHoldReleaseSweep,
promoteHeldTask,
releaseHeldTaskByEvent,
type HoldReleaseDeps,
type SlotReservation,
} from "../hold-release.js";
function git(cwd: string, args: string): void {
execSync(`git ${args}`, { cwd, stdio: "ignore" });
}
/** Directly set a task's stored column (test setup helper — bypasses adjacency
* validation so a card can be placed at an arbitrary workflow column). */
function setColumn(store: TaskStore, taskId: string, column: string): void {
const db = (store as unknown as { db: { prepare: (s: string) => { run: (...a: unknown[]) => unknown } } }).db;
db.prepare('UPDATE tasks SET "column" = ?, "columnMovedAt" = ? WHERE id = ?').run(
column,
new Date().toISOString(),
taskId,
);
}
/** Directly set a task's workflow selection row (bypasses step compilation). */
function setSelection(store: TaskStore, taskId: string, workflowId: string): void {
const db = (store as unknown as { db: { prepare: (s: string) => { run: (...a: unknown[]) => unknown } } }).db;
db.prepare(
`INSERT INTO task_workflow_selection (taskId, workflowId, stepIds, updatedAt)
VALUES (?, ?, '[]', ?)
ON CONFLICT(taskId) DO UPDATE SET workflowId = excluded.workflowId, updatedAt = excluded.updatedAt`,
).run(taskId, workflowId, new Date().toISOString());
}
/** Write a transitionPending marker directly (simulating a crash mid-transition). */
function setTransitionPending(store: TaskStore, taskId: string, toColumn: string): void {
const db = (store as unknown as { db: { prepare: (s: string) => { run: (...a: unknown[]) => unknown } } }).db;
db.prepare("UPDATE tasks SET transitionPending = ? WHERE id = ?").run(
JSON.stringify({ toColumn, hooksRemaining: ["default-workflow:postCommit"], startedAt: Date.now() }),
taskId,
);
}
const noReserveDeps: HoldReleaseDeps = { now: () => Date.now() };
describe("hold-release sweep (U6)", () => {
let rootDir = "";
let store: TaskStore;
beforeEach(async () => {
rootDir = mkdtempSync(join(tmpdir(), "u6-hold-release-"));
git(rootDir, "init -b main");
git(rootDir, "config user.name 'Fusion'");
git(rootDir, "config user.email 'hi@runfusion.ai'");
writeFileSync(join(rootDir, "README.md"), "root\n");
git(rootDir, "add README.md");
git(rootDir, "commit -m init");
store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
await store.init();
await store.updateGlobalSettings({ experimentalFeatures: { workflowColumns: true } });
});
afterEach(() => {
try { store?.close(); } catch { /* ignore */ }
if (rootDir) rmSync(rootDir, { recursive: true, force: true });
vi.useRealTimers();
vi.clearAllMocks();
});
// A held card in the DEFAULT workflow: a task resting in `todo`
// (hold release: capacity), which releases into `in-progress` (wip).
async function seedTodoCard(): Promise<string> {
const task = await store.createTask({ description: "card" });
setColumn(store, task.id, "todo");
return task.id;
}
it("flag OFF: sweep is a no-op (legacy scheduler path untouched)", async () => {
await store.updateGlobalSettings({ experimentalFeatures: { workflowColumns: false } });
const id = await seedTodoCard();
const result = await runHoldReleaseSweep(store, noReserveDeps);
expect(result.released).toEqual([]);
expect((await store.getTask(id))?.column).toBe("todo");
});
it("two holds, one slot: exactly one releases; the other releases next sweep after the slot frees", async () => {
await store.updateSettings({ maxConcurrent: 1 } as Parameters<typeof store.updateSettings>[0]);
const a = await seedTodoCard();
const b = await seedTodoCard();
const r1 = await runHoldReleaseSweep(store, noReserveDeps);
expect(r1.released.length).toBe(1);
const released = r1.released[0];
const stillHeld = released === a ? b : a;
expect((await store.getTask(released))?.column).toBe("in-progress");
expect((await store.getTask(stillHeld))?.column).toBe("todo");
// Free the slot by moving the released card out of in-progress.
await store.moveTask(released, "in-review", { moveSource: "engine", allowDirectInReviewMove: true });
const r2 = await runHoldReleaseSweep(store, noReserveDeps);
expect(r2.released).toContain(stillHeld);
expect((await store.getTask(stillHeld))?.column).toBe("in-progress");
});
it("sweep release into a full column is rejected by the in-txn check (capacity is not a guard, scheduler bypasses guards)", async () => {
await store.updateSettings({ maxConcurrent: 1 } as Parameters<typeof store.updateSettings>[0]);
const occupant = await store.createTask({ description: "occupant" });
setColumn(store, occupant.id, "in-progress");
const held = await seedTodoCard();
const result = await runHoldReleaseSweep(store, noReserveDeps);
expect(result.released).not.toContain(held);
expect((await store.getTask(held))?.column).toBe("todo");
});
it("capacity release respects cards mid-transitionPending (they hold the slot from commit time)", async () => {
await store.updateSettings({ maxConcurrent: 1 } as Parameters<typeof store.updateSettings>[0]);
// Occupant has committed into in-progress AND is mid-transitionPending — it
// holds the slot; the in-txn count must include it.
const occupant = await store.createTask({ description: "occupant" });
setColumn(store, occupant.id, "in-progress");
setTransitionPending(store, occupant.id, "in-progress");
const held = await seedTodoCard();
const result = await runHoldReleaseSweep(store, noReserveDeps);
expect((await store.getTask(held))?.column).toBe("todo");
expect(result.released).not.toContain(held);
});
it("paused and recovery-backoff tasks are skipped exactly as the legacy scheduler", async () => {
await store.updateSettings({ maxConcurrent: 5 } as Parameters<typeof store.updateSettings>[0]);
const paused = await seedTodoCard();
await store.updateTask(paused, { paused: true });
const backoff = await seedTodoCard();
await store.updateTask(backoff, { nextRecoveryAt: new Date(Date.now() + 60_000).toISOString() });
const result = await runHoldReleaseSweep(store, { now: () => Date.now() });
expect(result.released).not.toContain(paused);
expect(result.released).not.toContain(backoff);
expect((await store.getTask(paused))?.column).toBe("todo");
expect((await store.getTask(backoff))?.column).toBe("todo");
});
it("reservation-first: semaphore exhausted → no commit, card stays held", async () => {
await store.updateSettings({ maxConcurrent: 5 } as Parameters<typeof store.updateSettings>[0]);
const held = await seedTodoCard();
// reserveSlot returns null (semaphore exhausted) for a processing-column
// release — the move must never be issued.
const deps: HoldReleaseDeps = {
now: () => Date.now(),
reserveSlot: (): SlotReservation | null => null,
};
const result = await runHoldReleaseSweep(store, deps);
expect(result.released).not.toContain(held);
expect((await store.getTask(held))?.column).toBe("todo");
});
it("reservation is RELEASED when the move rejects on capacity", async () => {
await store.updateSettings({ maxConcurrent: 1 } as Parameters<typeof store.updateSettings>[0]);
const occupant = await store.createTask({ description: "occupant" });
setColumn(store, occupant.id, "in-progress");
const held = await seedTodoCard();
const releases: number[] = [];
let reserveCount = 0;
const deps: HoldReleaseDeps = {
now: () => Date.now(),
reserveSlot: (): SlotReservation | null => {
reserveCount += 1;
return { release: () => releases.push(1) };
},
};
const result = await runHoldReleaseSweep(store, deps);
expect(result.released).not.toContain(held);
// A reservation was taken (downstream pre-check passed since maxConcurrent
// read-through is evaluated against the snapshot) then released on the
// in-txn capacity rejection. If the pre-check already gated, reserveCount
// may be 0; if it reserved, it must have released exactly once.
if (reserveCount > 0) expect(releases.length).toBe(reserveCount);
});
});
// ── Timer / manual / external-event holds (custom workflows) ──────────────────
/** A custom workflow whose middle column is a hold with the given release kind.
* Columns: c-intake (intake) → c-hold (hold) → c-run (wip) → c-done (complete). */
function customHoldWorkflowIr(release: string, holdConfig: Record<string, unknown> = {}): WorkflowIr {
return {
version: "v2",
name: "custom-hold",
columns: [
{ id: "c-intake", name: "Intake", traits: [{ trait: "intake" }] },
{ id: "c-hold", name: "Hold", traits: [{ trait: "hold", config: { release, ...holdConfig } }] },
{ id: "c-run", name: "Run", traits: [{ trait: "wip", config: { limit: 5 } }] },
{ id: "c-done", name: "Done", traits: [{ trait: "complete" }] },
],
nodes: [
{ id: "start", kind: "start", column: "c-intake" },
{ id: "end", kind: "end", column: "c-done" },
],
edges: [{ from: "start", to: "end" }],
} as WorkflowIr;
}
describe("hold-release sweep — timer / manual / external-event (U6)", () => {
let rootDir = "";
let store: TaskStore;
beforeEach(async () => {
rootDir = mkdtempSync(join(tmpdir(), "u6-hold-kinds-"));
git(rootDir, "init -b main");
git(rootDir, "config user.name 'Fusion'");
git(rootDir, "config user.email 'hi@runfusion.ai'");
writeFileSync(join(rootDir, "README.md"), "root\n");
git(rootDir, "add README.md");
git(rootDir, "commit -m init");
store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
await store.init();
await store.updateGlobalSettings({ experimentalFeatures: { workflowColumns: true } });
});
afterEach(() => {
try { store?.close(); } catch { /* ignore */ }
if (rootDir) rmSync(rootDir, { recursive: true, force: true });
vi.useRealTimers();
});
async function seedCustomHold(release: string, holdConfig: Record<string, unknown> = {}): Promise<string> {
const def = await store.createWorkflowDefinition({ name: `wf-${release}`, ir: customHoldWorkflowIr(release, holdConfig) });
const task = await store.createTask({ description: `hold-${release}` });
setSelection(store, task.id, def.id);
setColumn(store, task.id, "c-hold");
return task.id;
}
it("timer release fires at the deadline under fake timers (no real sleeps)", async () => {
vi.useFakeTimers();
const base = Date.now();
const id = await seedCustomHold("timer", { durationMs: 10_000 });
// Re-stamp columnMovedAt to the fake-clock base so the deadline is base+10s.
const db = (store as unknown as { db: { prepare: (s: string) => { run: (...a: unknown[]) => unknown } } }).db;
db.prepare('UPDATE tasks SET "columnMovedAt" = ? WHERE id = ?').run(new Date(base).toISOString(), id);
// Before the deadline: not released.
const before = await runHoldReleaseSweep(store, { now: () => base + 5_000 });
expect(before.released).not.toContain(id);
expect((await store.getTask(id))?.column).toBe("c-hold");
// At/after the deadline: released into the downstream run column.
const after = await runHoldReleaseSweep(store, { now: () => base + 10_000 });
expect(after.released).toContain(id);
expect((await store.getTask(id))?.column).toBe("c-run");
});
it("manual hold: the sweep never auto-releases; an explicit promote does", async () => {
const id = await seedCustomHold("manual");
const swept = await runHoldReleaseSweep(store, { now: () => Date.now() });
expect(swept.released).not.toContain(id);
expect((await store.getTask(id))?.column).toBe("c-hold");
const promoted = await promoteHeldTask(store, id);
expect(promoted.released).toBe(true);
expect(promoted.toColumn).toBe("c-run");
expect((await store.getTask(id))?.column).toBe("c-run");
});
it("external-event hold: the sweep never auto-releases; an event release does; a stray event on a manual hold is a no-op", async () => {
const eventId = await seedCustomHold("external-event");
const swept = await runHoldReleaseSweep(store, { now: () => Date.now() });
expect(swept.released).not.toContain(eventId);
const released = await releaseHeldTaskByEvent(store, eventId, "webhook:approved");
expect(released.released).toBe(true);
expect((await store.getTask(eventId))?.column).toBe("c-run");
// A manual hold is NOT releasable by an external event.
const manualId = await seedCustomHold("manual");
const stray = await releaseHeldTaskByEvent(store, manualId, "webhook:approved");
expect(stray.released).toBe(false);
expect((await store.getTask(manualId))?.column).toBe("c-hold");
});
});
// ── Dependency gating (KTD-5 + FN-5719 dual-accept) ───────────────────────────
/** A custom workflow with a hold(dependency) column. */
function dependencyHoldWorkflowIr(): WorkflowIr {
return {
version: "v2",
name: "dep-hold",
columns: [
{ id: "d-intake", name: "Intake", traits: [{ trait: "intake" }] },
{ id: "d-hold", name: "Hold", traits: [{ trait: "hold", config: { release: "dependency" } }] },
{ id: "d-run", name: "Run", traits: [{ trait: "wip", config: { limit: 5 } }] },
{ id: "d-done", name: "Done", traits: [{ trait: "complete" }] },
],
nodes: [
{ id: "start", kind: "start", column: "d-intake" },
{ id: "end", kind: "end", column: "d-done" },
],
edges: [{ from: "start", to: "end" }],
} as WorkflowIr;
}
/** A custom "producer" workflow whose terminal column carries the complete flag
* under a NON-legacy column id (so the complete-flag path differs from the
* legacy done/in-review/archived signal — used for the dual-accept diff). */
function completeFlagWorkflowIr(): WorkflowIr {
return {
version: "v2",
name: "producer",
columns: [
{ id: "p-intake", name: "Intake", traits: [{ trait: "intake" }] },
{ id: "p-finished", name: "Finished", traits: [{ trait: "complete" }] },
],
nodes: [
{ id: "start", kind: "start", column: "p-intake" },
{ id: "end", kind: "end", column: "p-finished" },
],
edges: [{ from: "start", to: "end" }],
} as WorkflowIr;
}
describe("hold-release sweep — dependency gating (KTD-5)", () => {
let rootDir = "";
let store: TaskStore;
beforeEach(async () => {
rootDir = mkdtempSync(join(tmpdir(), "u6-dep-"));
git(rootDir, "init -b main");
git(rootDir, "config user.name 'Fusion'");
git(rootDir, "config user.email 'hi@runfusion.ai'");
writeFileSync(join(rootDir, "README.md"), "root\n");
git(rootDir, "add README.md");
git(rootDir, "commit -m init");
store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
await store.init();
await store.updateGlobalSettings({ experimentalFeatures: { workflowColumns: true } });
});
afterEach(() => {
try { store?.close(); } catch { /* ignore */ }
if (rootDir) rmSync(rootDir, { recursive: true, force: true });
vi.restoreAllMocks();
});
it("a dependency in another workflow's complete-flagged column unblocks the dependent; dual-accept logs a diff on disagreement", async () => {
const auditSpy = vi.spyOn(store, "recordRunAuditEvent");
const producerDef = await store.createWorkflowDefinition({ name: "producer", ir: completeFlagWorkflowIr() });
const dep = await store.createTask({ description: "producer task" });
setSelection(store, dep.id, producerDef.id);
// Producer NOT yet complete → dependent stays held.
setColumn(store, dep.id, "p-intake");
const depHoldDef = await store.createWorkflowDefinition({ name: "dep-hold", ir: dependencyHoldWorkflowIr() });
const dependent = await store.createTask({ description: "dependent", dependencies: [dep.id] });
setSelection(store, dependent.id, depHoldDef.id);
setColumn(store, dependent.id, "d-hold");
const r1 = await runHoldReleaseSweep(store, { now: () => Date.now() });
expect(r1.released).not.toContain(dependent.id);
expect((await store.getTask(dependent.id))?.column).toBe("d-hold");
// Move the producer into its complete-flagged column (NON-legacy id).
setColumn(store, dep.id, "p-finished");
auditSpy.mockClear();
const r2 = await runHoldReleaseSweep(store, { now: () => Date.now() });
expect(r2.released).toContain(dependent.id);
expect((await store.getTask(dependent.id))?.column).toBe("d-run");
// Dual-accept disagreement: the complete-flag says satisfied, but the legacy
// signal (column p-finished is NOT done/in-review/archived, no marker) says
// NOT satisfied → an audit-diff event was logged.
const diffLogged = auditSpy.mock.calls.some(
(call) => (call[0] as { mutationType?: string })?.mutationType === "merge:dependency-parity-diff",
);
expect(diffLogged).toBe(true);
});
});

View File

@@ -92,6 +92,12 @@ function createMockStore(overrides: Partial<TaskStore> = {}): TaskStore {
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
getRootDir: vi.fn().mockReturnValue("/test/project"),
getTasksDir: vi.fn().mockReturnValue("/test/project/.fusion/tasks"),
// U6: the hold/release sweep consults workflow selection + completion markers
// when the workflowColumns flag is ON; default mocks keep flag-OFF behavior
// (sweep early-returns before touching these).
getTaskWorkflowSelection: vi.fn().mockReturnValue(undefined),
getWorkflowDefinition: vi.fn().mockResolvedValue(undefined),
getCompletionHandoffAcceptedMarker: vi.fn().mockReturnValue(null),
on: vi.fn(),
off: vi.fn(),
...overrides,
@@ -530,6 +536,50 @@ describe("Scheduler", () => {
});
});
describe("U6 hold/release sweep integration (flag-gated)", () => {
function setupTodoStore(workflowColumns: boolean) {
vi.mocked(existsSync).mockReturnValue(true);
vi.mocked(readFile).mockResolvedValue("# Task\nDo something");
const todo = createMockTask({ id: "FN-1", column: "todo", dependencies: [] });
const store = createMockStore({
listTasks: vi.fn().mockResolvedValue([todo]),
getTask: vi.fn().mockResolvedValue(todo),
getSettings: vi.fn().mockResolvedValue({
maxConcurrent: 2,
maxWorktrees: 4,
experimentalFeatures: { workflowColumns },
}),
});
const scheduler = new Scheduler(store);
(scheduler as unknown as { running: boolean }).running = true;
return { store, scheduler };
}
it("flag-ON default-workflow pickup matches flag-OFF: same todo→in-progress dispatch", async () => {
// Flag-OFF baseline: the legacy pull-from-todo loop dispatches the card.
const off = setupTodoStore(false);
await off.scheduler.schedule();
const offMoves = vi.mocked(off.store.moveTask).mock.calls.map((c) => [c[0], c[1]]);
expect(offMoves).toContainEqual(["FN-1", "in-progress"]);
// Flag-ON: the sweep runs first (default-workflow todo is a capacity hold),
// then the legacy loop; the net dispatch is the SAME todo→in-progress move.
const on = setupTodoStore(true);
await on.scheduler.schedule();
const onMoves = vi.mocked(on.store.moveTask).mock.calls.map((c) => [c[0], c[1]]);
expect(onMoves).toContainEqual(["FN-1", "in-progress"]);
});
it("flag-OFF: the sweep never issues a scheduler-sourced move (legacy path byte-identical)", async () => {
const off = setupTodoStore(false);
await off.scheduler.schedule();
const schedulerSourcedMoves = vi
.mocked(off.store.moveTask)
.mock.calls.filter((c) => (c[2] as { moveSource?: string } | undefined)?.moveSource === "scheduler");
expect(schedulerSourcedMoves.length).toBe(0);
});
});
describe("backlog pressure reporter integration", () => {
it("invokes reporter from schedule when enabled", async () => {
vi.useFakeTimers();

View File

@@ -111,6 +111,22 @@ export class AgentSemaphore {
});
}
/**
* Synchronously reserve a slot if one is immediately available, without
* queuing. Returns true (and bumps `activeCount`) when a slot was taken,
* false when the semaphore is full. Used by the U6 hold/release sweep's
* reservation-first ordering (KTD-10): reserve worktree + semaphore BEFORE
* issuing a release move, and {@link release} the reservation if the move
* rejects on capacity. Unlike {@link acquire} it never enqueues a waiter.
*/
tryAcquire(): boolean {
if (this._active < this.limit) {
this._active++;
return true;
}
return false;
}
/**
* Release a previously acquired slot and unblock the next waiting caller
* (if any).

View File

@@ -0,0 +1,537 @@
/**
* Hold/release sweep — the generalized scheduler (U6, KTD-10, R3 behavior half).
*
* Flag-ON, the scheduler's poll becomes a *hold/release sweep*: for each
* workflow in use by live tasks, it finds cards resting at `hold`-trait columns
* and evaluates their release condition:
*
* - `manual` — released ONLY by an explicit {@link promoteHeldTask}
* call (U9's promote endpoint / CLI). The sweep never
* auto-releases a manual hold.
* - `external-event` — released ONLY by {@link releaseHeldTaskByEvent} (a
* webhook/API release, same shape as manual + an event
* tag).
* - `timer` — released when the injected clock passes the hold's
* deadline (`columnMovedAt + durationMs`, or an explicit
* `deadlineAt`). Fake-timer friendly (FN-5048): the clock
* is injected, never `Date.now()` baked in.
* - `capacity` — released when a downstream capacity (`wip`) column has a
* free slot (same counting rules as the in-txn check).
* - `dependency` — released when the card's dependencies are satisfied
* (KTD-5: dependency task's column has the `complete`
* trait flag in ITS resolved workflow; FN-5719 dual-accept
* also honors the legacy completion signal, logging an
* audit-diff when the two disagree).
*
* Eligible cards move via `store.moveTask(..., { moveSource: "scheduler" })`.
* A scheduler move bypasses trait guards (it is substrate-driven) but the in-txn
* capacity check is NOT a guard — it still runs (KTD-10), so two holds racing
* into one slot serialize: exactly one commits, the other rejects with
* `capacity-exhausted` and retries next sweep.
*
* Reservation ordering (KTD-10): for releases into a processing (capacity)
* column, the sweep reserves worktree + semaphore slots BEFORE issuing the move
* and releases the reservation if the move rejects on capacity — a card is never
* moved into a column it cannot actually start in, and a semaphore-exhausted
* interleaving leaves the card held with no commit.
*/
import {
isWorkflowColumnsEnabled,
resolveColumnCapacity,
resolveColumnFlags,
resolveColumnAdjacency,
TransitionRejectionError,
BUILTIN_CODING_WORKFLOW_IR,
getBuiltinWorkflow,
isBuiltinWorkflowId,
parseWorkflowIr,
type TaskStore,
type Task,
type Settings,
type WorkflowIr,
type WorkflowIrV2,
type WorkflowIrColumn,
} from "@fusion/core";
import { schedulerLog } from "./logger.js";
const DEFAULT_WORKFLOW_POOL_ID = "__default-workflow__";
/** A reservation handle returned by {@link HoldReleaseDeps.reserveSlot}. The
* sweep calls `release()` if the subsequent move rejects on capacity. */
export interface SlotReservation {
release(): void;
}
/** Injected dependencies so the sweep stays unit-testable with fake timers and
* without real worktree/session allocation. */
export interface HoldReleaseDeps {
/** Monotonic clock (ms). Inject a fake-timer-driven clock in tests; production
* passes `() => Date.now()`. */
now: () => number;
/**
* Reserve a worktree + semaphore slot for a card about to be released into a
* processing column (KTD-10 reservation-first). Returns `null` when no slot
* could be reserved (e.g. semaphore exhausted) — the sweep then leaves the
* card held without issuing a move. Returns a {@link SlotReservation} whose
* `release()` the sweep calls if the move rejects on capacity.
*
* Optional: when absent, releases into processing columns proceed without a
* reservation (the in-txn capacity check still arbitrates), which is the
* default-workflow legacy parity path where the scheduler dispatch loop owns
* worktree allocation via `allocateWorktree`.
*/
reserveSlot?: (task: Task, targetColumn: string) => SlotReservation | null;
/** Allocate a worktree path for a release into a processing column (passed
* through to `moveTask`'s `allocateWorktree`). */
allocateWorktree?: (task: Task, reservedNames: Set<string>) => string | null;
}
/** Outcome of one sweep pass (for tests + observability). */
export interface HoldReleaseResult {
released: string[];
/** taskId → reason it stayed held this pass. */
held: Array<{ taskId: string; reason: string }>;
}
// ── Workflow IR resolution (read-only, mirrors store + merge-trait) ───────────
async function resolveTaskWorkflowIr(store: TaskStore, taskId: string): Promise<WorkflowIr> {
let workflowId: string | undefined;
try {
workflowId = store.getTaskWorkflowSelection(taskId)?.workflowId;
} catch {
workflowId = undefined;
}
if (!workflowId) return BUILTIN_CODING_WORKFLOW_IR;
if (isBuiltinWorkflowId(workflowId)) {
const builtin = getBuiltinWorkflow(workflowId);
return builtin?.ir ?? BUILTIN_CODING_WORKFLOW_IR;
}
try {
const def = await store.getWorkflowDefinition(workflowId);
if (!def) return BUILTIN_CODING_WORKFLOW_IR;
return typeof def.ir === "string" ? parseWorkflowIr(def.ir) : def.ir;
} catch {
return BUILTIN_CODING_WORKFLOW_IR;
}
}
function effectiveWorkflowId(store: TaskStore, taskId: string): string {
try {
return store.getTaskWorkflowSelection(taskId)?.workflowId ?? DEFAULT_WORKFLOW_POOL_ID;
} catch {
return DEFAULT_WORKFLOW_POOL_ID;
}
}
function findColumn(ir: WorkflowIr, columnId: string): WorkflowIrColumn | undefined {
if (ir.version !== "v2") return undefined;
return (ir as WorkflowIrV2).columns.find((c) => c.id === columnId);
}
/** The hold trait config on a column, if any. */
function resolveHoldConfig(column: WorkflowIrColumn): Record<string, unknown> | undefined {
const flags = resolveColumnFlags(column);
if (!flags.hold) return undefined;
const ct = column.traits.find((t) => t.trait === "hold");
return ct?.config ?? {};
}
/** True when the card currently rests at a hold column. */
function isHeldTask(ir: WorkflowIr, task: Task): boolean {
const column = findColumn(ir, task.column);
if (!column) return false;
return resolveColumnFlags(column).hold === true;
}
/**
* Resolve the release target column for a held card.
*
* For `capacity` holds, the target is the nearest downstream column (by the
* workflow's column adjacency, breadth-first from the hold column) that carries
* a capacity (`wip`) trait — for the default workflow this is `in-progress`.
* For other release kinds the target is the first adjacency neighbor that is not
* the hold column itself (the forward step out of the hold).
*/
function resolveReleaseTarget(ir: WorkflowIr, fromColumn: string, preferCapacity: boolean): string | undefined {
const v2 = ir as WorkflowIrV2;
const orderedIds = Array.isArray(v2.columns) ? v2.columns.map((c) => c.id) : [];
const fromIdx = orderedIds.indexOf(fromColumn);
const adjacency = resolveColumnAdjacency(ir);
const neighbors = adjacency.get(fromColumn) ?? [];
if (preferCapacity) {
// Walk FORWARD in declared order for the nearest capacity-bearing column;
// the hold releases downstream, never backward.
for (let i = fromIdx + 1; i < orderedIds.length; i++) {
const col = findColumn(ir, orderedIds[i]);
if (col && resolveColumnFlags(col).countsTowardWip && neighbors.includes(orderedIds[i])) {
return orderedIds[i];
}
}
// No directly-adjacent capacity column: fall back to the nearest forward
// capacity column reachable via adjacency BFS.
const seen = new Set<string>([fromColumn]);
const queue = [...neighbors];
while (queue.length > 0) {
const candidate = queue.shift()!;
if (seen.has(candidate)) continue;
seen.add(candidate);
const col = findColumn(ir, candidate);
if (col && resolveColumnFlags(col).countsTowardWip) return candidate;
for (const next of adjacency.get(candidate) ?? []) {
if (!seen.has(next)) queue.push(next);
}
}
}
// Forward neighbor (declared-order next) if it is adjacent; else any neighbor
// that is forward in declared order; else the first neighbor.
const forwardId = fromIdx >= 0 ? orderedIds[fromIdx + 1] : undefined;
if (forwardId && neighbors.includes(forwardId)) return forwardId;
const forwardNeighbor = neighbors.find((n) => orderedIds.indexOf(n) > fromIdx);
if (forwardNeighbor) return forwardNeighbor;
return neighbors.find((n) => n !== fromColumn);
}
// ── Dependency satisfaction (KTD-5 + FN-5719 dual-accept) ─────────────────────
/** Legacy completion signal: dependency's column is a terminal/handoff column. */
function legacyDependencySatisfied(dep: Task): boolean {
return dep.column === "done" || dep.column === "in-review" || dep.column === "archived";
}
/**
* KTD-5 dependency satisfaction: the dependency task's current column has the
* `complete` trait flag in ITS resolved workflow. Dual-accept (FN-5719): the
* legacy completion signal (done/in-review/archived column, or an accepted
* completion-handoff marker) is also honored; when the two disagree an
* audit-diff event is logged.
*/
async function dependencySatisfied(store: TaskStore, dep: Task): Promise<boolean> {
const ir = await resolveTaskWorkflowIr(store, dep.id);
const column = findColumn(ir, dep.column);
const completeFlag = column ? resolveColumnFlags(column).complete === true : false;
let markerAccepted = false;
try {
markerAccepted = store.getCompletionHandoffAcceptedMarker(dep.id) !== null;
} catch {
markerAccepted = false;
}
const legacy = legacyDependencySatisfied(dep) || markerAccepted;
if (completeFlag !== legacy) {
try {
void store.recordRunAuditEvent?.({
taskId: dep.id,
agentId: "scheduler",
runId: `hold-release:${dep.id}`,
domain: "database",
mutationType: "merge:dependency-parity-diff",
target: dep.id,
metadata: {
depId: dep.id,
completeFlagResult: completeFlag,
legacyResult: legacy,
source: "hold-release.dependency",
},
});
} catch {
// Audit is best-effort.
}
}
// Dual-accept: satisfied if EITHER signal says so (the dual-accept window
// closes at graduation per U12; until then both are accepted).
return completeFlag || legacy;
}
async function allDependenciesSatisfied(store: TaskStore, task: Task, allTasks: Task[]): Promise<boolean> {
for (const depId of task.dependencies ?? []) {
const dep = allTasks.find((t) => t.id === depId);
if (!dep) continue; // missing dep does not block (matches scheduler posture)
if (!(await dependencySatisfied(store, dep))) return false;
}
return true;
}
// ── Timer release ─────────────────────────────────────────────────────────────
/** Resolve the timer deadline (ms epoch) for a timer hold, or `undefined` if not
* resolvable. Supports an explicit `deadlineAt` (ISO or ms) or a relative
* `durationMs`/`timerMs` measured from `columnMovedAt`. */
function resolveTimerDeadline(holdConfig: Record<string, unknown>, task: Task): number | undefined {
const deadlineAt = holdConfig.deadlineAt;
if (typeof deadlineAt === "number" && Number.isFinite(deadlineAt)) return deadlineAt;
if (typeof deadlineAt === "string") {
const parsed = Date.parse(deadlineAt);
if (Number.isFinite(parsed)) return parsed;
}
const duration =
(typeof holdConfig.durationMs === "number" ? holdConfig.durationMs : undefined) ??
(typeof holdConfig.timerMs === "number" ? holdConfig.timerMs : undefined);
if (typeof duration === "number" && Number.isFinite(duration)) {
const base = Date.parse(task.columnMovedAt ?? task.createdAt);
if (Number.isFinite(base)) return base + duration;
}
return undefined;
}
// ── Capacity availability (same counting rule as the in-txn check) ────────────
/**
* Count cards occupying the (workflow, column) capacity slot from a task
* snapshot, mirroring the store's in-txn count: cards in the column now, plus
* (when countPending) cards mid-`transitionPending` targeting it, scoped to the
* SAME effective workflow. This is the sweep's *pre-check* — the authoritative
* arbitration is still the in-txn check, which rejects a losing racer.
*/
function countCapacitySlot(
store: TaskStore,
allTasks: Task[],
targetColumn: string,
workflowId: string,
countPending: boolean,
): number {
let count = 0;
for (const t of allTasks) {
if (effectiveWorkflowId(store, t.id) !== workflowId) continue;
if (t.column === targetColumn) {
count += 1;
continue;
}
if (!countPending) continue;
const tp = (t as Task & { transitionPending?: { toColumn?: string } | null }).transitionPending;
if (tp && typeof tp === "object" && tp.toColumn === targetColumn) count += 1;
}
return count;
}
// ── The sweep ─────────────────────────────────────────────────────────────────
/**
* Run one hold/release sweep pass. No-op (returns empty) when the workflowColumns
* flag is OFF — flag-OFF scheduler behavior is byte-identical (the legacy
* pull-from-todo loop is untouched).
*/
export async function runHoldReleaseSweep(
store: TaskStore,
deps: HoldReleaseDeps,
): Promise<HoldReleaseResult> {
const result: HoldReleaseResult = { released: [], held: [] };
const settings = await store.getSettings();
if (!isWorkflowColumnsEnabled(settings)) return result;
const allTasks = await store.listTasks({ includeArchived: false });
for (const task of allTasks) {
// Skip paused / recovery-backoff tasks exactly as the legacy scheduler does.
if (task.paused || task.userPaused) {
continue;
}
if (task.nextRecoveryAt && Date.parse(task.nextRecoveryAt) > deps.now()) {
continue;
}
const ir = await resolveTaskWorkflowIr(store, task.id);
if (!isHeldTask(ir, task)) continue;
const column = findColumn(ir, task.column);
const holdConfig = column ? resolveHoldConfig(column) : undefined;
if (!column || !holdConfig) continue;
const release = typeof holdConfig.release === "string" ? holdConfig.release : "manual";
// manual / external-event are NEVER auto-released by the sweep.
if (release === "manual" || release === "external-event") {
result.held.push({ taskId: task.id, reason: `${release}-only` });
continue;
}
let shouldRelease = false;
if (release === "timer") {
const deadline = resolveTimerDeadline(holdConfig, task);
shouldRelease = deadline !== undefined && deps.now() >= deadline;
if (!shouldRelease) {
result.held.push({ taskId: task.id, reason: "timer-not-elapsed" });
continue;
}
} else if (release === "dependency") {
shouldRelease = await allDependenciesSatisfied(store, task, allTasks);
if (!shouldRelease) {
result.held.push({ taskId: task.id, reason: "deps-unsatisfied" });
continue;
}
} else if (release === "capacity") {
// Capacity holds release into the nearest downstream capacity column when a
// slot is free (pre-check); the in-txn check is the authority.
const target = resolveReleaseTarget(ir, task.column, true);
if (!target) {
result.held.push({ taskId: task.id, reason: "no-downstream-capacity-column" });
continue;
}
const capacity = resolveColumnCapacity(ir, target, settings);
if (capacity.hasCapacity && Number.isFinite(capacity.limit)) {
const workflowId = effectiveWorkflowId(store, task.id);
const occupants = countCapacitySlot(store, allTasks, target, workflowId, capacity.countPending);
if (occupants >= capacity.limit) {
result.held.push({ taskId: task.id, reason: "downstream-full" });
continue;
}
}
shouldRelease = true;
}
if (!shouldRelease) continue;
const target = resolveReleaseTarget(ir, task.column, release === "capacity");
if (!target) {
result.held.push({ taskId: task.id, reason: "no-release-target" });
continue;
}
const released = await issueRelease(store, deps, task, target, ir);
if (released) {
result.released.push(task.id);
} else {
result.held.push({ taskId: task.id, reason: "move-rejected-or-no-slot" });
}
}
return result;
}
/**
* Issue a single release move (`moveSource: "scheduler"`). For releases into a
* processing (capacity) column the reservation-first ordering (KTD-10) reserves
* worktree + semaphore before the move and releases the reservation if the move
* rejects on capacity. Returns true on a committed move, false otherwise (the
* card stays held).
*/
async function issueRelease(
store: TaskStore,
deps: HoldReleaseDeps,
task: Task,
target: string,
ir: WorkflowIr,
): Promise<boolean> {
const targetColumn = findColumn(ir, target);
const targetIsProcessing = targetColumn ? resolveColumnFlags(targetColumn).countsTowardWip === true : false;
let reservation: SlotReservation | null = null;
if (targetIsProcessing && deps.reserveSlot) {
reservation = deps.reserveSlot(task, target);
if (!reservation) {
// Semaphore/worktree exhausted — reservation-first means no move at all.
schedulerLog.log(`Hold release for ${task.id} deferred — no reservable slot for ${target}`);
return false;
}
}
try {
await store.moveTask(task.id, target, {
moveSource: "scheduler",
allocateWorktree:
targetIsProcessing && deps.allocateWorktree
? (reservedNames) => deps.allocateWorktree!(task, reservedNames)
: undefined,
});
return true;
} catch (error) {
if (error instanceof TransitionRejectionError && error.rejection.code === "capacity-exhausted") {
// Lost the in-txn race for the slot — release the reservation, stay held.
reservation?.release();
schedulerLog.log(`Hold release for ${task.id} rejected on capacity for ${target} — staying held`);
return false;
}
// Any other failure: release the reservation and let the card stay held.
reservation?.release();
schedulerLog.warn(
`Hold release for ${task.id} into ${target} failed: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
// ── Explicit (manual / external-event) releases ───────────────────────────────
/**
* Manually promote a held card out of its hold column (U9's promote endpoint /
* CLI calls this). Releases regardless of the hold's release kind — a manual
* promote is the explicit operator action the `manual` release kind waits for,
* and it is also accepted for other kinds as an operator override. The move
* still serializes through the in-txn capacity check (KTD-10): a promote into a
* full column rejects with `capacity-exhausted`, surfaced to the caller.
*/
export async function promoteHeldTask(
store: TaskStore,
taskId: string,
deps: Pick<HoldReleaseDeps, "reserveSlot" | "allocateWorktree"> = {},
): Promise<{ released: boolean; toColumn?: string; rejection?: string }> {
const task = await store.getTask(taskId);
if (!task) return { released: false, rejection: "task-not-found" };
const ir = await resolveTaskWorkflowIr(store, taskId);
if (!isHeldTask(ir, task)) {
return { released: false, rejection: "not-held" };
}
const target = resolveReleaseTarget(ir, task.column, true);
if (!target) return { released: false, rejection: "no-release-target" };
const released = await issueRelease(
store,
{ now: () => Date.now(), reserveSlot: deps.reserveSlot, allocateWorktree: deps.allocateWorktree },
task,
target,
ir,
);
return released ? { released: true, toColumn: target } : { released: false, rejection: "capacity-exhausted-or-no-slot" };
}
/**
* Release a held card on an external event (webhook/API). Same shape as
* {@link promoteHeldTask} plus an `eventTag` recorded in the audit; only acts on
* `external-event` holds (a no-op otherwise so a stray webhook can't release a
* manual/timer/capacity hold).
*/
export async function releaseHeldTaskByEvent(
store: TaskStore,
taskId: string,
eventTag: string,
deps: Pick<HoldReleaseDeps, "reserveSlot" | "allocateWorktree"> = {},
): Promise<{ released: boolean; toColumn?: string; rejection?: string }> {
const task = await store.getTask(taskId);
if (!task) return { released: false, rejection: "task-not-found" };
const ir = await resolveTaskWorkflowIr(store, taskId);
const column = findColumn(ir, task.column);
const holdConfig = column ? resolveHoldConfig(column) : undefined;
if (!column || !holdConfig || holdConfig.release !== "external-event") {
return { released: false, rejection: "not-external-event-hold" };
}
try {
void store.recordRunAuditEvent?.({
taskId,
agentId: "scheduler",
runId: `hold-release:event:${taskId}`,
domain: "database",
mutationType: "task:hold-release-event",
target: taskId,
metadata: { eventTag, fromColumn: task.column },
});
} catch {
// best-effort
}
const target = resolveReleaseTarget(ir, task.column, true);
if (!target) return { released: false, rejection: "no-release-target" };
const released = await issueRelease(
store,
{ now: () => Date.now(), reserveSlot: deps.reserveSlot, allocateWorktree: deps.allocateWorktree },
task,
target,
ir,
);
return released ? { released: true, toColumn: target } : { released: false, rejection: "capacity-exhausted-or-no-slot" };
}

View File

@@ -32,6 +32,8 @@ import type { AutoClaimSnapshotManager } from "./auto-claim-snapshot.js";
import { StaleTaskReporter } from "./stale-task-reporter.js";
import { BacklogPressureReporter } from "./backlog-pressure-reporter.js";
import { createRunAuditor, generateSyntheticRunId } from "./run-audit.js";
import { isWorkflowColumnsEnabled } from "@fusion/core";
import { runHoldReleaseSweep, type SlotReservation } from "./hold-release.js";
/**
* Check whether two sets of file scope paths overlap.
@@ -278,6 +280,20 @@ interface ConcurrencyGateSnapshot {
slack: number;
}
/**
* U6 (KTD-10): a per-(workflow, column) capacity gate, the generalization of the
* three legacy gates to workflow-defined WIP columns. Additive — the three-gate
* report shape (maxConcurrent/maxWorktrees/semaphore) is preserved verbatim; this
* is an optional extra field populated only when the workflowColumns flag is ON.
*/
interface PerColumnCapacityGate {
workflowId: string;
columnId: string;
used: number;
limit: number;
slack: number;
}
interface ConcurrencyGateDiagnostic {
available: number;
bindingGates: ConcurrencyGateName[];
@@ -289,6 +305,9 @@ interface ConcurrencyGateDiagnostic {
maxWorktrees: string[];
semaphore?: string[];
};
/** U6: additive per-column capacity gates (flag-ON only; omitted otherwise so
* the legacy three-gate report shape is byte-identical when the flag is OFF). */
perColumnGates?: PerColumnCapacityGate[];
}
function computeConcurrencyGateDiagnostic(params: {
@@ -299,6 +318,9 @@ function computeConcurrencyGateDiagnostic(params: {
semaphore?: AgentSemaphore;
inProgressTaskIds: string[];
available: number;
/** U6: additive per-column capacity gates (flag-ON only). Omitted → the legacy
* three-gate report is byte-identical. */
perColumnGates?: PerColumnCapacityGate[];
}): ConcurrencyGateDiagnostic {
const maxConcurrentGate: ConcurrencyGateSnapshot = {
used: params.agentSlots,
@@ -334,6 +356,8 @@ function computeConcurrencyGateDiagnostic(params: {
maxWorktrees: [...params.inProgressTaskIds],
semaphore: semaphoreGate ? [...params.inProgressTaskIds] : undefined,
},
// U6: additive only — present when flag-ON, omitted otherwise.
...(params.perColumnGates ? { perColumnGates: params.perColumnGates } : {}),
};
}
@@ -938,6 +962,8 @@ export class Scheduler {
semaphore: diagnostic.semaphoreGate,
holders: diagnostic.holders,
available: diagnostic.available,
// U6: additive per-column capacity gates (present only flag-ON).
...(diagnostic.perColumnGates ? { perColumnGates: diagnostic.perColumnGates } : {}),
},
});
} catch (error) {
@@ -1171,6 +1197,18 @@ export class Scheduler {
}
this.wasEnginePaused = false;
// ── U6: hold/release sweep (flag-ON only) ──────────────────────────────
// Flag OFF: this is skipped entirely — the legacy pull-from-todo loop
// below is byte-identical. Flag ON: the sweep evaluates hold-column
// release conditions (manual/timer/capacity/dependency/external-event) and
// releases eligible cards via moveSource:"scheduler", serializing through
// the in-txn capacity check. For the DEFAULT workflow the legacy loop below
// still drives todo→in-progress pickup (parity); the sweep adds custom-
// workflow hold handling and the generalized capacity-release path.
if (isWorkflowColumnsEnabled(settings)) {
await this.runHoldReleaseSweepPass();
}
// Count only in-progress tasks toward the worktree limit.
// In-review tasks with worktrees are idle (waiting to merge) and
// should not block new tasks from starting.
@@ -1207,6 +1245,19 @@ export class Scheduler {
semaphoreAvailable,
);
const inProgressTaskIds = inProgress.map((task) => task.id);
// U6 (KTD-10): when the workflowColumns flag is ON, report the default
// workflow's in-progress capacity as a per-column gate — the generalization
// of the legacy maxConcurrent gate (which reads through to the same value).
// Additive: omitted flag-OFF so the three-gate report shape is unchanged.
const perColumnGates = isWorkflowColumnsEnabled(settings)
? [{
workflowId: "__default-workflow__",
columnId: "in-progress",
used: agentSlots,
limit: maxConcurrent,
slack: maxConcurrent - agentSlots,
}]
: undefined;
const concurrencyGateDiagnostic = computeConcurrencyGateDiagnostic({
agentSlots,
maxConcurrent,
@@ -1215,6 +1266,7 @@ export class Scheduler {
semaphore: this.options.semaphore,
inProgressTaskIds,
available,
perColumnGates,
});
if (available <= 0) return;
@@ -1892,6 +1944,37 @@ export class Scheduler {
}
}
/**
* U6: run one hold/release sweep pass, wiring the scheduler's semaphore +
* worktree allocation into the reservation-first ordering (KTD-10). Failures
* are isolated so a sweep error never breaks the scheduling pass.
*/
private async runHoldReleaseSweepPass(): Promise<void> {
try {
await runHoldReleaseSweep(this.store, {
now: () => Date.now(),
reserveSlot: this.options.semaphore
? (): SlotReservation | null => {
const sem = this.options.semaphore!;
if (!sem.tryAcquire()) return null;
let released = false;
return {
release: () => {
if (released) return;
released = true;
sem.release();
},
};
}
: undefined,
allocateWorktree: (task, reservedNames) =>
planTaskWorktreePath(task, this.store.getRootDir(), undefined, reservedNames, {}),
});
} catch (error) {
schedulerLog.error("Hold/release sweep failed:", error);
}
}
/**
* Handle a mission-linked task column move.
* Keeps feature state synchronized with task columns across the full task