feat(FN-2862): merge fusion/fn-2862

packages/engine/src/__tests__/gridlock-detector.test.ts

@@ -0,0 +1,194 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import type { Settings, Task, TaskStore } from "@fusion/core";
+import { GridlockDetector } from "../gridlock-detector.js";
+
+function createTask(id: string, overrides: Partial<Task> = {}): Task {
+  return {
+    id,
+    title: id,
+    description: "desc",
+    column: "todo",
+    dependencies: [],
+    steps: [],
+    currentStep: 0,
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    log: [],
+    ...overrides,
+  };
+}
+
+function createSettings(overrides: Partial<Settings> = {}): Settings {
+  return {
+    maxConcurrent: 2,
+    maxWorktrees: 4,
+    pollIntervalMs: 15000,
+    groupOverlappingFiles: true,
+    autoMerge: true,
+    overlapIgnorePaths: [],
+    ...overrides,
+  } as Settings;
+}
+
+describe("GridlockDetector", () => {
+  let tasks: Task[];
+  let settings: Settings;
+  let scopes: Record<string, string[]>;
+  let onGridlock: ReturnType<typeof vi.fn>;
+  let store: TaskStore;
+  let detector: GridlockDetector;
+
+  beforeEach(() => {
+    tasks = [];
+    settings = createSettings();
+    scopes = {};
+    onGridlock = vi.fn();
+    store = {
+      listTasks: vi.fn(async () => tasks),
+      getSettings: vi.fn(async () => settings),
+      parseFileScopeFromPrompt: vi.fn(async (taskId: string) => scopes[taskId] ?? []),
+    } as unknown as TaskStore;
+    detector = new GridlockDetector(store, { onGridlock });
+  });
+
+  afterEach(() => {
+    detector.stop();
+  });
+
+  it("detects gridlock when all todo tasks are blocked by dependencies", async () => {
+    tasks = [
+      createTask("FN-1", { column: "todo", dependencies: ["FN-10"] }),
+      createTask("FN-2", { column: "todo", dependencies: ["FN-11"] }),
+      createTask("FN-3", { column: "in-progress" }),
+      createTask("FN-10", { column: "in-progress" }),
+      createTask("FN-11", { column: "in-progress" }),
+    ];
+
+    const event = await detector.detectGridlock();
+
+    expect(event).not.toBeNull();
+    expect(event?.blockedTaskIds).toEqual(["FN-1", "FN-2"]);
+    expect(event?.reasons).toEqual({ "FN-1": "dependency", "FN-2": "dependency" });
+    expect(event?.blockingTaskIds).toEqual(["FN-10", "FN-11"]);
+    expect(onGridlock).toHaveBeenCalledTimes(1);
+  });
+
+  it("detects gridlock when all todo tasks are blocked by file overlap", async () => {
+    tasks = [
+      createTask("FN-1", { column: "todo" }),
+      createTask("FN-2", { column: "todo" }),
+      createTask("FN-9", { column: "in-progress" }),
+    ];
+    scopes = {
+      "FN-1": ["packages/core/src/a.ts"],
+      "FN-2": ["packages/core/src/b.ts"],
+      "FN-9": ["packages/core/src/*"],
+    };
+
+    const event = await detector.detectGridlock();
+
+    expect(event?.blockedTaskIds).toEqual(["FN-1", "FN-2"]);
+    expect(event?.reasons).toEqual({ "FN-1": "overlap", "FN-2": "overlap" });
+    expect(event?.blockingTaskIds).toEqual(["FN-9"]);
+  });
+
+  it("does not detect gridlock when there are no schedulable tasks", async () => {
+    tasks = [createTask("FN-1", { column: "todo", paused: true }), createTask("FN-2", { column: "in-progress" })];
+
+    const event = await detector.detectGridlock();
+
+    expect(event).toBeNull();
+    expect(onGridlock).not.toHaveBeenCalled();
+  });
+
+  it("does not detect gridlock when at least one todo task is unblocked", async () => {
+    tasks = [
+      createTask("FN-1", { column: "todo", dependencies: ["FN-10"] }),
+      createTask("FN-2", { column: "todo" }),
+      createTask("FN-3", { column: "in-progress" }),
+      createTask("FN-10", { column: "todo" }),
+    ];
+
+    const event = await detector.detectGridlock();
+    expect(event).toBeNull();
+  });
+
+  it("deduplicates same blocked task set", async () => {
+    tasks = [
+      createTask("FN-1", { column: "todo", dependencies: ["FN-10"] }),
+      createTask("FN-2", { column: "in-progress" }),
+      createTask("FN-10", { column: "in-progress" }),
+    ];
+
+    await detector.detectGridlock();
+    await detector.detectGridlock();
+
+    expect(onGridlock).toHaveBeenCalledTimes(1);
+  });
+
+  it("fires again when blocked task set changes", async () => {
+    tasks = [
+      createTask("FN-1", { column: "todo", dependencies: ["FN-10"] }),
+      createTask("FN-2", { column: "in-progress" }),
+      createTask("FN-10", { column: "in-progress" }),
+    ];
+
+    await detector.detectGridlock();
+    tasks = [
+      ...tasks,
+      createTask("FN-3", { column: "todo", dependencies: ["FN-10"] }),
+    ];
+    await detector.detectGridlock();
+
+    expect(onGridlock).toHaveBeenCalledTimes(2);
+  });
+
+  it("resets dedup after resolution", async () => {
+    tasks = [
+      createTask("FN-1", { column: "todo", dependencies: ["FN-10"] }),
+      createTask("FN-2", { column: "in-progress" }),
+      createTask("FN-10", { column: "in-progress" }),
+    ];
+
+    await detector.detectGridlock();
+    tasks = [createTask("FN-1", { column: "todo" }), createTask("FN-2", { column: "in-progress" }), createTask("FN-10", { column: "done" })];
+    await detector.detectGridlock();
+
+    tasks = [
+      createTask("FN-1", { column: "todo", dependencies: ["FN-10"] }),
+      createTask("FN-2", { column: "in-progress" }),
+      createTask("FN-10", { column: "in-progress" }),
+    ];
+    await detector.detectGridlock();
+
+    expect(onGridlock).toHaveBeenCalledTimes(2);
+  });
+
+  it("respects paused and recovery-backoff tasks as non-schedulable", async () => {
+    const future = new Date(Date.now() + 60_000).toISOString();
+    tasks = [
+      createTask("FN-1", { column: "todo", paused: true, dependencies: ["FN-9"] }),
+      createTask("FN-2", { column: "todo", nextRecoveryAt: future, dependencies: ["FN-9"] }),
+      createTask("FN-3", { column: "in-progress" }),
+      createTask("FN-9", { column: "todo" }),
+    ];
+
+    const event = await detector.detectGridlock();
+    expect(event).toBeNull();
+  });
+
+  it("respects overlap ignore paths from settings", async () => {
+    settings = createSettings({ overlapIgnorePaths: ["docs/"] });
+    tasks = [
+      createTask("FN-1", { column: "todo" }),
+      createTask("FN-2", { column: "in-progress" }),
+    ];
+    scopes = {
+      "FN-1": ["docs/readme.md"],
+      "FN-2": ["docs/"],
+    };
+
+    const event = await detector.detectGridlock();
+    expect(event).toBeNull();
+  });
+});

packages/engine/src/__tests__/notifier.test.ts

@@ -66,6 +66,7 @@ describe("Ntfy notifier helpers", () => {
   it("includes planning-awaiting-input in default events", () => {
     expect(DEFAULT_NTFY_EVENTS).toContain("planning-awaiting-input");
     expect(resolveNtfyEvents(undefined)).toContain("planning-awaiting-input");
+    expect(DEFAULT_NTFY_EVENTS).toContain("gridlock");
   });
 
   it("checks planning-awaiting-input event enablement", () => {
@@ -139,6 +140,75 @@ describe("NtfyNotifier", () => {
     });
   });
 
+  describe("gridlock notifications", () => {
+    it("sends notification when gridlock event is enabled", async () => {
+      store.setSettings({ ntfyEnabled: true, ntfyTopic: "test-topic", ntfyEvents: ["gridlock"] });
+      fetchMock.mockResolvedValue({ ok: true });
+      notifier = new NtfyNotifier(store, { projectId: "proj-1" });
+      await notifier.start();
+
+      notifier.notifyGridlock({
+        blockedTaskCount: 2,
+        reasons: { "FN-001": "dependency", "FN-003": "overlap" },
+        blockedTaskIds: ["FN-001", "FN-003"],
+        blockingTaskIds: ["FN-002"],
+      });
+
+      await flushAsyncWork();
+      expect(fetchMock).toHaveBeenCalledTimes(1);
+      expect(fetchMock).toHaveBeenCalledWith(
+        "https://ntfy.sh/test-topic",
+        expect.objectContaining({
+          method: "POST",
+          headers: expect.objectContaining({
+            Title: "Pipeline gridlocked",
+            Priority: "high",
+          }),
+        }),
+      );
+    });
+
+    it("skips notification when gridlock event is disabled", async () => {
+      store.setSettings({ ntfyEnabled: true, ntfyTopic: "test-topic", ntfyEvents: ["failed"] });
+      fetchMock.mockResolvedValue({ ok: true });
+      notifier = new NtfyNotifier(store);
+      await notifier.start();
+
+      notifier.notifyGridlock({
+        blockedTaskCount: 1,
+        reasons: { "FN-001": "dependency" },
+        blockedTaskIds: ["FN-001"],
+        blockingTaskIds: ["FN-002"],
+      });
+
+      await flushAsyncWork();
+      expect(fetchMock).not.toHaveBeenCalled();
+    });
+
+    it("deduplicates by blocked task set", async () => {
+      store.setSettings({ ntfyEnabled: true, ntfyTopic: "test-topic", ntfyEvents: ["gridlock"] });
+      fetchMock.mockResolvedValue({ ok: true });
+      notifier = new NtfyNotifier(store);
+      await notifier.start();
+
+      notifier.notifyGridlock({
+        blockedTaskCount: 2,
+        reasons: { "FN-001": "dependency", "FN-003": "dependency" },
+        blockedTaskIds: ["FN-003", "FN-001"],
+        blockingTaskIds: ["FN-002"],
+      });
+      notifier.notifyGridlock({
+        blockedTaskCount: 2,
+        reasons: { "FN-001": "dependency", "FN-003": "dependency" },
+        blockedTaskIds: ["FN-001", "FN-003"],
+        blockingTaskIds: ["FN-002"],
+      });
+
+      await flushAsyncWork();
+      expect(fetchMock).toHaveBeenCalledTimes(1);
+    });
+  });
+
   describe("when enabled", () => {
     beforeEach(() => {
       store.setSettings({ ntfyEnabled: true, ntfyTopic: "test-topic" });

packages/engine/src/gridlock-detector.ts

@@ -0,0 +1,155 @@
+import type { MissionStore, Task, TaskStore } from "@fusion/core";
+import { createLogger } from "./logger.js";
+import { filterPathsByIgnoreList, pathsOverlap } from "./scheduler.js";
+
+const gridlockLog = createLogger("gridlock-detector");
+
+export interface GridlockEvent {
+  blockedTaskCount: number;
+  reasons: Record<string, "dependency" | "overlap">;
+  blockedTaskIds: string[];
+  blockingTaskIds: string[];
+}
+
+export interface GridlockDetectorOptions {
+  pollIntervalMs?: number;
+  missionStore?: MissionStore;
+  onGridlock?: (event: GridlockEvent) => void;
+}
+
+export class GridlockDetector {
+  private interval: ReturnType<typeof setInterval> | null = null;
+  private readonly pollIntervalMs: number;
+  private readonly missionStore?: MissionStore;
+  private readonly onGridlock?: (event: GridlockEvent) => void;
+  private lastGridlockKey: string | null = null;
+
+  constructor(
+    private readonly store: TaskStore,
+    options: GridlockDetectorOptions = {},
+  ) {
+    this.pollIntervalMs = options.pollIntervalMs ?? 30_000;
+    this.missionStore = options.missionStore;
+    this.onGridlock = options.onGridlock;
+  }
+
+  start(): void {
+    if (this.interval) return;
+    this.interval = setInterval(() => {
+      this.detectGridlock().catch((error) => {
+        gridlockLog.error("Failed gridlock detection cycle:", error);
+      });
+    }, this.pollIntervalMs);
+    gridlockLog.log(`Started (poll interval: ${this.pollIntervalMs}ms)`);
+  }
+
+  stop(): void {
+    if (!this.interval) return;
+    clearInterval(this.interval);
+    this.interval = null;
+    gridlockLog.log("Stopped");
+  }
+
+  async detectGridlock(): Promise<GridlockEvent | null> {
+    const [tasks, settings] = await Promise.all([
+      this.store.listTasks({ slim: true, includeArchived: false }),
+      this.store.getSettings(),
+    ]);
+
+    const now = Date.now();
+    const schedulable = tasks.filter((task) => {
+      if (task.column !== "todo" || task.paused) return false;
+      if (task.nextRecoveryAt && new Date(task.nextRecoveryAt).getTime() > now) return false;
+      if (this.isMissionBlocked(task)) return false;
+      return true;
+    });
+
+    if (schedulable.length === 0) {
+      this.lastGridlockKey = null;
+      return null;
+    }
+
+    const active = tasks.filter((task) => task.column === "in-progress" || (task.column === "in-review" && Boolean(task.worktree)));
+    if (active.length === 0) {
+      this.lastGridlockKey = null;
+      return null;
+    }
+
+    const overlapIgnorePaths = settings.overlapIgnorePaths ?? [];
+    const activeScopes = new Map<string, string[]>();
+    if (settings.groupOverlappingFiles) {
+      for (const task of active) {
+        const scope = filterPathsByIgnoreList(await this.store.parseFileScopeFromPrompt(task.id), overlapIgnorePaths);
+        if (scope.length > 0) {
+          activeScopes.set(task.id, scope);
+        }
+      }
+    }
+
+    const reasons: Record<string, "dependency" | "overlap"> = {};
+    const blockingTaskIds = new Set<string>();
+
+    for (const task of schedulable) {
+      const unmetDeps = task.dependencies.filter((depId) => {
+        const dep = tasks.find((candidate) => candidate.id === depId);
+        return dep && dep.column !== "done" && dep.column !== "in-review" && dep.column !== "archived";
+      });
+
+      if (unmetDeps.length > 0) {
+        reasons[task.id] = "dependency";
+        for (const depId of unmetDeps) blockingTaskIds.add(depId);
+        continue;
+      }
+
+      if (!settings.groupOverlappingFiles) continue;
+
+      const taskScope = filterPathsByIgnoreList(await this.store.parseFileScopeFromPrompt(task.id), overlapIgnorePaths);
+      if (taskScope.length === 0) continue;
+
+      for (const [activeId, activeScope] of activeScopes) {
+        if (pathsOverlap(taskScope, activeScope)) {
+          reasons[task.id] = "overlap";
+          blockingTaskIds.add(activeId);
+          break;
+        }
+      }
+    }
+
+    const blockedTaskIds = Object.keys(reasons).sort();
+    if (blockedTaskIds.length !== schedulable.length) {
+      this.lastGridlockKey = null;
+      return null;
+    }
+
+    const gridlockKey = blockedTaskIds.join(",");
+    const event: GridlockEvent = {
+      blockedTaskCount: blockedTaskIds.length,
+      reasons,
+      blockedTaskIds,
+      blockingTaskIds: Array.from(blockingTaskIds).sort(),
+    };
+
+    if (this.lastGridlockKey !== gridlockKey) {
+      this.lastGridlockKey = gridlockKey;
+      gridlockLog.warn(`Gridlock detected: blocked=${event.blockedTaskIds.join(",")}; blocking=${event.blockingTaskIds.join(",")}`);
+      this.onGridlock?.(event);
+    }
+
+    return event;
+  }
+
+  private isMissionBlocked(task: Task): boolean {
+    if (!this.missionStore || !task.sliceId) return false;
+    try {
+      const slice = this.missionStore.getSlice(task.sliceId);
+      if (!slice) return false;
+      const milestone = this.missionStore.getMilestone(slice.milestoneId);
+      if (!milestone) return false;
+      const mission = this.missionStore.getMission(milestone.missionId);
+      return mission?.status === "blocked";
+    } catch (error) {
+      gridlockLog.warn(`Mission lookup failed for ${task.id}:`, error);
+      return false;
+    }
+  }
+}

packages/engine/src/notifier.ts

@@ -1,4 +1,5 @@
 import type { Task, Column, Settings, MergeResult, NtfyNotificationEvent } from "@fusion/core";
+import type { GridlockEvent } from "./gridlock-detector.js";
 import { schedulerLog } from "./logger.js";
 
 export interface NtfyNotifierOptions {
@@ -19,6 +20,7 @@ export const DEFAULT_NTFY_EVENTS: readonly NtfyNotificationEvent[] = [
   "awaiting-approval",
   "awaiting-user-review",
   "planning-awaiting-input",
+  "gridlock",
 ] as const;
 
 export interface NtfyNotificationConfigInput {
@@ -49,6 +51,7 @@ interface NtfyConfig {
 
 /** Event types for task notification deduplication */
 type TaskNotificationEvent = "in-review" | "merged" | "failed" | "awaiting-approval" | "awaiting-user-review";
+type AnyNotificationEvent = TaskNotificationEvent | "gridlock";
 
 /**
  * Format a task identifier for notifications.
@@ -359,7 +362,39 @@ export class NtfyNotifier {
     this.ntfyBaseUrl = resolveNtfyBaseUrl(settings.ntfyBaseUrl, this.defaultNtfyBaseUrl);
   }
 
-  private isEventEnabled(event: TaskNotificationEvent): boolean {
+  notifyGridlock(event: GridlockEvent): void {
+    if (!this.config.enabled || !this.config.topic || !this.isEventEnabled("gridlock")) return;
+
+    const blockedTasks = event.blockedTaskIds.sort();
+    const reasonSummary = Object.values(event.reasons).reduce((acc, reason) => {
+      acc[reason] = (acc[reason] ?? 0) + 1;
+      return acc;
+    }, {} as Record<"dependency" | "overlap", number>);
+
+    const reasons: string[] = [];
+    if (reasonSummary.dependency) reasons.push(`${reasonSummary.dependency} dependency`);
+    if (reasonSummary.overlap) reasons.push(`${reasonSummary.overlap} overlap`);
+
+    const clickUrl = buildNtfyClickUrl({
+      dashboardHost: this.config.dashboardHost,
+      projectId: this.projectId,
+    });
+
+    const dedupKey = `gridlock:${blockedTasks.join(",")}`;
+    this.maybeNotifyByKey(dedupKey, () =>
+      sendNtfyNotification({
+        ntfyBaseUrl: this.ntfyBaseUrl,
+        topic: this.config.topic!,
+        title: "Pipeline gridlocked",
+        message: `${event.blockedTaskCount} todo tasks are blocked (${reasons.join(", ")}). Blocked: ${blockedTasks.join(", ")}. Blocking: ${event.blockingTaskIds.join(", ") || "none"}.`,
+        priority: "high",
+        clickUrl,
+        signal: this.abortController?.signal,
+      }),
+    );
+  }
+
+  private isEventEnabled(event: AnyNotificationEvent): boolean {
     return isNtfyEventEnabled(this.config.events, event);
   }
 
@@ -368,8 +403,10 @@ export class NtfyNotifier {
     eventType: TaskNotificationEvent,
     notifyFn: () => Promise<void>,
   ): void {
-    const key = `${taskId}:${eventType}`;
+    this.maybeNotifyByKey(`${taskId}:${eventType}`, notifyFn);
+  }
 
+  private maybeNotifyByKey(key: string, notifyFn: () => Promise<void>): void {
     if (this.notifiedEvents.has(key)) {
       return;
     }

packages/engine/src/project-engine.ts

@@ -16,6 +16,7 @@ import type { ProjectRuntimeConfig } from "./project-runtime.js";
 import { PrMonitor } from "./pr-monitor.js";
 import { PrCommentHandler } from "./pr-comment-handler.js";
 import { NtfyNotifier } from "./notifier.js";
+import { GridlockDetector } from "./gridlock-detector.js";
 import { CronRunner, createAiPromptExecutor } from "./cron-runner.js";
 import type { RoutineRunner } from "./routine-runner.js";
 import { aiMergeTask } from "./merger.js";
@@ -127,6 +128,7 @@ export class ProjectEngine {
   private prMonitor?: PrMonitor;
   private prCommentHandler?: PrCommentHandler;
   private notifier?: NtfyNotifier;
+  private gridlockDetector?: GridlockDetector;
   private cronRunner?: CronRunner;
   private automationStore?: AutomationStoreType;
   private remoteTunnelManager?: TunnelProcessManager;
@@ -225,6 +227,11 @@ export class ProjectEngine {
       await this.notifier.start();
     }
 
+    this.gridlockDetector = new GridlockDetector(store, {
+      onGridlock: (event) => this.notifier?.notifyGridlock(event),
+    });
+    this.gridlockDetector.start();
+
     // 4. Initialize AutomationStore + CronRunner
     this.setAutomationSubsystemHealth(
       "initializing",
@@ -388,6 +395,7 @@ export class ProjectEngine {
 
     // Stop auxiliary subsystems
     this.notifier?.stop();
+    this.gridlockDetector?.stop();
     this.cronRunner?.stop();
     this.setAutomationSubsystemHealth("not-initialized", "Automation subsystem stopped");