feat(FN-3019): merge fusion/fn-3019

Commits merged:
- feat(FN-3019): complete Step 4 — document gridlock cooldown semantics
- fix(FN-3019): normalize legacy custom provider payloads for typecheck
- test(FN-3019): complete Step 2 — add gridlock cooldown regression coverage
- fix(FN-3019): repair notifier class structure after cooldown refactor
- feat(FN-3019): complete Step 1 — add gridlock notification cooldown

Files changed:
docs/architecture.md                               |  3 +-
 docs/settings-reference.md                         |  4 +-
 .../app/components/CustomProvidersSection.tsx      | 25 ++++++----
 .../engine/src/__tests__/gridlock-detector.test.ts |  5 +-
 packages/engine/src/__tests__/notifier.test.ts     | 47 ++++++++++++++++--
 packages/engine/src/gridlock-detector.ts           | 16 +++++--
 packages/engine/src/notifier.ts                    | 55 ++++++++++++----------
 packages/engine/src/project-engine.ts              |  1 +
 8 files changed, 109 insertions(+), 47 deletions(-)

Fusion-Task-Id: FN-3019
This commit is contained in:
Fusion
2026-04-30 00:18:41 -07:00
committed by gsxdsm
parent e100f2ab7f
commit 115ea77d12
8 changed files with 110 additions and 48 deletions

View File

@@ -326,7 +326,7 @@ See [Memory Plugin Contract](./memory-plugin-contract.md) for the full plan.
- `AgentSemaphore` (`concurrency.ts`) — slot acquisition - `AgentSemaphore` (`concurrency.ts`) — slot acquisition
- `RecoveryPolicy` (`recovery-policy.ts`) — retry/recovery decision policy - `RecoveryPolicy` (`recovery-policy.ts`) — retry/recovery decision policy
- `StuckTaskDetector` (`stuck-task-detector.ts`) — inactivity/loop stall detection - `StuckTaskDetector` (`stuck-task-detector.ts`) — inactivity/loop stall detection
- `GridlockDetector` (`gridlock-detector.ts`) — detects all-blocked todo pipelines and emits notification events - `GridlockDetector` (`gridlock-detector.ts`) — detects all-blocked todo pipelines and emits notification events (plus explicit clear signals when gridlock resolves)
- `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification - `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification
- `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions
- `UsageLimitPauser` (`usage-limit-detector.ts`) and `withRateLimitRetry` (`rate-limit-retry.ts`) - `UsageLimitPauser` (`usage-limit-detector.ts`) and `withRateLimitRetry` (`rate-limit-retry.ts`)
@@ -341,6 +341,7 @@ See [Memory Plugin Contract](./memory-plugin-contract.md) for the full plan.
- `Notifier` (`notifier.ts`) — legacy ntfy compatibility shim (`NtfyNotifier`) plus shared ntfy helpers - `Notifier` (`notifier.ts`) — legacy ntfy compatibility shim (`NtfyNotifier`) plus shared ntfy helpers
- Runtime ownership: `NtfyNotifier` no longer owns an independent task-lifecycle listener graph; `ProjectEngine` injects the canonical `NotificationService` instance so task lifecycle notifications (`task:moved`, `task:updated`, `task:merged`) are emitted through a single path. - Runtime ownership: `NtfyNotifier` no longer owns an independent task-lifecycle listener graph; `ProjectEngine` injects the canonical `NotificationService` instance so task lifecycle notifications (`task:moved`, `task:updated`, `task:merged`) are emitted through a single path.
- Compatibility scope: `NtfyNotifier` remains responsible for gridlock-only compatibility notifications (`notifyGridlock`) and legacy helper APIs. - Compatibility scope: `NtfyNotifier` remains responsible for gridlock-only compatibility notifications (`notifyGridlock`) and legacy helper APIs.
- Legacy gridlock ntfy delivery is cooldown-throttled: first detection notifies immediately, subsequent detections are suppressed for 15 minutes (even if blocked-task membership changes), and the cooldown resets as soon as gridlock fully clears.
- `NotificationService` (`notification/notification-service.ts`) — provider lifecycle + event dispatch orchestration - `NotificationService` (`notification/notification-service.ts`) — provider lifecycle + event dispatch orchestration
- `NotificationProvider` interface (`@fusion/core` `notification/provider.ts`) — pluggable provider contract - `NotificationProvider` interface (`@fusion/core` `notification/provider.ts`) — pluggable provider contract
- Built-in providers: `NtfyNotificationProvider` (`notification/ntfy-provider.ts`), `WebhookNotificationProvider` (`notification/webhook-provider.ts`) - Built-in providers: `NtfyNotificationProvider` (`notification/ntfy-provider.ts`), `WebhookNotificationProvider` (`notification/webhook-provider.ts`)

View File

@@ -41,7 +41,7 @@ Defaults from `DEFAULT_GLOBAL_SETTINGS`; key scope from `GLOBAL_SETTINGS_KEYS`.
| `ntfyEnabled` | `boolean` | `false` | Enable ntfy push notifications. | | `ntfyEnabled` | `boolean` | `false` | Enable ntfy push notifications. |
| `ntfyTopic` | `string` | `undefined` | ntfy topic name. | | `ntfyTopic` | `string` | `undefined` | ntfy topic name. |
| `ntfyBaseUrl` | `string` | `undefined` | Optional custom ntfy server base URL (must use `http://` or `https://`). If blank/unset, Fusion uses `https://ntfy.sh` for both runtime and test notifications. | | `ntfyBaseUrl` | `string` | `undefined` | Optional custom ntfy server base URL (must use `http://` or `https://`). If blank/unset, Fusion uses `https://ntfy.sh` for both runtime and test notifications. |
| `ntfyEvents` | `("in-review" \| "merged" \| "failed" \| "awaiting-approval" \| "awaiting-user-review" \| "planning-awaiting-input" \| "gridlock")[]` | `["in-review","merged","failed","awaiting-approval","awaiting-user-review","planning-awaiting-input","gridlock"]` | Event types that trigger ntfy notifications. `planning-awaiting-input` fires when planning mode is waiting on user input. `gridlock` fires when all schedulable todo tasks are blocked. | | `ntfyEvents` | `("in-review" \| "merged" \| "failed" \| "awaiting-approval" \| "awaiting-user-review" \| "planning-awaiting-input" \| "gridlock")[]` | `["in-review","merged","failed","awaiting-approval","awaiting-user-review","planning-awaiting-input","gridlock"]` | Event types that trigger ntfy notifications. `planning-awaiting-input` fires when planning mode is waiting on user input. `gridlock` fires when all schedulable todo tasks are blocked; delivery is cooldown-throttled (first alert immediately, then suppressed for 15 minutes until gridlock resolves). |
| `ntfyDashboardHost` | `string` | `undefined` | Dashboard host used to build deep links in notifications. | | `ntfyDashboardHost` | `string` | `undefined` | Dashboard host used to build deep links in notifications. |
| `webhookEnabled` | `boolean` | `false` | Enable webhook notifications for task lifecycle events. Part of the legacy flat settings; prefer `notificationProviders` for new setups. | | `webhookEnabled` | `boolean` | `false` | Enable webhook notifications for task lifecycle events. Part of the legacy flat settings; prefer `notificationProviders` for new setups. |
| `webhookUrl` | `string` | `undefined` | Webhook endpoint URL. Must be `http://` or `https://`. Part of legacy flat settings. | | `webhookUrl` | `string` | `undefined` | Webhook endpoint URL. Must be `http://` or `https://`. Part of legacy flat settings. |
@@ -115,7 +115,7 @@ When `id` is `"ntfy"` in `notificationProviders`, the provider `config` supports
|---|---|---:|---| |---|---|---:|---|
| `topic` | `string` | _required_ | ntfy topic name (164 chars, alphanumeric + `-_`). | | `topic` | `string` | _required_ | ntfy topic name (164 chars, alphanumeric + `-_`). |
| `ntfyBaseUrl` | `string` | `"https://ntfy.sh"` | Optional custom ntfy server URL. | | `ntfyBaseUrl` | `string` | `"https://ntfy.sh"` | Optional custom ntfy server URL. |
| `events` | `("in-review" \| "merged" \| "failed" \| "awaiting-approval" \| "awaiting-user-review" \| "planning-awaiting-input" \| "gridlock")[]` | `DEFAULT_NTFY_EVENTS` | Event filter list used by the provider. | | `events` | `("in-review" \| "merged" \| "failed" \| "awaiting-approval" \| "awaiting-user-review" \| "planning-awaiting-input" \| "gridlock")[]` | `DEFAULT_NTFY_EVENTS` | Event filter list used by the provider. For `gridlock`, enabled events are still cooldown-throttled at runtime (15-minute suppression window, reset on full resolution). |
| `dashboardHost` | `string` | `undefined` | Dashboard host for deep links in notifications. | | `dashboardHost` | `string` | `undefined` | Dashboard host for deep links in notifications. |
Disable daily update checks globally: Disable daily update checks globally:

View File

@@ -14,24 +14,31 @@ type ProviderApiType = CustomProvider["apiType"];
const API_TYPES: ProviderApiType[] = ["openai-compatible", "anthropic-compatible"]; const API_TYPES: ProviderApiType[] = ["openai-compatible", "anthropic-compatible"];
function normalizeProviders(result: Awaited<ReturnType<typeof fetchCustomProviders>>): CustomProvider[] { type LegacyProvider = {
const legacyProviders = Array.isArray(result) id: string;
? result name?: string;
: Array.isArray((result as { providers?: unknown }).providers) baseUrl: string;
? (result as { providers: (typeof result)[number][] }).providers api: "openai-completions" | "openai-responses" | "anthropic-messages" | "google-generative-ai";
: []; apiKey?: string;
models?: Array<{ id: string; name?: string }>;
};
return legacyProviders.map((provider) => { function normalizeProviders(result: Awaited<ReturnType<typeof fetchCustomProviders>>): CustomProvider[] {
const providerRecords: Array<CustomProvider | LegacyProvider> = Array.isArray(result)
? (result as Array<CustomProvider | LegacyProvider>)
: ((result as { providers?: Array<CustomProvider | LegacyProvider> }).providers ?? []);
return providerRecords.map((provider) => {
if ("apiType" in provider) { if ("apiType" in provider) {
return provider as unknown as CustomProvider; return provider;
} }
return { return {
id: provider.id, id: provider.id,
name: provider.name ?? provider.id, name: provider.name?.trim() || provider.id,
apiType: provider.api === "anthropic-messages" ? "anthropic-compatible" : "openai-compatible",
baseUrl: provider.baseUrl, baseUrl: provider.baseUrl,
...(provider.apiKey ? { apiKey: provider.apiKey } : {}), ...(provider.apiKey ? { apiKey: provider.apiKey } : {}),
apiType: provider.api === "anthropic-messages" ? "anthropic-compatible" : "openai-compatible",
models: (provider.models ?? []).map((model) => ({ models: (provider.models ?? []).map((model) => ({
id: model.id, id: model.id,
name: model.name ?? model.id, name: model.name ?? model.id,

View File

@@ -35,6 +35,7 @@ describe("GridlockDetector", () => {
let settings: Settings; let settings: Settings;
let scopes: Record<string, string[]>; let scopes: Record<string, string[]>;
let onGridlock: ReturnType<typeof vi.fn>; let onGridlock: ReturnType<typeof vi.fn>;
let onGridlockCleared: ReturnType<typeof vi.fn>;
let store: TaskStore; let store: TaskStore;
let detector: GridlockDetector; let detector: GridlockDetector;
@@ -43,12 +44,13 @@ describe("GridlockDetector", () => {
settings = createSettings(); settings = createSettings();
scopes = {}; scopes = {};
onGridlock = vi.fn(); onGridlock = vi.fn();
onGridlockCleared = vi.fn();
store = { store = {
listTasks: vi.fn(async () => tasks), listTasks: vi.fn(async () => tasks),
getSettings: vi.fn(async () => settings), getSettings: vi.fn(async () => settings),
parseFileScopeFromPrompt: vi.fn(async (taskId: string) => scopes[taskId] ?? []), parseFileScopeFromPrompt: vi.fn(async (taskId: string) => scopes[taskId] ?? []),
} as unknown as TaskStore; } as unknown as TaskStore;
detector = new GridlockDetector(store, { onGridlock }); detector = new GridlockDetector(store, { onGridlock, onGridlockCleared });
}); });
afterEach(() => { afterEach(() => {
@@ -162,6 +164,7 @@ describe("GridlockDetector", () => {
await detector.detectGridlock(); await detector.detectGridlock();
expect(onGridlock).toHaveBeenCalledTimes(2); expect(onGridlock).toHaveBeenCalledTimes(2);
expect(onGridlockCleared).toHaveBeenCalledTimes(1);
}); });
it("respects paused and recovery-backoff tasks as non-schedulable", async () => { it("respects paused and recovery-backoff tasks as non-schedulable", async () => {

View File

@@ -142,6 +142,15 @@ describe("NtfyNotifier", () => {
}); });
describe("gridlock notifications", () => { describe("gridlock notifications", () => {
beforeEach(() => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-01-01T00:00:00.000Z"));
});
afterEach(() => {
vi.useRealTimers();
});
it("sends notification when gridlock event is enabled", async () => { it("sends notification when gridlock event is enabled", async () => {
store.setSettings({ ntfyEnabled: true, ntfyTopic: "test-topic", ntfyEvents: ["gridlock"] }); store.setSettings({ ntfyEnabled: true, ntfyTopic: "test-topic", ntfyEvents: ["gridlock"] });
fetchMock.mockResolvedValue({ ok: true }); fetchMock.mockResolvedValue({ ok: true });
@@ -186,7 +195,7 @@ describe("NtfyNotifier", () => {
expect(fetchMock).not.toHaveBeenCalled(); expect(fetchMock).not.toHaveBeenCalled();
}); });
it("deduplicates by blocked task set", async () => { it("suppresses repeated gridlock notifications during the 15-minute cooldown even when blocked set changes", async () => {
store.setSettings({ ntfyEnabled: true, ntfyTopic: "test-topic", ntfyEvents: ["gridlock"] }); store.setSettings({ ntfyEnabled: true, ntfyTopic: "test-topic", ntfyEvents: ["gridlock"] });
fetchMock.mockResolvedValue({ ok: true }); fetchMock.mockResolvedValue({ ok: true });
notifier = new NtfyNotifier(store); notifier = new NtfyNotifier(store);
@@ -198,16 +207,44 @@ describe("NtfyNotifier", () => {
blockedTaskIds: ["FN-003", "FN-001"], blockedTaskIds: ["FN-003", "FN-001"],
blockingTaskIds: ["FN-002"], blockingTaskIds: ["FN-002"],
}); });
vi.advanceTimersByTime(5 * 60 * 1000);
notifier.notifyGridlock({ notifier.notifyGridlock({
blockedTaskCount: 2, blockedTaskCount: 3,
reasons: { "FN-001": "dependency", "FN-003": "dependency" }, reasons: { "FN-001": "dependency", "FN-003": "dependency", "FN-004": "overlap" },
blockedTaskIds: ["FN-001", "FN-003"], blockedTaskIds: ["FN-001", "FN-003", "FN-004"],
blockingTaskIds: ["FN-002"], blockingTaskIds: ["FN-002", "FN-005"],
}); });
await flushAsyncWork(); await flushAsyncWork();
expect(fetchMock).toHaveBeenCalledTimes(1); expect(fetchMock).toHaveBeenCalledTimes(1);
}); });
it("allows a new gridlock notification immediately after resolution reset", async () => {
store.setSettings({ ntfyEnabled: true, ntfyTopic: "test-topic", ntfyEvents: ["gridlock"] });
fetchMock.mockResolvedValue({ ok: true });
notifier = new NtfyNotifier(store);
await notifier.start();
notifier.notifyGridlock({
blockedTaskCount: 1,
reasons: { "FN-001": "dependency" },
blockedTaskIds: ["FN-001"],
blockingTaskIds: ["FN-002"],
});
vi.advanceTimersByTime(60_000);
notifier.notifyGridlock(null);
notifier.notifyGridlock({
blockedTaskCount: 1,
reasons: { "FN-009": "overlap" },
blockedTaskIds: ["FN-009"],
blockingTaskIds: ["FN-010"],
});
await flushAsyncWork();
expect(fetchMock).toHaveBeenCalledTimes(2);
});
}); });
describe("when enabled", () => { describe("when enabled", () => {

View File

@@ -15,6 +15,7 @@ export interface GridlockDetectorOptions {
pollIntervalMs?: number; pollIntervalMs?: number;
missionStore?: MissionStore; missionStore?: MissionStore;
onGridlock?: (event: GridlockEvent) => void; onGridlock?: (event: GridlockEvent) => void;
onGridlockCleared?: () => void;
} }
export class GridlockDetector { export class GridlockDetector {
@@ -22,6 +23,7 @@ export class GridlockDetector {
private readonly pollIntervalMs: number; private readonly pollIntervalMs: number;
private readonly missionStore?: MissionStore; private readonly missionStore?: MissionStore;
private readonly onGridlock?: (event: GridlockEvent) => void; private readonly onGridlock?: (event: GridlockEvent) => void;
private readonly onGridlockCleared?: () => void;
private lastGridlockKey: string | null = null; private lastGridlockKey: string | null = null;
constructor( constructor(
@@ -31,6 +33,7 @@ export class GridlockDetector {
this.pollIntervalMs = options.pollIntervalMs ?? 30_000; this.pollIntervalMs = options.pollIntervalMs ?? 30_000;
this.missionStore = options.missionStore; this.missionStore = options.missionStore;
this.onGridlock = options.onGridlock; this.onGridlock = options.onGridlock;
this.onGridlockCleared = options.onGridlockCleared;
} }
start(): void { start(): void {
@@ -65,13 +68,13 @@ export class GridlockDetector {
}); });
if (schedulable.length === 0) { if (schedulable.length === 0) {
this.lastGridlockKey = null; this.clearGridlockState();
return null; return null;
} }
const active = tasks.filter((task) => task.column === "in-progress" || (task.column === "in-review" && Boolean(task.worktree))); const active = tasks.filter((task) => task.column === "in-progress" || (task.column === "in-review" && Boolean(task.worktree)));
if (active.length === 0) { if (active.length === 0) {
this.lastGridlockKey = null; this.clearGridlockState();
return null; return null;
} }
@@ -117,7 +120,7 @@ export class GridlockDetector {
const blockedTaskIds = Object.keys(reasons).sort(); const blockedTaskIds = Object.keys(reasons).sort();
if (blockedTaskIds.length !== schedulable.length) { if (blockedTaskIds.length !== schedulable.length) {
this.lastGridlockKey = null; this.clearGridlockState();
return null; return null;
} }
@@ -138,6 +141,13 @@ export class GridlockDetector {
return event; return event;
} }
private clearGridlockState(): void {
if (this.lastGridlockKey !== null) {
this.lastGridlockKey = null;
this.onGridlockCleared?.();
}
}
private isMissionBlocked(task: Task): boolean { private isMissionBlocked(task: Task): boolean {
if (!this.missionStore || !task.sliceId) return false; if (!this.missionStore || !task.sliceId) return false;
try { try {

View File

@@ -13,6 +13,7 @@ export interface NtfyNotifierOptions {
export type NtfyNotificationPriority = "low" | "default" | "high" | "urgent"; export type NtfyNotificationPriority = "low" | "default" | "high" | "urgent";
const DEFAULT_NTFY_BASE_URL = "https://ntfy.sh"; const DEFAULT_NTFY_BASE_URL = "https://ntfy.sh";
const GRIDLOCK_NOTIFICATION_COOLDOWN_MS = 15 * 60 * 1000;
export const DEFAULT_NTFY_EVENTS: readonly NtfyNotificationEvent[] = [ export const DEFAULT_NTFY_EVENTS: readonly NtfyNotificationEvent[] = [
"in-review", "in-review",
@@ -176,8 +177,8 @@ export class NtfyNotifier {
private ntfyBaseUrl: string; private ntfyBaseUrl: string;
private readonly defaultNtfyBaseUrl: string; private readonly defaultNtfyBaseUrl: string;
private readonly projectId?: string; private readonly projectId?: string;
private notifiedEvents: Set<string> = new Set();
private abortController: AbortController | null = null; private abortController: AbortController | null = null;
private lastGridlockNotificationAt: number | null = null;
constructor( constructor(
private store: NtfyNotifierStore, private store: NtfyNotifierStore,
@@ -230,10 +231,23 @@ export class NtfyNotifier {
this.ntfyBaseUrl = resolveNtfyBaseUrl(settings.ntfyBaseUrl, this.defaultNtfyBaseUrl); this.ntfyBaseUrl = resolveNtfyBaseUrl(settings.ntfyBaseUrl, this.defaultNtfyBaseUrl);
} }
notifyGridlock(event: GridlockEvent): void { notifyGridlock(event: GridlockEvent | null): void {
if (event === null) {
this.lastGridlockNotificationAt = null;
return;
}
if (!this.config.enabled || !this.config.topic || !this.isEventEnabled("gridlock")) return; if (!this.config.enabled || !this.config.topic || !this.isEventEnabled("gridlock")) return;
const blockedTasks = event.blockedTaskIds.sort(); const now = Date.now();
if (
this.lastGridlockNotificationAt !== null
&& now - this.lastGridlockNotificationAt < GRIDLOCK_NOTIFICATION_COOLDOWN_MS
) {
return;
}
const blockedTasks = [...event.blockedTaskIds].sort();
const reasonSummary = Object.values(event.reasons).reduce((acc, reason) => { const reasonSummary = Object.values(event.reasons).reduce((acc, reason) => {
acc[reason] = (acc[reason] ?? 0) + 1; acc[reason] = (acc[reason] ?? 0) + 1;
return acc; return acc;
@@ -248,35 +262,24 @@ export class NtfyNotifier {
projectId: this.projectId, projectId: this.projectId,
}); });
const dedupKey = `gridlock:${blockedTasks.join(",")}`; this.lastGridlockNotificationAt = now;
this.maybeNotifyByKey(dedupKey, () => sendNtfyNotification({
sendNtfyNotification({ ntfyBaseUrl: this.ntfyBaseUrl,
ntfyBaseUrl: this.ntfyBaseUrl, topic: this.config.topic!,
topic: this.config.topic!, title: "Pipeline gridlocked",
title: "Pipeline gridlocked", message: `${event.blockedTaskCount} todo tasks are blocked (${reasons.join(", ")}). Blocked: ${blockedTasks.join(", ")}. Blocking: ${event.blockingTaskIds.join(", ") || "none"}.`,
message: `${event.blockedTaskCount} todo tasks are blocked (${reasons.join(", ")}). Blocked: ${blockedTasks.join(", ")}. Blocking: ${event.blockingTaskIds.join(", ") || "none"}.`, priority: "high",
priority: "high", clickUrl,
clickUrl, signal: this.abortController?.signal,
signal: this.abortController?.signal, }).catch(() => {
}), // sendNtfyNotification already logs; notifier must stay best-effort
); });
} }
private isEventEnabled(event: AnyNotificationEvent): boolean { private isEventEnabled(event: AnyNotificationEvent): boolean {
return isNtfyEventEnabled(this.config.events, event); return isNtfyEventEnabled(this.config.events, event);
} }
private maybeNotifyByKey(key: string, notifyFn: () => Promise<void>): void {
if (this.notifiedEvents.has(key)) {
return;
}
this.notifiedEvents.add(key);
notifyFn().catch(() => {
// sendNtfyNotification already logs; notifier must stay best-effort
});
}
getConfig(): NtfyConfig { getConfig(): NtfyConfig {
return { ...this.config, events: [...this.config.events] }; return { ...this.config, events: [...this.config.events] };
} }

View File

@@ -282,6 +282,7 @@ export class ProjectEngine {
this.gridlockDetector = new GridlockDetector(store, { this.gridlockDetector = new GridlockDetector(store, {
onGridlock: (event) => this.notifier?.notifyGridlock(event), onGridlock: (event) => this.notifier?.notifyGridlock(event),
onGridlockCleared: () => this.notifier?.notifyGridlock(null),
}); });
this.gridlockDetector.start(); this.gridlockDetector.start();