feat(FN-3008): notify when model settings fall back to defaults
Merges FN-3008 to add a "fallback-used" notification system: the engine now emits events when AI model fallbacks are triggered, dispatches notifications via ntfy/webhook providers, surfaces a session banner in the dashboard, and exposes a settings toggle to enable or disable these alerts. Fusion-Task-Id: FN-3008
This commit is contained in:
5
.changeset/fn-3008-fallback-used-notification.md
Normal file
5
.changeset/fn-3008-fallback-used-notification.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
Add a dedicated `fallback-used` notification event that fires when Fusion recovers from a retryable model failure by switching to a configured fallback model, and expose it in global notification settings for ntfy/webhook filtering.
|
||||
@@ -42,7 +42,7 @@ Defaults from `DEFAULT_GLOBAL_SETTINGS`; key scope from `GLOBAL_SETTINGS_KEYS`.
|
||||
| `ntfyEnabled` | `boolean` | `false` | Enable ntfy push notifications. |
|
||||
| `ntfyTopic` | `string` | `undefined` | ntfy topic name. |
|
||||
| `ntfyBaseUrl` | `string` | `undefined` | Optional custom ntfy server base URL (must use `http://` or `https://`). If blank/unset, Fusion uses `https://ntfy.sh` for both runtime and test notifications. |
|
||||
| `ntfyEvents` | `("in-review" \| "merged" \| "failed" \| "awaiting-approval" \| "awaiting-user-review" \| "planning-awaiting-input" \| "gridlock")[]` | `["in-review","merged","failed","awaiting-approval","awaiting-user-review","planning-awaiting-input","gridlock"]` | Event types that trigger ntfy notifications. `planning-awaiting-input` fires when planning mode is waiting on user input. `gridlock` fires when all schedulable todo tasks are blocked; delivery is cooldown-throttled (first alert immediately, then suppressed for 15 minutes until gridlock resolves). |
|
||||
| `ntfyEvents` | `("in-review" \| "merged" \| "failed" \| "awaiting-approval" \| "awaiting-user-review" \| "planning-awaiting-input" \| "gridlock" \| "fallback-used")[]` | `["in-review","merged","failed","awaiting-approval","awaiting-user-review","planning-awaiting-input","gridlock","fallback-used"]` | Event types that trigger ntfy notifications. `planning-awaiting-input` fires when planning mode is waiting on user input. `gridlock` fires when all schedulable todo tasks are blocked; delivery is cooldown-throttled (first alert immediately, then suppressed for 15 minutes until gridlock resolves). `fallback-used` fires when Fusion recovers from a retryable model failure by switching to a configured fallback model. |
|
||||
| `ntfyDashboardHost` | `string` | `undefined` | Dashboard host used to build deep links in notifications. |
|
||||
| `webhookEnabled` | `boolean` | `false` | Enable webhook notifications for task lifecycle events. Part of the legacy flat settings; prefer `notificationProviders` for new setups. |
|
||||
| `webhookUrl` | `string` | `undefined` | Webhook endpoint URL. Must be `http://` or `https://`. Part of legacy flat settings. |
|
||||
|
||||
@@ -317,7 +317,16 @@ describe("GlobalSettingsStore", () => {
|
||||
// After clear, reading back gives the default value
|
||||
// (either undefined on disk with default applied, or default written directly)
|
||||
const settings = await store.getSettings();
|
||||
expect(settings.ntfyEvents).toEqual(["in-review", "merged", "failed", "awaiting-approval", "awaiting-user-review", "planning-awaiting-input", "gridlock"]);
|
||||
expect(settings.ntfyEvents).toEqual([
|
||||
"in-review",
|
||||
"merged",
|
||||
"failed",
|
||||
"awaiting-approval",
|
||||
"awaiting-user-review",
|
||||
"planning-awaiting-input",
|
||||
"gridlock",
|
||||
"fallback-used",
|
||||
]);
|
||||
});
|
||||
|
||||
it("handles concurrent updates safely via locking", async () => {
|
||||
|
||||
@@ -153,6 +153,7 @@ describe("NotificationDispatcher", () => {
|
||||
"awaiting-user-review",
|
||||
"planning-awaiting-input",
|
||||
"gridlock",
|
||||
"fallback-used",
|
||||
]);
|
||||
expect(DEFAULT_GLOBAL_SETTINGS.notificationProviders).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -23,7 +23,16 @@ export const DEFAULT_GLOBAL_SETTINGS = {
|
||||
ntfyEnabled: false,
|
||||
ntfyTopic: undefined,
|
||||
ntfyBaseUrl: undefined,
|
||||
ntfyEvents: ["in-review", "merged", "failed", "awaiting-approval", "awaiting-user-review", "planning-awaiting-input", "gridlock"],
|
||||
ntfyEvents: [
|
||||
"in-review",
|
||||
"merged",
|
||||
"failed",
|
||||
"awaiting-approval",
|
||||
"awaiting-user-review",
|
||||
"planning-awaiting-input",
|
||||
"gridlock",
|
||||
"fallback-used",
|
||||
],
|
||||
ntfyDashboardHost: undefined,
|
||||
webhookEnabled: false,
|
||||
webhookUrl: undefined,
|
||||
|
||||
@@ -208,7 +208,15 @@ export interface WorkflowStep {
|
||||
|
||||
/** Input for creating a new workflow step. */
|
||||
/** Event types that can trigger ntfy notifications */
|
||||
export type NtfyNotificationEvent = "in-review" | "merged" | "failed" | "awaiting-approval" | "awaiting-user-review" | "planning-awaiting-input" | "gridlock";
|
||||
export type NtfyNotificationEvent =
|
||||
| "in-review"
|
||||
| "merged"
|
||||
| "failed"
|
||||
| "awaiting-approval"
|
||||
| "awaiting-user-review"
|
||||
| "planning-awaiting-input"
|
||||
| "gridlock"
|
||||
| "fallback-used";
|
||||
|
||||
/** Known notification event types. Providers may support additional custom events. */
|
||||
export const NOTIFICATION_EVENTS = [
|
||||
@@ -219,6 +227,7 @@ export const NOTIFICATION_EVENTS = [
|
||||
"awaiting-user-review",
|
||||
"planning-awaiting-input",
|
||||
"gridlock",
|
||||
"fallback-used",
|
||||
] as const;
|
||||
|
||||
/** Notification event type. Known events plus provider-specific custom events. */
|
||||
@@ -226,7 +235,7 @@ export type NotificationEvent = (typeof NOTIFICATION_EVENTS)[number] | (string &
|
||||
|
||||
/** Standard payload shape shared across notification providers. */
|
||||
export interface NotificationPayload {
|
||||
taskId: string;
|
||||
taskId?: string;
|
||||
taskTitle?: string;
|
||||
taskDescription?: string;
|
||||
event: NotificationEvent;
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
/* === Session Notification Banner === */
|
||||
|
||||
.session-notification-banner {
|
||||
--session-notification-banner-max-height: min(60vh, calc(var(--space-2xl) * 13));
|
||||
--session-notification-list-max-height: min(48vh, calc(var(--space-2xl) * 9));
|
||||
--session-notification-touch-size: calc(var(--space-xl) + var(--space-md));
|
||||
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 30;
|
||||
@@ -8,14 +12,14 @@
|
||||
flex-direction: column;
|
||||
gap: var(--space-sm);
|
||||
padding: var(--space-sm) var(--space-lg);
|
||||
border-bottom: 1px solid var(--border);
|
||||
border-left: 4px solid var(--triage);
|
||||
border-bottom: var(--btn-border-width) solid var(--border);
|
||||
border-left: var(--space-xs) solid var(--triage);
|
||||
background: var(--surface);
|
||||
box-sizing: border-box;
|
||||
max-height: 420px;
|
||||
max-height: var(--session-notification-banner-max-height);
|
||||
opacity: 1;
|
||||
transition: opacity var(--transition-fast), max-height var(--transition-fast);
|
||||
animation: session-notification-banner-enter 180ms ease-out;
|
||||
animation: session-notification-banner-enter var(--transition-fast);
|
||||
}
|
||||
|
||||
@keyframes session-notification-banner-enter {
|
||||
@@ -25,7 +29,7 @@
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
max-height: 420px;
|
||||
max-height: var(--session-notification-banner-max-height);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,14 +56,15 @@
|
||||
.session-notification-banner__dismiss-all {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
border: 1px solid transparent;
|
||||
gap: var(--space-sm);
|
||||
min-height: var(--session-notification-touch-size);
|
||||
border: var(--btn-border-width) solid transparent;
|
||||
background: transparent;
|
||||
color: var(--text-muted);
|
||||
font-size: 12px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
border-radius: var(--radius-sm);
|
||||
padding: 4px 8px;
|
||||
padding: var(--space-xs) var(--space-sm);
|
||||
cursor: pointer;
|
||||
transition: background var(--transition-fast), color var(--transition-fast), border-color var(--transition-fast);
|
||||
}
|
||||
@@ -70,12 +75,19 @@
|
||||
border-color: var(--border);
|
||||
}
|
||||
|
||||
.session-notification-banner__dismiss-all:focus-visible,
|
||||
.session-notification-banner__resume:focus-visible,
|
||||
.session-notification-banner__dismiss:focus-visible {
|
||||
outline: none;
|
||||
box-shadow: var(--focus-ring-strong);
|
||||
}
|
||||
|
||||
.session-notification-banner__list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
gap: var(--space-sm);
|
||||
overflow-y: auto;
|
||||
max-height: 300px;
|
||||
max-height: var(--session-notification-list-max-height);
|
||||
}
|
||||
|
||||
.session-notification-banner__item {
|
||||
@@ -83,16 +95,21 @@
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-sm);
|
||||
padding: 8px 10px;
|
||||
border: 1px solid var(--border);
|
||||
padding: var(--space-sm) var(--space-md);
|
||||
border: var(--btn-border-width) solid var(--border);
|
||||
border-radius: var(--radius-sm);
|
||||
background: color-mix(in srgb, var(--triage) 8%, var(--surface));
|
||||
}
|
||||
|
||||
.session-notification-banner__item--error {
|
||||
border-color: color-mix(in srgb, var(--color-error) 35%, var(--border));
|
||||
background: var(--status-error-bg, color-mix(in srgb, var(--color-error) 12%, var(--surface)));
|
||||
}
|
||||
|
||||
.session-notification-banner__item-main {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
gap: var(--space-sm);
|
||||
min-width: 0;
|
||||
flex: 1;
|
||||
}
|
||||
@@ -102,13 +119,17 @@
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.session-notification-banner__type-icon--error {
|
||||
color: var(--color-error);
|
||||
}
|
||||
|
||||
.session-notification-banner__text {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.session-notification-banner__title {
|
||||
margin: 0;
|
||||
font-size: 13px;
|
||||
font-size: 0.8125rem;
|
||||
font-weight: 600;
|
||||
color: var(--text);
|
||||
white-space: nowrap;
|
||||
@@ -117,25 +138,26 @@
|
||||
}
|
||||
|
||||
.session-notification-banner__meta {
|
||||
margin: 2px 0 0;
|
||||
font-size: 12px;
|
||||
margin: var(--space-xs) 0 0;
|
||||
font-size: 0.75rem;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.session-notification-banner__actions {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
gap: var(--space-sm);
|
||||
}
|
||||
|
||||
.session-notification-banner__resume {
|
||||
border: 1px solid color-mix(in srgb, var(--triage) 60%, var(--border));
|
||||
min-height: var(--session-notification-touch-size);
|
||||
border: var(--btn-border-width) solid color-mix(in srgb, var(--triage) 60%, var(--border));
|
||||
background: transparent;
|
||||
color: var(--triage);
|
||||
border-radius: var(--radius-sm);
|
||||
font-size: 12px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
padding: 4px 10px;
|
||||
padding: var(--space-xs) var(--space-md);
|
||||
cursor: pointer;
|
||||
transition: background var(--transition-fast), color var(--transition-fast), border-color var(--transition-fast);
|
||||
}
|
||||
@@ -145,14 +167,24 @@
|
||||
border-color: var(--triage);
|
||||
}
|
||||
|
||||
.session-notification-banner__item--error .session-notification-banner__resume {
|
||||
border-color: color-mix(in srgb, var(--color-error) 60%, var(--border));
|
||||
color: var(--color-error);
|
||||
}
|
||||
|
||||
.session-notification-banner__item--error .session-notification-banner__resume:hover {
|
||||
background: color-mix(in srgb, var(--color-error) 14%, transparent);
|
||||
border-color: var(--color-error);
|
||||
}
|
||||
|
||||
.session-notification-banner__dismiss {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
width: var(--session-notification-touch-size);
|
||||
height: var(--session-notification-touch-size);
|
||||
border-radius: var(--radius-sm);
|
||||
border: 1px solid transparent;
|
||||
border: var(--btn-border-width) solid transparent;
|
||||
background: transparent;
|
||||
color: var(--text-muted);
|
||||
cursor: pointer;
|
||||
@@ -168,7 +200,7 @@
|
||||
@media (max-width: 768px) {
|
||||
.session-notification-banner {
|
||||
padding: var(--space-sm) var(--space-md);
|
||||
gap: 8px;
|
||||
gap: var(--space-sm);
|
||||
}
|
||||
|
||||
.session-notification-banner__header {
|
||||
|
||||
@@ -238,6 +238,7 @@ const DEFAULT_NTFY_EVENTS: NtfyNotificationEvent[] = [
|
||||
"awaiting-user-review",
|
||||
"planning-awaiting-input",
|
||||
"gridlock",
|
||||
"fallback-used",
|
||||
];
|
||||
|
||||
const NOTIFICATION_EVENT_OPTIONS: Array<{ event: NtfyNotificationEvent; label: string; description: string }> = [
|
||||
@@ -248,6 +249,7 @@ const NOTIFICATION_EVENT_OPTIONS: Array<{ event: NtfyNotificationEvent; label: s
|
||||
{ event: "awaiting-user-review", label: "User review needed", description: "When an agent hands off a task for human review (high priority)" },
|
||||
{ event: "planning-awaiting-input", label: "Planning needs input", description: "When planning mode is waiting for your response to continue" },
|
||||
{ event: "gridlock", label: "Pipeline gridlocked", description: "When all schedulable todo tasks are blocked and work cannot advance" },
|
||||
{ event: "fallback-used", label: "Fallback model used (recovered)", description: "When Fusion recovers from a retryable model failure by switching to a fallback model" },
|
||||
];
|
||||
|
||||
/** Well-known experimental feature flags with display labels.
|
||||
|
||||
@@ -2416,6 +2416,18 @@ describe("SettingsModal", () => {
|
||||
expect(screen.getByRole("button", { name: /Test notification/ })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows fallback-used event option for both providers", async () => {
|
||||
mockFetchSettings.mockResolvedValueOnce({ ...defaultSettings, ntfyEnabled: true, ntfyTopic: "test-topic" });
|
||||
renderModal();
|
||||
await waitForSettingsModalReady();
|
||||
await openNotificationsSection();
|
||||
|
||||
expect(screen.getByLabelText("Fallback model used (recovered)")).toBeInTheDocument();
|
||||
|
||||
await userEvent.click(screen.getByLabelText("Webhook notifications"));
|
||||
expect(screen.getAllByLabelText("Fallback model used (recovered)").length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("shows webhook fields when webhook provider is enabled", async () => {
|
||||
renderModal();
|
||||
await waitForSettingsModalReady();
|
||||
@@ -2493,9 +2505,11 @@ describe("SettingsModal", () => {
|
||||
const inReview = screen.getByLabelText("Task completed (in-review)") as HTMLInputElement;
|
||||
const failed = screen.getByLabelText("Task failed") as HTMLInputElement;
|
||||
const merged = screen.getByLabelText("Task merged") as HTMLInputElement;
|
||||
const fallbackUsed = screen.getByLabelText("Fallback model used (recovered)") as HTMLInputElement;
|
||||
expect(inReview.checked).toBe(true);
|
||||
expect(failed.checked).toBe(true);
|
||||
expect(merged.checked).toBe(false);
|
||||
expect(fallbackUsed.checked).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ const defaultSettings = {
|
||||
defaultPresetBySize: {},
|
||||
ntfyEnabled: false,
|
||||
ntfyTopic: undefined,
|
||||
ntfyEvents: ["in-review", "merged", "failed", "awaiting-approval", "awaiting-user-review"],
|
||||
ntfyEvents: ["in-review", "merged", "failed", "awaiting-approval", "awaiting-user-review", "fallback-used"],
|
||||
webhookEnabled: false,
|
||||
webhookUrl: undefined,
|
||||
webhookFormat: undefined,
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
buildNtfyClickUrl,
|
||||
isNtfyEventEnabled,
|
||||
resolveNtfyEvents,
|
||||
notifyFallbackUsed,
|
||||
} from "../notifier.js";
|
||||
import { NotificationService } from "../notification/notification-service.js";
|
||||
|
||||
@@ -68,6 +69,7 @@ describe("Ntfy notifier helpers", () => {
|
||||
expect(DEFAULT_NTFY_EVENTS).toContain("planning-awaiting-input");
|
||||
expect(resolveNtfyEvents(undefined)).toContain("planning-awaiting-input");
|
||||
expect(DEFAULT_NTFY_EVENTS).toContain("gridlock");
|
||||
expect(DEFAULT_NTFY_EVENTS).toContain("fallback-used");
|
||||
});
|
||||
|
||||
it("checks planning-awaiting-input event enablement", () => {
|
||||
@@ -1123,6 +1125,35 @@ describe("NtfyNotifier", () => {
|
||||
await sharedService.stop();
|
||||
});
|
||||
|
||||
it("dispatches and deduplicates fallback-used notifications", async () => {
|
||||
notifier = new NtfyNotifier(store);
|
||||
await notifier.start();
|
||||
|
||||
await notifyFallbackUsed({
|
||||
primaryModel: "anthropic/claude-sonnet-4-5",
|
||||
fallbackModel: "openai/gpt-4o",
|
||||
triggerPoint: "session-creation",
|
||||
taskId: "FN-900",
|
||||
taskTitle: "Fallback task",
|
||||
});
|
||||
await notifyFallbackUsed({
|
||||
primaryModel: "anthropic/claude-sonnet-4-5",
|
||||
fallbackModel: "openai/gpt-4o",
|
||||
triggerPoint: "session-creation",
|
||||
taskId: "FN-900",
|
||||
taskTitle: "Fallback task",
|
||||
});
|
||||
await flushAsyncWork();
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://ntfy.sh/test-topic",
|
||||
expect.objectContaining({
|
||||
body: expect.stringContaining("switched from anthropic/claude-sonnet-4-5 to openai/gpt-4o"),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("allows notifications for different tasks independently", async () => {
|
||||
notifier = new NtfyNotifier(store);
|
||||
await notifier.start();
|
||||
|
||||
@@ -42,6 +42,7 @@ describe("NtfyNotificationProvider", () => {
|
||||
["awaiting-approval", "Plan needs approval for FN-1", "needs your approval", "high"],
|
||||
["awaiting-user-review", "User review needed for FN-1", "needs human review", "high"],
|
||||
["planning-awaiting-input", "Planning input needed for FN-1", "awaiting your input", "high"],
|
||||
["fallback-used", "Fallback model used for FN-1", "switched from", "high"],
|
||||
])("maps %s event correctly", async (event, expectedTitle, messagePart, priority) => {
|
||||
await provider.sendNotification(event as any, { taskId: "FN-1", taskTitle: "T", event: event as any });
|
||||
|
||||
@@ -62,6 +63,7 @@ describe("NtfyNotificationProvider", () => {
|
||||
expect(provider.isEventSupported("awaiting-approval" as any)).toBe(true);
|
||||
expect(provider.isEventSupported("awaiting-user-review" as any)).toBe(true);
|
||||
expect(provider.isEventSupported("planning-awaiting-input" as any)).toBe(true);
|
||||
expect(provider.isEventSupported("fallback-used" as any)).toBe(true);
|
||||
expect(provider.isEventSupported("custom-event" as any)).toBe(false);
|
||||
});
|
||||
|
||||
|
||||
@@ -681,6 +681,93 @@ describe("piLog structured diagnostics", () => {
|
||||
expect(hasModelLog).toBe(true);
|
||||
});
|
||||
|
||||
it("fires fallback hook on session-creation fallback", async () => {
|
||||
const createAgentSessionMock = vi.mocked(createAgentSession);
|
||||
const onFallbackModelUsed = vi.fn();
|
||||
createAgentSessionMock.mockReset();
|
||||
createAgentSessionMock
|
||||
.mockRejectedValueOnce(new Error("429 Too Many Requests"))
|
||||
.mockResolvedValueOnce({
|
||||
session: {
|
||||
model: { provider: "test", id: "fallback-model" },
|
||||
prompt: vi.fn(),
|
||||
subscribe: vi.fn(),
|
||||
dispose: vi.fn(),
|
||||
setThinkingLevel: vi.fn(),
|
||||
sessionFile: undefined,
|
||||
},
|
||||
} as any);
|
||||
|
||||
await createFnAgent({
|
||||
cwd: "/test/project",
|
||||
systemPrompt: "Test",
|
||||
defaultProvider: "test",
|
||||
defaultModelId: "primary-model",
|
||||
fallbackProvider: "test",
|
||||
fallbackModelId: "fallback-model",
|
||||
taskId: "FN-1",
|
||||
taskTitle: "My Task",
|
||||
onFallbackModelUsed,
|
||||
});
|
||||
|
||||
expect(onFallbackModelUsed).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
triggerPoint: "session-creation",
|
||||
primaryModel: "test/test-model",
|
||||
fallbackModel: "test/test-model",
|
||||
taskId: "FN-1",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("fires fallback hook on prompt-time fallback", async () => {
|
||||
const createAgentSessionMock = vi.mocked(createAgentSession);
|
||||
const onFallbackModelUsed = vi.fn();
|
||||
|
||||
const primarySession = {
|
||||
model: { provider: "test", id: "primary-model" },
|
||||
prompt: vi.fn().mockRejectedValue(new Error("429 Too Many Requests")),
|
||||
subscribe: vi.fn(),
|
||||
dispose: vi.fn(),
|
||||
setThinkingLevel: vi.fn(),
|
||||
sessionFile: undefined,
|
||||
} as unknown as AgentSession;
|
||||
|
||||
const fallbackSession = {
|
||||
model: { provider: "test", id: "fallback-model" },
|
||||
prompt: vi.fn().mockResolvedValue(undefined),
|
||||
subscribe: vi.fn(),
|
||||
dispose: vi.fn(),
|
||||
setThinkingLevel: vi.fn(),
|
||||
sessionFile: undefined,
|
||||
} as unknown as AgentSession;
|
||||
|
||||
createAgentSessionMock.mockReset();
|
||||
createAgentSessionMock
|
||||
.mockResolvedValueOnce({ session: primarySession } as any)
|
||||
.mockResolvedValueOnce({ session: fallbackSession } as any);
|
||||
|
||||
const { session } = await createFnAgent({
|
||||
cwd: "/test/project",
|
||||
systemPrompt: "Test",
|
||||
defaultProvider: "test",
|
||||
defaultModelId: "primary-model",
|
||||
fallbackProvider: "test",
|
||||
fallbackModelId: "fallback-model",
|
||||
taskId: "FN-2",
|
||||
onFallbackModelUsed,
|
||||
});
|
||||
|
||||
await (session as any).promptWithFallback("prompt text");
|
||||
|
||||
expect(onFallbackModelUsed).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
triggerPoint: "prompt-time",
|
||||
taskId: "FN-2",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("logs warning on primary model failure and fallback attempt", async () => {
|
||||
const createAgentSessionMock = vi.mocked(createAgentSession);
|
||||
createAgentSessionMock.mockReset();
|
||||
|
||||
@@ -137,6 +137,7 @@ describe("WebhookNotificationProvider", () => {
|
||||
["awaiting-user-review", "needs human review before it can proceed"],
|
||||
["planning-awaiting-input", "is awaiting your input during planning"],
|
||||
["gridlock", "Pipeline gridlocked"],
|
||||
["fallback-used", "Fusion recovered by switching from"],
|
||||
["unknown-event", 'Event "unknown-event" for task My Task'],
|
||||
])("message formatting for %s", async (event, expectedPart) => {
|
||||
fetchMock.mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
import type { AgentSession, SessionManager, ToolDefinition } from "@mariozechner/pi-coding-agent";
|
||||
import type { SkillSelectionContext } from "./skill-resolver.js";
|
||||
import type { FallbackModelUsedPayload } from "./pi.js";
|
||||
|
||||
/**
|
||||
* Options for creating an agent session.
|
||||
@@ -69,6 +70,11 @@ export interface AgentRuntimeOptions {
|
||||
* the runtime's internal setup latency rather than unbounded.
|
||||
*/
|
||||
beforeSpawnSession?: () => Promise<void> | void;
|
||||
/** Callback fired when runtime falls back from primary model to fallback model. */
|
||||
onFallbackModelUsed?: (payload: FallbackModelUsedPayload) => Promise<void> | void;
|
||||
/** Optional task context for fallback notifications. */
|
||||
taskId?: string;
|
||||
taskTitle?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -55,6 +55,7 @@ import {
|
||||
import { getTaskCompletionBlockerForStore } from "./task-completion.js";
|
||||
import { createFusionAuthStorage, getModelRegistryModelsPath } from "./auth-storage.js";
|
||||
import { createRunVerificationTool } from "./run-verification-tool.js";
|
||||
import { notifyFallbackUsed } from "./notifier.js";
|
||||
|
||||
// Re-export for backward compatibility (tests import from executor.ts)
|
||||
export { summarizeToolArgs } from "./agent-logger.js";
|
||||
@@ -2746,6 +2747,9 @@ export class TaskExecutor {
|
||||
sessionManager,
|
||||
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
||||
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||||
taskId: task.id,
|
||||
taskTitle: detail.title,
|
||||
onFallbackModelUsed: notifyFallbackUsed,
|
||||
});
|
||||
|
||||
if (isResuming) {
|
||||
|
||||
@@ -155,6 +155,7 @@ import {
|
||||
import { describeModel, promptWithFallback } from "./pi.js";
|
||||
import { accumulateSessionTokenUsage } from "./session-token-usage.js";
|
||||
import { createResolvedAgentSession, extractRuntimeHint } from "./agent-session-helpers.js";
|
||||
import { notifyFallbackUsed } from "./notifier.js";
|
||||
import { buildSessionSkillContext } from "./session-skill-context.js";
|
||||
import type { WorktreePool } from "./worktree-pool.js";
|
||||
import { AgentLogger } from "./agent-logger.js";
|
||||
@@ -2336,6 +2337,8 @@ You are assisting with a paused \`git pull --rebase\`.
|
||||
? settings.defaultModelIdOverride
|
||||
: settings.defaultModelId,
|
||||
defaultThinkingLevel: settings.defaultThinkingLevel,
|
||||
taskId,
|
||||
onFallbackModelUsed: notifyFallbackUsed,
|
||||
});
|
||||
|
||||
const prompt = [
|
||||
|
||||
@@ -23,6 +23,7 @@ import type {
|
||||
import { createFnAgent, promptWithFallback, type AgentResult } from "./pi.js";
|
||||
import { createResolvedAgentSession, extractRuntimeHint } from "./agent-session-helpers.js";
|
||||
import { createLogger } from "./logger.js";
|
||||
import { notifyFallbackUsed } from "./notifier.js";
|
||||
|
||||
/** Logger for the mission execution loop subsystem. */
|
||||
export const loopLog = createLogger("mission-loop");
|
||||
@@ -357,6 +358,9 @@ export class MissionExecutionLoop extends EventEmitter {
|
||||
onText: (_delta) => {
|
||||
// Could stream this to a log entry if needed
|
||||
},
|
||||
taskId: task?.id,
|
||||
taskTitle: task?.title,
|
||||
onFallbackModelUsed: notifyFallbackUsed,
|
||||
});
|
||||
session = { session: sessionResult.session, sessionFile: sessionResult.sessionFile };
|
||||
|
||||
|
||||
@@ -228,6 +228,15 @@ export class NotificationService {
|
||||
);
|
||||
}
|
||||
|
||||
async dispatch(eventType: NotificationEvent, payload: NotificationPayload): Promise<void> {
|
||||
if (!this.notificationsEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const dedupTaskId = payload.taskId ?? "global";
|
||||
this.maybeNotify(dedupTaskId, eventType, payload);
|
||||
}
|
||||
|
||||
private createTaskPayload(task: Task, event: NotificationEvent): NotificationPayload {
|
||||
return {
|
||||
taskId: task.id,
|
||||
|
||||
@@ -33,7 +33,8 @@ type SupportedNtfyEvent =
|
||||
| "failed"
|
||||
| "awaiting-approval"
|
||||
| "awaiting-user-review"
|
||||
| "planning-awaiting-input";
|
||||
| "planning-awaiting-input"
|
||||
| "fallback-used";
|
||||
|
||||
const SUPPORTED_EVENTS = new Set<SupportedNtfyEvent>([
|
||||
"in-review",
|
||||
@@ -42,6 +43,7 @@ const SUPPORTED_EVENTS = new Set<SupportedNtfyEvent>([
|
||||
"awaiting-approval",
|
||||
"awaiting-user-review",
|
||||
"planning-awaiting-input",
|
||||
"fallback-used",
|
||||
]);
|
||||
|
||||
export class NtfyNotificationProvider implements NotificationProvider {
|
||||
@@ -92,8 +94,9 @@ export class NtfyNotificationProvider implements NotificationProvider {
|
||||
};
|
||||
}
|
||||
|
||||
const taskId = payload.taskId ?? "unknown-task";
|
||||
const taskLike = {
|
||||
id: payload.taskId,
|
||||
id: taskId,
|
||||
title: payload.taskTitle,
|
||||
description: payload.taskDescription ?? "",
|
||||
} as Pick<Task, "id" | "title" | "description"> as Task;
|
||||
@@ -107,35 +110,40 @@ export class NtfyNotificationProvider implements NotificationProvider {
|
||||
|
||||
const contentByEvent: Record<SupportedNtfyEvent, { title: string; message: string; priority: "default" | "high" }> = {
|
||||
"in-review": {
|
||||
title: `Task ${payload.taskId} completed`,
|
||||
title: `Task ${taskId} completed`,
|
||||
message: `Task "${identifier}" is ready for review`,
|
||||
priority: "default",
|
||||
},
|
||||
merged: {
|
||||
title: `Task ${payload.taskId} merged`,
|
||||
title: `Task ${taskId} merged`,
|
||||
message: `Task "${identifier}" has been merged to main`,
|
||||
priority: "default",
|
||||
},
|
||||
failed: {
|
||||
title: `Task ${payload.taskId} failed`,
|
||||
title: `Task ${taskId} failed`,
|
||||
message: `Task "${identifier}" has failed and needs attention`,
|
||||
priority: "high",
|
||||
},
|
||||
"awaiting-approval": {
|
||||
title: `Plan needs approval for ${payload.taskId}`,
|
||||
title: `Plan needs approval for ${taskId}`,
|
||||
message: `Task "${identifier}" needs your approval before it can proceed`,
|
||||
priority: "high",
|
||||
},
|
||||
"awaiting-user-review": {
|
||||
title: `User review needed for ${payload.taskId}`,
|
||||
title: `User review needed for ${taskId}`,
|
||||
message: `Task "${identifier}" needs human review before it can proceed`,
|
||||
priority: "high",
|
||||
},
|
||||
"planning-awaiting-input": {
|
||||
title: `Planning input needed for ${payload.taskId}`,
|
||||
title: `Planning input needed for ${taskId}`,
|
||||
message: `Task "${identifier}" is awaiting your input during planning`,
|
||||
priority: "high",
|
||||
},
|
||||
"fallback-used": {
|
||||
title: `Fallback model used${payload.taskId ? ` for ${payload.taskId}` : ""}`,
|
||||
message: `Fusion switched from ${String(payload.metadata?.primaryModel ?? "primary model")} to ${String(payload.metadata?.fallbackModel ?? "fallback model")} after a retryable failure (${String(payload.metadata?.triggerPoint ?? "unknown trigger")}).`,
|
||||
priority: "high",
|
||||
},
|
||||
};
|
||||
|
||||
const content = contentByEvent[event as SupportedNtfyEvent];
|
||||
|
||||
@@ -133,6 +133,8 @@ export class WebhookNotificationProvider implements NotificationProvider {
|
||||
return `Task "${identifier}" is awaiting your input during planning`;
|
||||
case "gridlock":
|
||||
return "Pipeline gridlocked";
|
||||
case "fallback-used":
|
||||
return `Fusion recovered by switching from ${String(payload.metadata?.primaryModel ?? "primary model")} to ${String(payload.metadata?.fallbackModel ?? "fallback model")} (${String(payload.metadata?.triggerPoint ?? "unknown trigger")})`;
|
||||
default:
|
||||
return `Event "${event}" for task ${identifier}`;
|
||||
}
|
||||
@@ -145,7 +147,7 @@ export class WebhookNotificationProvider implements NotificationProvider {
|
||||
|
||||
const description = payload.taskDescription ?? "";
|
||||
const snippet = description.length > 200 ? `${description.slice(0, 200)}...` : description;
|
||||
return `${payload.taskId}: ${snippet}`;
|
||||
return `${payload.taskId ?? "unknown-task"}: ${snippet}`;
|
||||
}
|
||||
|
||||
private formatPayload(payload: NotificationPayload, message: string): Record<string, unknown> {
|
||||
|
||||
@@ -23,6 +23,7 @@ export const DEFAULT_NTFY_EVENTS: readonly NtfyNotificationEvent[] = [
|
||||
"awaiting-user-review",
|
||||
"planning-awaiting-input",
|
||||
"gridlock",
|
||||
"fallback-used",
|
||||
] as const;
|
||||
|
||||
export interface NtfyNotificationConfigInput {
|
||||
@@ -53,7 +54,7 @@ interface NtfyConfig {
|
||||
|
||||
/** Event types for task notification deduplication */
|
||||
type TaskNotificationEvent = "in-review" | "merged" | "failed" | "awaiting-approval" | "awaiting-user-review";
|
||||
type AnyNotificationEvent = TaskNotificationEvent | "gridlock";
|
||||
type AnyNotificationEvent = TaskNotificationEvent | "gridlock" | "fallback-used";
|
||||
|
||||
/**
|
||||
* Format a task identifier for notifications.
|
||||
@@ -166,6 +167,35 @@ export async function sendNtfyNotification({
|
||||
* It keeps legacy APIs (getConfig, notifyGridlock) while delegating task event
|
||||
* notifications to the pluggable provider-based notification module.
|
||||
*/
|
||||
let activeNotificationService: NotificationService | undefined;
|
||||
|
||||
export interface FallbackNotificationInput {
|
||||
primaryModel: string;
|
||||
fallbackModel: string;
|
||||
triggerPoint: "session-creation" | "prompt-time";
|
||||
taskId?: string;
|
||||
taskTitle?: string;
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
export async function notifyFallbackUsed(input: FallbackNotificationInput): Promise<void> {
|
||||
if (!activeNotificationService) {
|
||||
return;
|
||||
}
|
||||
|
||||
await activeNotificationService.dispatch("fallback-used", {
|
||||
taskId: input.taskId,
|
||||
taskTitle: input.taskTitle,
|
||||
event: "fallback-used",
|
||||
timestamp: input.timestamp,
|
||||
metadata: {
|
||||
primaryModel: input.primaryModel,
|
||||
fallbackModel: input.fallbackModel,
|
||||
triggerPoint: input.triggerPoint,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export class NtfyNotifier {
|
||||
private config: NtfyConfig = {
|
||||
enabled: false,
|
||||
@@ -192,6 +222,7 @@ export class NtfyNotifier {
|
||||
projectId: this.projectId,
|
||||
ntfyBaseUrl: options.ntfyBaseUrl,
|
||||
});
|
||||
activeNotificationService = this.notificationService;
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
|
||||
@@ -393,6 +393,15 @@ export async function compactSessionContext(
|
||||
}
|
||||
}
|
||||
|
||||
export interface FallbackModelUsedPayload {
|
||||
primaryModel: string;
|
||||
fallbackModel: string;
|
||||
triggerPoint: "session-creation" | "prompt-time";
|
||||
taskId?: string;
|
||||
taskTitle?: string;
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
export interface AgentOptions {
|
||||
cwd: string;
|
||||
systemPrompt: string;
|
||||
@@ -429,6 +438,11 @@ export interface AgentOptions {
|
||||
/** Last-chance abort hook fired immediately before `createAgentSession`.
|
||||
* See `AgentRuntimeOptions.beforeSpawnSession`. */
|
||||
beforeSpawnSession?: () => Promise<void> | void;
|
||||
/** Callback fired when runtime falls back from primary model to fallback model. */
|
||||
onFallbackModelUsed?: (payload: FallbackModelUsedPayload) => Promise<void> | void;
|
||||
/** Optional task context for fallback notifications. */
|
||||
taskId?: string;
|
||||
taskTitle?: string;
|
||||
}
|
||||
|
||||
function resolveConfiguredModel(
|
||||
@@ -1204,6 +1218,20 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
|
||||
});
|
||||
};
|
||||
|
||||
const emitFallbackUsed = async (triggerPoint: "session-creation" | "prompt-time"): Promise<void> => {
|
||||
if (!options.onFallbackModelUsed || !selectedModel || !fallbackModel) {
|
||||
return;
|
||||
}
|
||||
await options.onFallbackModelUsed({
|
||||
primaryModel: `${selectedModel.provider}/${selectedModel.id}`,
|
||||
fallbackModel: `${fallbackModel.provider}/${fallbackModel.id}`,
|
||||
triggerPoint,
|
||||
taskId: options.taskId,
|
||||
taskTitle: options.taskTitle,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
};
|
||||
|
||||
let sessionResult;
|
||||
let usingFallback = false;
|
||||
try {
|
||||
@@ -1217,6 +1245,7 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
|
||||
piLog.warn(`Primary model failed (${err.message}), trying fallback`);
|
||||
usingFallback = true;
|
||||
sessionResult = await createSessionWithModel(fallbackModel);
|
||||
await emitFallbackUsed("session-creation");
|
||||
piLog.log("Fallback session created successfully");
|
||||
}
|
||||
|
||||
@@ -1278,6 +1307,7 @@ export async function createFnAgent(options: AgentOptions): Promise<AgentResult>
|
||||
}
|
||||
|
||||
const fallbackSessionResult = await createSessionWithModel(fallbackModel);
|
||||
await emitFallbackUsed("prompt-time");
|
||||
const fallbackSession = fallbackSessionResult.session as PromptableSession;
|
||||
installToolResultContentGuard(fallbackSession as unknown as AgentToolHookSession);
|
||||
installMessageContentGuard(
|
||||
|
||||
@@ -17,6 +17,7 @@ import { AgentLogger } from "./agent-logger.js";
|
||||
import { reviewerLog } from "./logger.js";
|
||||
import { checkSessionError } from "./usage-limit-detector.js";
|
||||
import { resolveAgentInstructions, buildSystemPromptWithInstructions } from "./agent-instructions.js";
|
||||
import { notifyFallbackUsed } from "./notifier.js";
|
||||
import { createMemoryGetTool, createMemorySearchTool } from "./agent-tools.js";
|
||||
|
||||
export const REVIEWER_SYSTEM_PROMPT = `You are an independent code and plan reviewer.
|
||||
@@ -254,6 +255,8 @@ export interface ReviewOptions {
|
||||
store?: TaskStore;
|
||||
/** Task ID for agent log persistence. Required alongside `store`. */
|
||||
taskId?: string;
|
||||
/** Optional task title for fallback-used notification context. */
|
||||
taskTitle?: string;
|
||||
/** Task with optional assignedAgentId for skill selection. */
|
||||
task?: { assignedAgentId?: string | null };
|
||||
/** User comments on the task (author === "user"). For spec reviews, the reviewer explicitly checks that every comment is addressed. */
|
||||
@@ -488,6 +491,9 @@ export async function reviewStep(
|
||||
defaultThinkingLevel: options.defaultThinkingLevel,
|
||||
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
||||
...(skillContext?.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||||
taskId: options.taskId,
|
||||
taskTitle: options.taskTitle,
|
||||
onFallbackModelUsed: notifyFallbackUsed,
|
||||
beforeSpawnSession: async () => {
|
||||
if (!options.store) return;
|
||||
let finalSettings: Settings | undefined;
|
||||
|
||||
@@ -30,6 +30,7 @@ import { AgentSemaphore } from "./concurrency.js";
|
||||
import { StuckTaskDetector } from "./stuck-task-detector.js";
|
||||
import { AgentLogger } from "./agent-logger.js";
|
||||
import { createLogger } from "./logger.js";
|
||||
import { notifyFallbackUsed } from "./notifier.js";
|
||||
import { isContextLimitError } from "./context-limit-detector.js";
|
||||
import { checkSessionError } from "./usage-limit-detector.js";
|
||||
import {
|
||||
@@ -1012,6 +1013,9 @@ Follow instructions precisely and avoid unrelated changes.`,
|
||||
},
|
||||
// Skill selection from step-session executor options
|
||||
...(this.options.skillSelection ? { skillSelection: this.options.skillSelection } : {}),
|
||||
taskId: taskDetail.id,
|
||||
taskTitle: taskDetail.title,
|
||||
onFallbackModelUsed: notifyFallbackUsed,
|
||||
});
|
||||
session = createResult.session;
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ import { buildSessionSkillContext } from "./session-skill-context.js";
|
||||
import { PRIORITY_SPECIFY, type AgentSemaphore } from "./concurrency.js";
|
||||
import { AgentLogger } from "./agent-logger.js";
|
||||
import { resolveAgentInstructions, buildSystemPromptWithInstructions } from "./agent-instructions.js";
|
||||
import { notifyFallbackUsed } from "./notifier.js";
|
||||
import { planLog, reviewerLog, formatError } from "./logger.js";
|
||||
import {
|
||||
isUsageLimitError,
|
||||
@@ -1029,6 +1030,9 @@ export class TriageProcessor {
|
||||
defaultThinkingLevel: settings.defaultThinkingLevel,
|
||||
// Skill selection: use assigned agent skills if available, otherwise role fallback
|
||||
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||||
taskId: task.id,
|
||||
taskTitle: task.title,
|
||||
onFallbackModelUsed: notifyFallbackUsed,
|
||||
});
|
||||
|
||||
const modelDesc = describeModel(session);
|
||||
@@ -1226,6 +1230,9 @@ export class TriageProcessor {
|
||||
defaultModelId: planningFallbackModelId,
|
||||
defaultThinkingLevel: settings.defaultThinkingLevel,
|
||||
...(skillContext.skillSelectionContext ? { skillSelection: skillContext.skillSelectionContext } : {}),
|
||||
taskId: task.id,
|
||||
taskTitle: task.title,
|
||||
onFallbackModelUsed: notifyFallbackUsed,
|
||||
});
|
||||
|
||||
session = fallbackResult.session;
|
||||
|
||||
Reference in New Issue
Block a user