feat(FN-2119): merge fusion/fn-2119

This commit is contained in:
gsxdsm
2026-04-19 03:21:38 -07:00
parent 3f8a2e9442
commit bbc0ce89be
13 changed files with 122 additions and 14 deletions

View File

@@ -0,0 +1,5 @@
---
"@gsxdsm/fusion": patch
---
Fix: manually paused agents no longer get auto-unpaused by the self-healing system. Only rate-limit-triggered pauses are auto-unpaused.

View File

@@ -73,6 +73,7 @@ Defaults from `DEFAULT_PROJECT_SETTINGS`; key scope from `PROJECT_SETTINGS_KEYS`
| Setting | Type | Default | Description | | Setting | Type | Default | Description |
|---|---|---:|---| |---|---|---:|---|
| `globalPause` | `boolean` | `false` | Hard stop: terminate active engine sessions and pause scheduling immediately. | | `globalPause` | `boolean` | `false` | Hard stop: terminate active engine sessions and pause scheduling immediately. |
| `globalPauseReason` | `string` | `undefined` | Optional reason for `globalPause` (`"rate-limit"` for automatic pauses, `"manual"` for user-triggered pauses). Cleared on unpause. |
| `enginePaused` | `boolean` | `false` | Soft pause: stop dispatching new work while letting active sessions finish. | | `enginePaused` | `boolean` | `false` | Soft pause: stop dispatching new work while letting active sessions finish. |
| `maxConcurrent` | `number` | `2` | Max concurrent task-lane AI agents (triage, executor, merge). | | `maxConcurrent` | `number` | `2` | Max concurrent task-lane AI agents (triage, executor, merge). |
| `maxTriageConcurrent` | `number` | `2` | Max concurrent triage/specification agents. | | `maxTriageConcurrent` | `number` | `2` | Max concurrent triage/specification agents. |
@@ -119,7 +120,7 @@ Defaults from `DEFAULT_PROJECT_SETTINGS`; key scope from `PROJECT_SETTINGS_KEYS`
| `taskStuckTimeoutMs` | `number` | `undefined` | Inactivity timeout for stuck-task recovery. | | `taskStuckTimeoutMs` | `number` | `undefined` | Inactivity timeout for stuck-task recovery. |
| `aiSessionTtlMs` | `number` | `604800000` | TTL in ms for persisted planning/subtask/mission sessions (7 days). | | `aiSessionTtlMs` | `number` | `604800000` | TTL in ms for persisted planning/subtask/mission sessions (7 days). |
| `aiSessionCleanupIntervalMs` | `number` | `3600000` | Interval in ms for AI session cleanup sweeps (1 hour). | | `aiSessionCleanupIntervalMs` | `number` | `3600000` | Interval in ms for AI session cleanup sweeps (1 hour). |
| `autoUnpauseEnabled` | `boolean` | `true` | Auto-unpause after rate-limit-triggered pauses. | | `autoUnpauseEnabled` | `boolean` | `true` | Auto-unpause after rate-limit-triggered pauses; manual pauses stay paused until explicitly unpaused by the user. |
| `autoUnpauseBaseDelayMs` | `number` | `300000` | Base unpause delay in ms (5 min). | | `autoUnpauseBaseDelayMs` | `number` | `300000` | Base unpause delay in ms (5 min). |
| `autoUnpauseMaxDelayMs` | `number` | `3600000` | Max auto-unpause delay in ms (1 hour). | | `autoUnpauseMaxDelayMs` | `number` | `3600000` | Max auto-unpause delay in ms (1 hour). |
| `maxStuckKills` | `number` | `6` | Max stuck-task terminations before permanent failure. | | `maxStuckKills` | `number` | `6` | Max stuck-task terminations before permanent failure. |

View File

@@ -52,6 +52,7 @@ export const DEFAULT_GLOBAL_SETTINGS = {
/** Default values for project-level settings. */ /** Default values for project-level settings. */
export const DEFAULT_PROJECT_SETTINGS = { export const DEFAULT_PROJECT_SETTINGS = {
globalPause: false, globalPause: false,
globalPauseReason: undefined,
enginePaused: false, enginePaused: false,
maxConcurrent: 2, maxConcurrent: 2,
maxTriageConcurrent: 2, maxTriageConcurrent: 2,

View File

@@ -1007,6 +1007,9 @@ export interface ProjectSettings {
* global emergency stop for the entire AI engine. * global emergency stop for the entire AI engine.
* Individual per-task pause flags are unaffected. */ * Individual per-task pause flags are unaffected. */
globalPause?: boolean; globalPause?: boolean;
/** Tracks why globalPause was activated. "rate-limit" for automatic pauses,
* "manual" for user-initiated. Cleared on unpause. */
globalPauseReason?: string;
/** Engine pause (soft pause): when true, the scheduler and triage /** Engine pause (soft pause): when true, the scheduler and triage
* processor stop dispatching **new** work (scheduling, triage * processor stop dispatching **new** work (scheduling, triage
* specification, and auto-merge), but currently running agent sessions * specification, and auto-merge), but currently running agent sessions

View File

@@ -960,8 +960,11 @@ describe("App global pause (hard stop)", () => {
expect(screen.getByTitle("Start AI engine")).toBeTruthy(); expect(screen.getByTitle("Start AI engine")).toBeTruthy();
}); });
// Should call updateSettings with globalPause: true // Should call updateSettings with globalPause and manual reason
expect(updateSettings).toHaveBeenCalledWith({ globalPause: true }, "proj_123"); expect(updateSettings).toHaveBeenCalledWith(
{ globalPause: true, globalPauseReason: "manual" },
"proj_123",
);
}); });
it("reverts global pause state on updateSettings failure", async () => { it("reverts global pause state on updateSettings failure", async () => {

View File

@@ -81,7 +81,37 @@ describe("useAppSettings", () => {
}); });
expect(result.current.globalPaused).toBe(true); expect(result.current.globalPaused).toBe(true);
expect(mockUpdateSettings).toHaveBeenCalledWith({ globalPause: false }, "proj_123"); expect(mockUpdateSettings).toHaveBeenCalledWith(
{ globalPause: false, globalPauseReason: undefined },
"proj_123",
);
});
it("sets globalPauseReason to manual when pausing", async () => {
mockFetchSettings.mockResolvedValueOnce({
autoMerge: false,
globalPause: false,
enginePaused: false,
githubTokenConfigured: true,
taskStuckTimeoutMs: 600000,
showQuickChatFAB: false,
} as never);
const { result } = renderHook(() => useAppSettings("proj_123"));
await waitFor(() => {
expect(result.current.globalPaused).toBe(false);
});
await act(async () => {
await result.current.toggleGlobalPause();
});
expect(result.current.globalPaused).toBe(true);
expect(mockUpdateSettings).toHaveBeenCalledWith(
{ globalPause: true, globalPauseReason: "manual" },
"proj_123",
);
}); });
it("refresh() re-fetches and updates state", async () => { it("refresh() re-fetches and updates state", async () => {

View File

@@ -93,7 +93,13 @@ export function useAppSettings(projectId?: string): UseAppSettingsResult {
setGlobalPaused(next); setGlobalPaused(next);
try { try {
await updateSettings({ globalPause: next }, projectId); await updateSettings(
{
globalPause: next,
globalPauseReason: next ? "manual" : undefined,
},
projectId,
);
} catch { } catch {
setGlobalPaused(!next); setGlobalPaused(!next);
} }

View File

@@ -5459,7 +5459,10 @@ describe("TaskExecutor usage limit detection", () => {
"FN-001", "FN-001",
"rate_limit_error: Rate limit exceeded", "rate_limit_error: Rate limit exceeded",
); );
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true }); expect(store.updateSettings).toHaveBeenCalledWith({
globalPause: true,
globalPauseReason: "rate-limit",
});
// Task should still be marked as failed // Task should still be marked as failed
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { status: "failed", error: "rate_limit_error: Rate limit exceeded" }); expect(store.updateTask).toHaveBeenCalledWith("FN-001", { status: "failed", error: "rate_limit_error: Rate limit exceeded" });
expect(onError).toHaveBeenCalled(); expect(onError).toHaveBeenCalled();

View File

@@ -720,7 +720,10 @@ describe("aiMergeTask — usage limit detection", () => {
"FN-050", "FN-050",
"rate_limit_error: Rate limit exceeded", "rate_limit_error: Rate limit exceeded",
); );
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true }); expect(store.updateSettings).toHaveBeenCalledWith({
globalPause: true,
globalPauseReason: "rate-limit",
});
}); });
it("triggers global pause when session.prompt() resolves with exhausted-retry error on state.error", async () => { it("triggers global pause when session.prompt() resolves with exhausted-retry error on state.error", async () => {

View File

@@ -142,7 +142,48 @@ describe("SelfHealingManager", () => {
// ── Auto-unpause ───────────────────────────────────────────────── // ── Auto-unpause ─────────────────────────────────────────────────
describe("auto-unpause", () => { describe("auto-unpause", () => {
it("schedules unpause when globalPause transitions false→true", async () => { it("does not schedule unpause when globalPauseReason is 'manual'", async () => {
manager.start();
store.emit("settings:updated", {
settings: {
globalPause: true,
globalPauseReason: "manual",
autoUnpauseEnabled: true,
autoUnpauseBaseDelayMs: 100,
autoUnpauseMaxDelayMs: 800,
},
previous: { globalPause: false },
});
await vi.advanceTimersByTimeAsync(500);
expect(store.updateSettings).not.toHaveBeenCalled();
});
it("auto-unpauses when globalPauseReason is 'rate-limit'", async () => {
manager.start();
store.emit("settings:updated", {
settings: {
globalPause: true,
globalPauseReason: "rate-limit",
autoUnpauseEnabled: true,
autoUnpauseBaseDelayMs: 100,
autoUnpauseMaxDelayMs: 800,
},
previous: { globalPause: false },
});
await vi.advanceTimersByTimeAsync(150);
expect(store.updateSettings).toHaveBeenCalledWith({
globalPause: false,
globalPauseReason: undefined,
});
});
it("auto-unpauses when globalPauseReason is undefined (backward compat)", async () => {
manager.start(); manager.start();
store.emit("settings:updated", { store.emit("settings:updated", {
@@ -152,7 +193,10 @@ describe("SelfHealingManager", () => {
await vi.advanceTimersByTimeAsync(150); await vi.advanceTimersByTimeAsync(150);
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: false }); expect(store.updateSettings).toHaveBeenCalledWith({
globalPause: false,
globalPauseReason: undefined,
});
}); });
it("does not schedule unpause when autoUnpauseEnabled is false", async () => { it("does not schedule unpause when autoUnpauseEnabled is false", async () => {

View File

@@ -205,6 +205,11 @@ export class SelfHealingManager {
return; return;
} }
if (settings.globalPauseReason === "manual") {
log.log("Global pause activated manually — auto-unpause skipped, requires manual intervention");
return;
}
// If pause re-triggered within 60s of our last unpause, escalate backoff // If pause re-triggered within 60s of our last unpause, escalate backoff
if (this.lastUnpauseAt && (Date.now() - this.lastUnpauseAt) < 60_000) { if (this.lastUnpauseAt && (Date.now() - this.lastUnpauseAt) < 60_000) {
this.unpauseAttempt++; this.unpauseAttempt++;
@@ -258,7 +263,7 @@ export class SelfHealingManager {
log.warn("Auto-unpause: clearing globalPause"); log.warn("Auto-unpause: clearing globalPause");
this.lastUnpauseAt = Date.now(); this.lastUnpauseAt = Date.now();
await this.store.updateSettings({ globalPause: false }); await this.store.updateSettings({ globalPause: false, globalPauseReason: undefined });
// Note: if the rate limit is still active, the next agent session will // Note: if the rate limit is still active, the next agent session will
// hit it again → UsageLimitPauser triggers globalPause → our listener // hit it again → UsageLimitPauser triggers globalPause → our listener

View File

@@ -141,13 +141,16 @@ describe("UsageLimitPauser", () => {
vi.clearAllMocks(); vi.clearAllMocks();
}); });
it("calls store.updateSettings({ globalPause: true }) on usage limit hit", async () => { it("calls store.updateSettings({ globalPause: true, globalPauseReason: \"rate-limit\" }) on usage limit hit", async () => {
const store = createMockStore(); const store = createMockStore();
const pauser = new UsageLimitPauser(store); const pauser = new UsageLimitPauser(store);
await pauser.onUsageLimitHit("executor", "FN-001", "rate_limit_error: Rate limit exceeded"); await pauser.onUsageLimitHit("executor", "FN-001", "rate_limit_error: Rate limit exceeded");
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true }); expect(store.updateSettings).toHaveBeenCalledWith({
globalPause: true,
globalPauseReason: "rate-limit",
});
}); });
it("logs the triggering error on the task via store.logEntry", async () => { it("logs the triggering error on the task via store.logEntry", async () => {

View File

@@ -44,7 +44,8 @@ export function isUsageLimitError(errorMessage: string): boolean {
/** /**
* Lightweight coordinator that agents call when they detect usage-limit errors. * Lightweight coordinator that agents call when they detect usage-limit errors.
* Triggers the global pause mechanism by calling `store.updateSettings({ globalPause: true })`. * Triggers the global pause mechanism by calling
* `store.updateSettings({ globalPause: true, globalPauseReason: "rate-limit" })`.
* *
* **Idempotency:** Tracks an internal `paused` flag so that multiple concurrent * **Idempotency:** Tracks an internal `paused` flag so that multiple concurrent
* agents hitting limits only trigger one pause. The flag resets when `globalPause` * agents hitting limits only trigger one pause. The flag resets when `globalPause`
@@ -107,7 +108,7 @@ export class UsageLimitPauser {
); );
// Activate global pause // Activate global pause
await this.store.updateSettings({ globalPause: true }); await this.store.updateSettings({ globalPause: true, globalPauseReason: "rate-limit" });
log.warn("⚠ Global pause activated — all automated activity will halt"); log.warn("⚠ Global pause activated — all automated activity will halt");
} }