feat(FN-2119): merge fusion/fn-2119
This commit is contained in:
5
.changeset/fix-manual-pause-auto-unpause.md
Normal file
5
.changeset/fix-manual-pause-auto-unpause.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@gsxdsm/fusion": patch
|
||||
---
|
||||
|
||||
Fix: manually paused agents no longer get auto-unpaused by the self-healing system. Only rate-limit-triggered pauses are auto-unpaused.
|
||||
@@ -73,6 +73,7 @@ Defaults from `DEFAULT_PROJECT_SETTINGS`; key scope from `PROJECT_SETTINGS_KEYS`
|
||||
| Setting | Type | Default | Description |
|
||||
|---|---|---:|---|
|
||||
| `globalPause` | `boolean` | `false` | Hard stop: terminate active engine sessions and pause scheduling immediately. |
|
||||
| `globalPauseReason` | `string` | `undefined` | Optional reason for `globalPause` (`"rate-limit"` for automatic pauses, `"manual"` for user-triggered pauses). Cleared on unpause. |
|
||||
| `enginePaused` | `boolean` | `false` | Soft pause: stop dispatching new work while letting active sessions finish. |
|
||||
| `maxConcurrent` | `number` | `2` | Max concurrent task-lane AI agents (triage, executor, merge). |
|
||||
| `maxTriageConcurrent` | `number` | `2` | Max concurrent triage/specification agents. |
|
||||
@@ -119,7 +120,7 @@ Defaults from `DEFAULT_PROJECT_SETTINGS`; key scope from `PROJECT_SETTINGS_KEYS`
|
||||
| `taskStuckTimeoutMs` | `number` | `undefined` | Inactivity timeout for stuck-task recovery. |
|
||||
| `aiSessionTtlMs` | `number` | `604800000` | TTL in ms for persisted planning/subtask/mission sessions (7 days). |
|
||||
| `aiSessionCleanupIntervalMs` | `number` | `3600000` | Interval in ms for AI session cleanup sweeps (1 hour). |
|
||||
| `autoUnpauseEnabled` | `boolean` | `true` | Auto-unpause after rate-limit-triggered pauses. |
|
||||
| `autoUnpauseEnabled` | `boolean` | `true` | Auto-unpause after rate-limit-triggered pauses; manual pauses stay paused until explicitly unpaused by the user. |
|
||||
| `autoUnpauseBaseDelayMs` | `number` | `300000` | Base unpause delay in ms (5 min). |
|
||||
| `autoUnpauseMaxDelayMs` | `number` | `3600000` | Max auto-unpause delay in ms (1 hour). |
|
||||
| `maxStuckKills` | `number` | `6` | Max stuck-task terminations before permanent failure. |
|
||||
|
||||
@@ -52,6 +52,7 @@ export const DEFAULT_GLOBAL_SETTINGS = {
|
||||
/** Default values for project-level settings. */
|
||||
export const DEFAULT_PROJECT_SETTINGS = {
|
||||
globalPause: false,
|
||||
globalPauseReason: undefined,
|
||||
enginePaused: false,
|
||||
maxConcurrent: 2,
|
||||
maxTriageConcurrent: 2,
|
||||
|
||||
@@ -1007,6 +1007,9 @@ export interface ProjectSettings {
|
||||
* global emergency stop for the entire AI engine.
|
||||
* Individual per-task pause flags are unaffected. */
|
||||
globalPause?: boolean;
|
||||
/** Tracks why globalPause was activated. "rate-limit" for automatic pauses,
|
||||
* "manual" for user-initiated. Cleared on unpause. */
|
||||
globalPauseReason?: string;
|
||||
/** Engine pause (soft pause): when true, the scheduler and triage
|
||||
* processor stop dispatching **new** work (scheduling, triage
|
||||
* specification, and auto-merge), but currently running agent sessions
|
||||
|
||||
@@ -960,8 +960,11 @@ describe("App global pause (hard stop)", () => {
|
||||
expect(screen.getByTitle("Start AI engine")).toBeTruthy();
|
||||
});
|
||||
|
||||
// Should call updateSettings with globalPause: true
|
||||
expect(updateSettings).toHaveBeenCalledWith({ globalPause: true }, "proj_123");
|
||||
// Should call updateSettings with globalPause and manual reason
|
||||
expect(updateSettings).toHaveBeenCalledWith(
|
||||
{ globalPause: true, globalPauseReason: "manual" },
|
||||
"proj_123",
|
||||
);
|
||||
});
|
||||
|
||||
it("reverts global pause state on updateSettings failure", async () => {
|
||||
|
||||
@@ -81,7 +81,37 @@ describe("useAppSettings", () => {
|
||||
});
|
||||
|
||||
expect(result.current.globalPaused).toBe(true);
|
||||
expect(mockUpdateSettings).toHaveBeenCalledWith({ globalPause: false }, "proj_123");
|
||||
expect(mockUpdateSettings).toHaveBeenCalledWith(
|
||||
{ globalPause: false, globalPauseReason: undefined },
|
||||
"proj_123",
|
||||
);
|
||||
});
|
||||
|
||||
it("sets globalPauseReason to manual when pausing", async () => {
|
||||
mockFetchSettings.mockResolvedValueOnce({
|
||||
autoMerge: false,
|
||||
globalPause: false,
|
||||
enginePaused: false,
|
||||
githubTokenConfigured: true,
|
||||
taskStuckTimeoutMs: 600000,
|
||||
showQuickChatFAB: false,
|
||||
} as never);
|
||||
|
||||
const { result } = renderHook(() => useAppSettings("proj_123"));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.globalPaused).toBe(false);
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
await result.current.toggleGlobalPause();
|
||||
});
|
||||
|
||||
expect(result.current.globalPaused).toBe(true);
|
||||
expect(mockUpdateSettings).toHaveBeenCalledWith(
|
||||
{ globalPause: true, globalPauseReason: "manual" },
|
||||
"proj_123",
|
||||
);
|
||||
});
|
||||
|
||||
it("refresh() re-fetches and updates state", async () => {
|
||||
|
||||
@@ -93,7 +93,13 @@ export function useAppSettings(projectId?: string): UseAppSettingsResult {
|
||||
setGlobalPaused(next);
|
||||
|
||||
try {
|
||||
await updateSettings({ globalPause: next }, projectId);
|
||||
await updateSettings(
|
||||
{
|
||||
globalPause: next,
|
||||
globalPauseReason: next ? "manual" : undefined,
|
||||
},
|
||||
projectId,
|
||||
);
|
||||
} catch {
|
||||
setGlobalPaused(!next);
|
||||
}
|
||||
|
||||
@@ -5459,7 +5459,10 @@ describe("TaskExecutor usage limit detection", () => {
|
||||
"FN-001",
|
||||
"rate_limit_error: Rate limit exceeded",
|
||||
);
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true });
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||
globalPause: true,
|
||||
globalPauseReason: "rate-limit",
|
||||
});
|
||||
// Task should still be marked as failed
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { status: "failed", error: "rate_limit_error: Rate limit exceeded" });
|
||||
expect(onError).toHaveBeenCalled();
|
||||
|
||||
@@ -720,7 +720,10 @@ describe("aiMergeTask — usage limit detection", () => {
|
||||
"FN-050",
|
||||
"rate_limit_error: Rate limit exceeded",
|
||||
);
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true });
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||
globalPause: true,
|
||||
globalPauseReason: "rate-limit",
|
||||
});
|
||||
});
|
||||
|
||||
it("triggers global pause when session.prompt() resolves with exhausted-retry error on state.error", async () => {
|
||||
|
||||
@@ -142,7 +142,48 @@ describe("SelfHealingManager", () => {
|
||||
// ── Auto-unpause ─────────────────────────────────────────────────
|
||||
|
||||
describe("auto-unpause", () => {
|
||||
it("schedules unpause when globalPause transitions false→true", async () => {
|
||||
it("does not schedule unpause when globalPauseReason is 'manual'", async () => {
|
||||
manager.start();
|
||||
|
||||
store.emit("settings:updated", {
|
||||
settings: {
|
||||
globalPause: true,
|
||||
globalPauseReason: "manual",
|
||||
autoUnpauseEnabled: true,
|
||||
autoUnpauseBaseDelayMs: 100,
|
||||
autoUnpauseMaxDelayMs: 800,
|
||||
},
|
||||
previous: { globalPause: false },
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
|
||||
expect(store.updateSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("auto-unpauses when globalPauseReason is 'rate-limit'", async () => {
|
||||
manager.start();
|
||||
|
||||
store.emit("settings:updated", {
|
||||
settings: {
|
||||
globalPause: true,
|
||||
globalPauseReason: "rate-limit",
|
||||
autoUnpauseEnabled: true,
|
||||
autoUnpauseBaseDelayMs: 100,
|
||||
autoUnpauseMaxDelayMs: 800,
|
||||
},
|
||||
previous: { globalPause: false },
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(150);
|
||||
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||
globalPause: false,
|
||||
globalPauseReason: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("auto-unpauses when globalPauseReason is undefined (backward compat)", async () => {
|
||||
manager.start();
|
||||
|
||||
store.emit("settings:updated", {
|
||||
@@ -152,7 +193,10 @@ describe("SelfHealingManager", () => {
|
||||
|
||||
await vi.advanceTimersByTimeAsync(150);
|
||||
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: false });
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||
globalPause: false,
|
||||
globalPauseReason: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not schedule unpause when autoUnpauseEnabled is false", async () => {
|
||||
|
||||
@@ -205,6 +205,11 @@ export class SelfHealingManager {
|
||||
return;
|
||||
}
|
||||
|
||||
if (settings.globalPauseReason === "manual") {
|
||||
log.log("Global pause activated manually — auto-unpause skipped, requires manual intervention");
|
||||
return;
|
||||
}
|
||||
|
||||
// If pause re-triggered within 60s of our last unpause, escalate backoff
|
||||
if (this.lastUnpauseAt && (Date.now() - this.lastUnpauseAt) < 60_000) {
|
||||
this.unpauseAttempt++;
|
||||
@@ -258,7 +263,7 @@ export class SelfHealingManager {
|
||||
|
||||
log.warn("Auto-unpause: clearing globalPause");
|
||||
this.lastUnpauseAt = Date.now();
|
||||
await this.store.updateSettings({ globalPause: false });
|
||||
await this.store.updateSettings({ globalPause: false, globalPauseReason: undefined });
|
||||
|
||||
// Note: if the rate limit is still active, the next agent session will
|
||||
// hit it again → UsageLimitPauser triggers globalPause → our listener
|
||||
|
||||
@@ -141,13 +141,16 @@ describe("UsageLimitPauser", () => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("calls store.updateSettings({ globalPause: true }) on usage limit hit", async () => {
|
||||
it("calls store.updateSettings({ globalPause: true, globalPauseReason: \"rate-limit\" }) on usage limit hit", async () => {
|
||||
const store = createMockStore();
|
||||
const pauser = new UsageLimitPauser(store);
|
||||
|
||||
await pauser.onUsageLimitHit("executor", "FN-001", "rate_limit_error: Rate limit exceeded");
|
||||
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true });
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||
globalPause: true,
|
||||
globalPauseReason: "rate-limit",
|
||||
});
|
||||
});
|
||||
|
||||
it("logs the triggering error on the task via store.logEntry", async () => {
|
||||
|
||||
@@ -44,7 +44,8 @@ export function isUsageLimitError(errorMessage: string): boolean {
|
||||
|
||||
/**
|
||||
* Lightweight coordinator that agents call when they detect usage-limit errors.
|
||||
* Triggers the global pause mechanism by calling `store.updateSettings({ globalPause: true })`.
|
||||
* Triggers the global pause mechanism by calling
|
||||
* `store.updateSettings({ globalPause: true, globalPauseReason: "rate-limit" })`.
|
||||
*
|
||||
* **Idempotency:** Tracks an internal `paused` flag so that multiple concurrent
|
||||
* agents hitting limits only trigger one pause. The flag resets when `globalPause`
|
||||
@@ -107,7 +108,7 @@ export class UsageLimitPauser {
|
||||
);
|
||||
|
||||
// Activate global pause
|
||||
await this.store.updateSettings({ globalPause: true });
|
||||
await this.store.updateSettings({ globalPause: true, globalPauseReason: "rate-limit" });
|
||||
|
||||
log.warn("⚠ Global pause activated — all automated activity will halt");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user