feat(FN-2119): merge fusion/fn-2119
This commit is contained in:
5
.changeset/fix-manual-pause-auto-unpause.md
Normal file
5
.changeset/fix-manual-pause-auto-unpause.md
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
---
|
||||||
|
"@gsxdsm/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
Fix: manually paused agents no longer get auto-unpaused by the self-healing system. Only rate-limit-triggered pauses are auto-unpaused.
|
||||||
@@ -73,6 +73,7 @@ Defaults from `DEFAULT_PROJECT_SETTINGS`; key scope from `PROJECT_SETTINGS_KEYS`
|
|||||||
| Setting | Type | Default | Description |
|
| Setting | Type | Default | Description |
|
||||||
|---|---|---:|---|
|
|---|---|---:|---|
|
||||||
| `globalPause` | `boolean` | `false` | Hard stop: terminate active engine sessions and pause scheduling immediately. |
|
| `globalPause` | `boolean` | `false` | Hard stop: terminate active engine sessions and pause scheduling immediately. |
|
||||||
|
| `globalPauseReason` | `string` | `undefined` | Optional reason for `globalPause` (`"rate-limit"` for automatic pauses, `"manual"` for user-triggered pauses). Cleared on unpause. |
|
||||||
| `enginePaused` | `boolean` | `false` | Soft pause: stop dispatching new work while letting active sessions finish. |
|
| `enginePaused` | `boolean` | `false` | Soft pause: stop dispatching new work while letting active sessions finish. |
|
||||||
| `maxConcurrent` | `number` | `2` | Max concurrent task-lane AI agents (triage, executor, merge). |
|
| `maxConcurrent` | `number` | `2` | Max concurrent task-lane AI agents (triage, executor, merge). |
|
||||||
| `maxTriageConcurrent` | `number` | `2` | Max concurrent triage/specification agents. |
|
| `maxTriageConcurrent` | `number` | `2` | Max concurrent triage/specification agents. |
|
||||||
@@ -119,7 +120,7 @@ Defaults from `DEFAULT_PROJECT_SETTINGS`; key scope from `PROJECT_SETTINGS_KEYS`
|
|||||||
| `taskStuckTimeoutMs` | `number` | `undefined` | Inactivity timeout for stuck-task recovery. |
|
| `taskStuckTimeoutMs` | `number` | `undefined` | Inactivity timeout for stuck-task recovery. |
|
||||||
| `aiSessionTtlMs` | `number` | `604800000` | TTL in ms for persisted planning/subtask/mission sessions (7 days). |
|
| `aiSessionTtlMs` | `number` | `604800000` | TTL in ms for persisted planning/subtask/mission sessions (7 days). |
|
||||||
| `aiSessionCleanupIntervalMs` | `number` | `3600000` | Interval in ms for AI session cleanup sweeps (1 hour). |
|
| `aiSessionCleanupIntervalMs` | `number` | `3600000` | Interval in ms for AI session cleanup sweeps (1 hour). |
|
||||||
| `autoUnpauseEnabled` | `boolean` | `true` | Auto-unpause after rate-limit-triggered pauses. |
|
| `autoUnpauseEnabled` | `boolean` | `true` | Auto-unpause after rate-limit-triggered pauses; manual pauses stay paused until explicitly unpaused by the user. |
|
||||||
| `autoUnpauseBaseDelayMs` | `number` | `300000` | Base unpause delay in ms (5 min). |
|
| `autoUnpauseBaseDelayMs` | `number` | `300000` | Base unpause delay in ms (5 min). |
|
||||||
| `autoUnpauseMaxDelayMs` | `number` | `3600000` | Max auto-unpause delay in ms (1 hour). |
|
| `autoUnpauseMaxDelayMs` | `number` | `3600000` | Max auto-unpause delay in ms (1 hour). |
|
||||||
| `maxStuckKills` | `number` | `6` | Max stuck-task terminations before permanent failure. |
|
| `maxStuckKills` | `number` | `6` | Max stuck-task terminations before permanent failure. |
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ export const DEFAULT_GLOBAL_SETTINGS = {
|
|||||||
/** Default values for project-level settings. */
|
/** Default values for project-level settings. */
|
||||||
export const DEFAULT_PROJECT_SETTINGS = {
|
export const DEFAULT_PROJECT_SETTINGS = {
|
||||||
globalPause: false,
|
globalPause: false,
|
||||||
|
globalPauseReason: undefined,
|
||||||
enginePaused: false,
|
enginePaused: false,
|
||||||
maxConcurrent: 2,
|
maxConcurrent: 2,
|
||||||
maxTriageConcurrent: 2,
|
maxTriageConcurrent: 2,
|
||||||
|
|||||||
@@ -1007,6 +1007,9 @@ export interface ProjectSettings {
|
|||||||
* global emergency stop for the entire AI engine.
|
* global emergency stop for the entire AI engine.
|
||||||
* Individual per-task pause flags are unaffected. */
|
* Individual per-task pause flags are unaffected. */
|
||||||
globalPause?: boolean;
|
globalPause?: boolean;
|
||||||
|
/** Tracks why globalPause was activated. "rate-limit" for automatic pauses,
|
||||||
|
* "manual" for user-initiated. Cleared on unpause. */
|
||||||
|
globalPauseReason?: string;
|
||||||
/** Engine pause (soft pause): when true, the scheduler and triage
|
/** Engine pause (soft pause): when true, the scheduler and triage
|
||||||
* processor stop dispatching **new** work (scheduling, triage
|
* processor stop dispatching **new** work (scheduling, triage
|
||||||
* specification, and auto-merge), but currently running agent sessions
|
* specification, and auto-merge), but currently running agent sessions
|
||||||
|
|||||||
@@ -960,8 +960,11 @@ describe("App global pause (hard stop)", () => {
|
|||||||
expect(screen.getByTitle("Start AI engine")).toBeTruthy();
|
expect(screen.getByTitle("Start AI engine")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
// Should call updateSettings with globalPause: true
|
// Should call updateSettings with globalPause and manual reason
|
||||||
expect(updateSettings).toHaveBeenCalledWith({ globalPause: true }, "proj_123");
|
expect(updateSettings).toHaveBeenCalledWith(
|
||||||
|
{ globalPause: true, globalPauseReason: "manual" },
|
||||||
|
"proj_123",
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("reverts global pause state on updateSettings failure", async () => {
|
it("reverts global pause state on updateSettings failure", async () => {
|
||||||
|
|||||||
@@ -81,7 +81,37 @@ describe("useAppSettings", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(result.current.globalPaused).toBe(true);
|
expect(result.current.globalPaused).toBe(true);
|
||||||
expect(mockUpdateSettings).toHaveBeenCalledWith({ globalPause: false }, "proj_123");
|
expect(mockUpdateSettings).toHaveBeenCalledWith(
|
||||||
|
{ globalPause: false, globalPauseReason: undefined },
|
||||||
|
"proj_123",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("sets globalPauseReason to manual when pausing", async () => {
|
||||||
|
mockFetchSettings.mockResolvedValueOnce({
|
||||||
|
autoMerge: false,
|
||||||
|
globalPause: false,
|
||||||
|
enginePaused: false,
|
||||||
|
githubTokenConfigured: true,
|
||||||
|
taskStuckTimeoutMs: 600000,
|
||||||
|
showQuickChatFAB: false,
|
||||||
|
} as never);
|
||||||
|
|
||||||
|
const { result } = renderHook(() => useAppSettings("proj_123"));
|
||||||
|
|
||||||
|
await waitFor(() => {
|
||||||
|
expect(result.current.globalPaused).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
await act(async () => {
|
||||||
|
await result.current.toggleGlobalPause();
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.current.globalPaused).toBe(true);
|
||||||
|
expect(mockUpdateSettings).toHaveBeenCalledWith(
|
||||||
|
{ globalPause: true, globalPauseReason: "manual" },
|
||||||
|
"proj_123",
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("refresh() re-fetches and updates state", async () => {
|
it("refresh() re-fetches and updates state", async () => {
|
||||||
|
|||||||
@@ -93,7 +93,13 @@ export function useAppSettings(projectId?: string): UseAppSettingsResult {
|
|||||||
setGlobalPaused(next);
|
setGlobalPaused(next);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await updateSettings({ globalPause: next }, projectId);
|
await updateSettings(
|
||||||
|
{
|
||||||
|
globalPause: next,
|
||||||
|
globalPauseReason: next ? "manual" : undefined,
|
||||||
|
},
|
||||||
|
projectId,
|
||||||
|
);
|
||||||
} catch {
|
} catch {
|
||||||
setGlobalPaused(!next);
|
setGlobalPaused(!next);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5459,7 +5459,10 @@ describe("TaskExecutor usage limit detection", () => {
|
|||||||
"FN-001",
|
"FN-001",
|
||||||
"rate_limit_error: Rate limit exceeded",
|
"rate_limit_error: Rate limit exceeded",
|
||||||
);
|
);
|
||||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true });
|
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||||
|
globalPause: true,
|
||||||
|
globalPauseReason: "rate-limit",
|
||||||
|
});
|
||||||
// Task should still be marked as failed
|
// Task should still be marked as failed
|
||||||
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { status: "failed", error: "rate_limit_error: Rate limit exceeded" });
|
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { status: "failed", error: "rate_limit_error: Rate limit exceeded" });
|
||||||
expect(onError).toHaveBeenCalled();
|
expect(onError).toHaveBeenCalled();
|
||||||
|
|||||||
@@ -720,7 +720,10 @@ describe("aiMergeTask — usage limit detection", () => {
|
|||||||
"FN-050",
|
"FN-050",
|
||||||
"rate_limit_error: Rate limit exceeded",
|
"rate_limit_error: Rate limit exceeded",
|
||||||
);
|
);
|
||||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true });
|
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||||
|
globalPause: true,
|
||||||
|
globalPauseReason: "rate-limit",
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("triggers global pause when session.prompt() resolves with exhausted-retry error on state.error", async () => {
|
it("triggers global pause when session.prompt() resolves with exhausted-retry error on state.error", async () => {
|
||||||
|
|||||||
@@ -142,7 +142,48 @@ describe("SelfHealingManager", () => {
|
|||||||
// ── Auto-unpause ─────────────────────────────────────────────────
|
// ── Auto-unpause ─────────────────────────────────────────────────
|
||||||
|
|
||||||
describe("auto-unpause", () => {
|
describe("auto-unpause", () => {
|
||||||
it("schedules unpause when globalPause transitions false→true", async () => {
|
it("does not schedule unpause when globalPauseReason is 'manual'", async () => {
|
||||||
|
manager.start();
|
||||||
|
|
||||||
|
store.emit("settings:updated", {
|
||||||
|
settings: {
|
||||||
|
globalPause: true,
|
||||||
|
globalPauseReason: "manual",
|
||||||
|
autoUnpauseEnabled: true,
|
||||||
|
autoUnpauseBaseDelayMs: 100,
|
||||||
|
autoUnpauseMaxDelayMs: 800,
|
||||||
|
},
|
||||||
|
previous: { globalPause: false },
|
||||||
|
});
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(500);
|
||||||
|
|
||||||
|
expect(store.updateSettings).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("auto-unpauses when globalPauseReason is 'rate-limit'", async () => {
|
||||||
|
manager.start();
|
||||||
|
|
||||||
|
store.emit("settings:updated", {
|
||||||
|
settings: {
|
||||||
|
globalPause: true,
|
||||||
|
globalPauseReason: "rate-limit",
|
||||||
|
autoUnpauseEnabled: true,
|
||||||
|
autoUnpauseBaseDelayMs: 100,
|
||||||
|
autoUnpauseMaxDelayMs: 800,
|
||||||
|
},
|
||||||
|
previous: { globalPause: false },
|
||||||
|
});
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(150);
|
||||||
|
|
||||||
|
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||||
|
globalPause: false,
|
||||||
|
globalPauseReason: undefined,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("auto-unpauses when globalPauseReason is undefined (backward compat)", async () => {
|
||||||
manager.start();
|
manager.start();
|
||||||
|
|
||||||
store.emit("settings:updated", {
|
store.emit("settings:updated", {
|
||||||
@@ -152,7 +193,10 @@ describe("SelfHealingManager", () => {
|
|||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(150);
|
await vi.advanceTimersByTimeAsync(150);
|
||||||
|
|
||||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: false });
|
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||||
|
globalPause: false,
|
||||||
|
globalPauseReason: undefined,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not schedule unpause when autoUnpauseEnabled is false", async () => {
|
it("does not schedule unpause when autoUnpauseEnabled is false", async () => {
|
||||||
|
|||||||
@@ -205,6 +205,11 @@ export class SelfHealingManager {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (settings.globalPauseReason === "manual") {
|
||||||
|
log.log("Global pause activated manually — auto-unpause skipped, requires manual intervention");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// If pause re-triggered within 60s of our last unpause, escalate backoff
|
// If pause re-triggered within 60s of our last unpause, escalate backoff
|
||||||
if (this.lastUnpauseAt && (Date.now() - this.lastUnpauseAt) < 60_000) {
|
if (this.lastUnpauseAt && (Date.now() - this.lastUnpauseAt) < 60_000) {
|
||||||
this.unpauseAttempt++;
|
this.unpauseAttempt++;
|
||||||
@@ -258,7 +263,7 @@ export class SelfHealingManager {
|
|||||||
|
|
||||||
log.warn("Auto-unpause: clearing globalPause");
|
log.warn("Auto-unpause: clearing globalPause");
|
||||||
this.lastUnpauseAt = Date.now();
|
this.lastUnpauseAt = Date.now();
|
||||||
await this.store.updateSettings({ globalPause: false });
|
await this.store.updateSettings({ globalPause: false, globalPauseReason: undefined });
|
||||||
|
|
||||||
// Note: if the rate limit is still active, the next agent session will
|
// Note: if the rate limit is still active, the next agent session will
|
||||||
// hit it again → UsageLimitPauser triggers globalPause → our listener
|
// hit it again → UsageLimitPauser triggers globalPause → our listener
|
||||||
|
|||||||
@@ -141,13 +141,16 @@ describe("UsageLimitPauser", () => {
|
|||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("calls store.updateSettings({ globalPause: true }) on usage limit hit", async () => {
|
it("calls store.updateSettings({ globalPause: true, globalPauseReason: \"rate-limit\" }) on usage limit hit", async () => {
|
||||||
const store = createMockStore();
|
const store = createMockStore();
|
||||||
const pauser = new UsageLimitPauser(store);
|
const pauser = new UsageLimitPauser(store);
|
||||||
|
|
||||||
await pauser.onUsageLimitHit("executor", "FN-001", "rate_limit_error: Rate limit exceeded");
|
await pauser.onUsageLimitHit("executor", "FN-001", "rate_limit_error: Rate limit exceeded");
|
||||||
|
|
||||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true });
|
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||||
|
globalPause: true,
|
||||||
|
globalPauseReason: "rate-limit",
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("logs the triggering error on the task via store.logEntry", async () => {
|
it("logs the triggering error on the task via store.logEntry", async () => {
|
||||||
|
|||||||
@@ -44,7 +44,8 @@ export function isUsageLimitError(errorMessage: string): boolean {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Lightweight coordinator that agents call when they detect usage-limit errors.
|
* Lightweight coordinator that agents call when they detect usage-limit errors.
|
||||||
* Triggers the global pause mechanism by calling `store.updateSettings({ globalPause: true })`.
|
* Triggers the global pause mechanism by calling
|
||||||
|
* `store.updateSettings({ globalPause: true, globalPauseReason: "rate-limit" })`.
|
||||||
*
|
*
|
||||||
* **Idempotency:** Tracks an internal `paused` flag so that multiple concurrent
|
* **Idempotency:** Tracks an internal `paused` flag so that multiple concurrent
|
||||||
* agents hitting limits only trigger one pause. The flag resets when `globalPause`
|
* agents hitting limits only trigger one pause. The flag resets when `globalPause`
|
||||||
@@ -107,7 +108,7 @@ export class UsageLimitPauser {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Activate global pause
|
// Activate global pause
|
||||||
await this.store.updateSettings({ globalPause: true });
|
await this.store.updateSettings({ globalPause: true, globalPauseReason: "rate-limit" });
|
||||||
|
|
||||||
log.warn("⚠ Global pause activated — all automated activity will halt");
|
log.warn("⚠ Global pause activated — all automated activity will halt");
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user