feat(FN-2119): merge fusion/fn-2119
This commit is contained in:
@@ -5459,7 +5459,10 @@ describe("TaskExecutor usage limit detection", () => {
|
||||
"FN-001",
|
||||
"rate_limit_error: Rate limit exceeded",
|
||||
);
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true });
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||
globalPause: true,
|
||||
globalPauseReason: "rate-limit",
|
||||
});
|
||||
// Task should still be marked as failed
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-001", { status: "failed", error: "rate_limit_error: Rate limit exceeded" });
|
||||
expect(onError).toHaveBeenCalled();
|
||||
|
||||
@@ -720,7 +720,10 @@ describe("aiMergeTask — usage limit detection", () => {
|
||||
"FN-050",
|
||||
"rate_limit_error: Rate limit exceeded",
|
||||
);
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true });
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||
globalPause: true,
|
||||
globalPauseReason: "rate-limit",
|
||||
});
|
||||
});
|
||||
|
||||
it("triggers global pause when session.prompt() resolves with exhausted-retry error on state.error", async () => {
|
||||
|
||||
@@ -142,7 +142,48 @@ describe("SelfHealingManager", () => {
|
||||
// ── Auto-unpause ─────────────────────────────────────────────────
|
||||
|
||||
describe("auto-unpause", () => {
|
||||
it("schedules unpause when globalPause transitions false→true", async () => {
|
||||
it("does not schedule unpause when globalPauseReason is 'manual'", async () => {
|
||||
manager.start();
|
||||
|
||||
store.emit("settings:updated", {
|
||||
settings: {
|
||||
globalPause: true,
|
||||
globalPauseReason: "manual",
|
||||
autoUnpauseEnabled: true,
|
||||
autoUnpauseBaseDelayMs: 100,
|
||||
autoUnpauseMaxDelayMs: 800,
|
||||
},
|
||||
previous: { globalPause: false },
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
|
||||
expect(store.updateSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("auto-unpauses when globalPauseReason is 'rate-limit'", async () => {
|
||||
manager.start();
|
||||
|
||||
store.emit("settings:updated", {
|
||||
settings: {
|
||||
globalPause: true,
|
||||
globalPauseReason: "rate-limit",
|
||||
autoUnpauseEnabled: true,
|
||||
autoUnpauseBaseDelayMs: 100,
|
||||
autoUnpauseMaxDelayMs: 800,
|
||||
},
|
||||
previous: { globalPause: false },
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(150);
|
||||
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||
globalPause: false,
|
||||
globalPauseReason: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("auto-unpauses when globalPauseReason is undefined (backward compat)", async () => {
|
||||
manager.start();
|
||||
|
||||
store.emit("settings:updated", {
|
||||
@@ -152,7 +193,10 @@ describe("SelfHealingManager", () => {
|
||||
|
||||
await vi.advanceTimersByTimeAsync(150);
|
||||
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: false });
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||
globalPause: false,
|
||||
globalPauseReason: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not schedule unpause when autoUnpauseEnabled is false", async () => {
|
||||
|
||||
@@ -205,6 +205,11 @@ export class SelfHealingManager {
|
||||
return;
|
||||
}
|
||||
|
||||
if (settings.globalPauseReason === "manual") {
|
||||
log.log("Global pause activated manually — auto-unpause skipped, requires manual intervention");
|
||||
return;
|
||||
}
|
||||
|
||||
// If pause re-triggered within 60s of our last unpause, escalate backoff
|
||||
if (this.lastUnpauseAt && (Date.now() - this.lastUnpauseAt) < 60_000) {
|
||||
this.unpauseAttempt++;
|
||||
@@ -258,7 +263,7 @@ export class SelfHealingManager {
|
||||
|
||||
log.warn("Auto-unpause: clearing globalPause");
|
||||
this.lastUnpauseAt = Date.now();
|
||||
await this.store.updateSettings({ globalPause: false });
|
||||
await this.store.updateSettings({ globalPause: false, globalPauseReason: undefined });
|
||||
|
||||
// Note: if the rate limit is still active, the next agent session will
|
||||
// hit it again → UsageLimitPauser triggers globalPause → our listener
|
||||
|
||||
@@ -141,13 +141,16 @@ describe("UsageLimitPauser", () => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("calls store.updateSettings({ globalPause: true }) on usage limit hit", async () => {
|
||||
it("calls store.updateSettings({ globalPause: true, globalPauseReason: \"rate-limit\" }) on usage limit hit", async () => {
|
||||
const store = createMockStore();
|
||||
const pauser = new UsageLimitPauser(store);
|
||||
|
||||
await pauser.onUsageLimitHit("executor", "FN-001", "rate_limit_error: Rate limit exceeded");
|
||||
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({ globalPause: true });
|
||||
expect(store.updateSettings).toHaveBeenCalledWith({
|
||||
globalPause: true,
|
||||
globalPauseReason: "rate-limit",
|
||||
});
|
||||
});
|
||||
|
||||
it("logs the triggering error on the task via store.logEntry", async () => {
|
||||
|
||||
@@ -44,7 +44,8 @@ export function isUsageLimitError(errorMessage: string): boolean {
|
||||
|
||||
/**
|
||||
* Lightweight coordinator that agents call when they detect usage-limit errors.
|
||||
* Triggers the global pause mechanism by calling `store.updateSettings({ globalPause: true })`.
|
||||
* Triggers the global pause mechanism by calling
|
||||
* `store.updateSettings({ globalPause: true, globalPauseReason: "rate-limit" })`.
|
||||
*
|
||||
* **Idempotency:** Tracks an internal `paused` flag so that multiple concurrent
|
||||
* agents hitting limits only trigger one pause. The flag resets when `globalPause`
|
||||
@@ -107,7 +108,7 @@ export class UsageLimitPauser {
|
||||
);
|
||||
|
||||
// Activate global pause
|
||||
await this.store.updateSettings({ globalPause: true });
|
||||
await this.store.updateSettings({ globalPause: true, globalPauseReason: "rate-limit" });
|
||||
|
||||
log.warn("⚠ Global pause activated — all automated activity will halt");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user