fix(dashboard,core): bound AI session prompt() with timeout + abort
Subtask, mission-interview, and milestone/slice-interview sessions could pin
their `generating` state forever when the underlying provider stream stalled
silently or a tool call hung. Wrap each `agent.session.prompt()` in a new
GenerationGuard helper (per-session AbortController + timer) so a stuck turn
becomes a bounded error users can retry. Adds matching `stop*Generation`
exports and threads abort through cleanup so dismissing a modal cancels the
in-flight call instead of leaking it.
Also closes the gh-cli tool hang vector: `runGhAsync` / `runGhJsonAsync` now
accept `{ signal, timeoutMs }` (default 30s). Github-touching extension tools
forward the AI tool's signal so an aborted agent kills the `gh` child instead
of orphaning it.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
132
packages/dashboard/src/__tests__/ai-session-timeout.test.ts
Normal file
132
packages/dashboard/src/__tests__/ai-session-timeout.test.ts
Normal file
@@ -0,0 +1,132 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { GenerationGuard, isAbortError, createAbortError } from "../ai-session-timeout.js";
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe("GenerationGuard", () => {
|
||||
it("resolves with the operation result when it completes before the timeout", async () => {
|
||||
const guard = new GenerationGuard();
|
||||
const onTimeout = vi.fn();
|
||||
|
||||
const result = await guard.run("s1", 1_000, { onTimeout }, async () => "ok");
|
||||
|
||||
expect(result).toBe("ok");
|
||||
expect(onTimeout).not.toHaveBeenCalled();
|
||||
expect(guard.has("s1")).toBe(false);
|
||||
});
|
||||
|
||||
it("fires onTimeout, aborts the operation, and rejects with AbortError when the timer fires", async () => {
|
||||
const guard = new GenerationGuard();
|
||||
const onTimeout = vi.fn();
|
||||
|
||||
let opSettled = false;
|
||||
const promise = guard.run("s1", 1_000, { onTimeout }, async () => {
|
||||
// Simulate a hung prompt() that never resolves on its own.
|
||||
await new Promise<void>(() => { /* intentionally never resolves */ });
|
||||
opSettled = true;
|
||||
return "should-not-reach";
|
||||
});
|
||||
|
||||
// Catch promise rejection now so it doesn't become unhandled when timers advance.
|
||||
const settled = promise.then(
|
||||
(v) => ({ ok: true as const, v }),
|
||||
(err) => ({ ok: false as const, err }),
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
const outcome = await settled;
|
||||
|
||||
expect(outcome.ok).toBe(false);
|
||||
expect(outcome.ok ? null : outcome.err).toSatisfy((err: unknown) => isAbortError(err));
|
||||
expect(onTimeout).toHaveBeenCalledTimes(1);
|
||||
expect(opSettled).toBe(false);
|
||||
expect(guard.has("s1")).toBe(false);
|
||||
});
|
||||
|
||||
it("fires onUserStop (not onTimeout) when stop() aborts before the timer", async () => {
|
||||
const guard = new GenerationGuard();
|
||||
const onTimeout = vi.fn();
|
||||
const onUserStop = vi.fn();
|
||||
|
||||
const promise = guard.run("s1", 10_000, { onTimeout, onUserStop }, async () => {
|
||||
await new Promise<void>(() => { /* hang */ });
|
||||
});
|
||||
const settled = promise.then(
|
||||
() => ({ ok: true as const }),
|
||||
(err) => ({ ok: false as const, err }),
|
||||
);
|
||||
|
||||
// Stop before the timer fires.
|
||||
expect(guard.stop("s1")).toBe(true);
|
||||
|
||||
const outcome = await settled;
|
||||
expect(outcome.ok).toBe(false);
|
||||
expect(onTimeout).not.toHaveBeenCalled();
|
||||
expect(onUserStop).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Subsequent stop is a no-op.
|
||||
expect(guard.stop("s1")).toBe(false);
|
||||
});
|
||||
|
||||
it("re-entrant run for the same id aborts the prior generation", async () => {
|
||||
const guard = new GenerationGuard();
|
||||
const firstUserStop = vi.fn();
|
||||
|
||||
const first = guard.run(
|
||||
"s1",
|
||||
10_000,
|
||||
{ onTimeout: vi.fn(), onUserStop: firstUserStop },
|
||||
async () => { await new Promise<void>(() => { /* hang */ }); },
|
||||
);
|
||||
const firstSettled = first.then(
|
||||
() => ({ ok: true as const }),
|
||||
(err) => ({ ok: false as const, err }),
|
||||
);
|
||||
|
||||
const second = guard.run("s1", 10_000, { onTimeout: vi.fn() }, async () => "fresh");
|
||||
|
||||
await expect(second).resolves.toBe("fresh");
|
||||
const firstOutcome = await firstSettled;
|
||||
expect(firstOutcome.ok).toBe(false);
|
||||
// Re-entrant cancellation goes through the internal cancel path, not the
|
||||
// user-facing stop, so onUserStop is intentionally not fired for the
|
||||
// displaced generation. The displaced caller still observes AbortError.
|
||||
expect(firstUserStop).not.toHaveBeenCalled();
|
||||
expect(guard.has("s1")).toBe(false);
|
||||
});
|
||||
|
||||
it("reset() aborts every active generation", async () => {
|
||||
const guard = new GenerationGuard();
|
||||
|
||||
const a = guard.run("a", 10_000, { onTimeout: vi.fn() }, async () => {
|
||||
await new Promise<void>(() => { /* hang */ });
|
||||
});
|
||||
const b = guard.run("b", 10_000, { onTimeout: vi.fn() }, async () => {
|
||||
await new Promise<void>(() => { /* hang */ });
|
||||
});
|
||||
const aSettled = a.catch((err) => err);
|
||||
const bSettled = b.catch((err) => err);
|
||||
|
||||
guard.reset();
|
||||
|
||||
expect(isAbortError(await aSettled)).toBe(true);
|
||||
expect(isAbortError(await bSettled)).toBe(true);
|
||||
expect(guard.has("a")).toBe(false);
|
||||
expect(guard.has("b")).toBe(false);
|
||||
});
|
||||
|
||||
it("createAbortError is identifiable via isAbortError", () => {
|
||||
expect(isAbortError(createAbortError())).toBe(true);
|
||||
expect(isAbortError(new Error("other"))).toBe(false);
|
||||
expect(isAbortError("not an error")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -33,6 +33,8 @@ import {
|
||||
skipTargetInterview,
|
||||
MILESTONE_INTERVIEW_SYSTEM_PROMPT,
|
||||
SLICE_INTERVIEW_SYSTEM_PROMPT,
|
||||
stopMilestoneSliceInterviewGeneration,
|
||||
GENERATION_TIMEOUT_MS,
|
||||
type MilestoneInterviewSummary,
|
||||
type SliceInterviewSummary,
|
||||
} from "../milestone-slice-interview.js";
|
||||
@@ -769,4 +771,83 @@ describe("milestone-slice-interview module", () => {
|
||||
expect(SLICE_INTERVIEW_SYSTEM_PROMPT).toContain("acceptanceCriteria");
|
||||
});
|
||||
});
|
||||
|
||||
describe("generation timeout / abort", () => {
|
||||
it("marks the session as error when initial generation exceeds GENERATION_TIMEOUT_MS", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
let resolveHungPrompt: (() => void) | undefined;
|
||||
mockCreateFnAgent.mockImplementationOnce(async () => ({
|
||||
session: {
|
||||
state: { messages: [] },
|
||||
prompt: vi.fn(async () => {
|
||||
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||
}),
|
||||
dispose: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const sessionId = await createTargetInterviewSession(
|
||||
"10.0.1.10",
|
||||
"milestone",
|
||||
"milestone-stuck",
|
||||
"Hung milestone interview",
|
||||
undefined,
|
||||
"/tmp/project",
|
||||
);
|
||||
|
||||
// Yield so the guard registration runs.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(GENERATION_TIMEOUT_MS);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
const session = getTargetInterviewSession(sessionId);
|
||||
expect(session?.error).toMatch(/timed out/i);
|
||||
|
||||
resolveHungPrompt?.();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("stopMilestoneSliceInterviewGeneration aborts an in-flight session and marks it stopped", async () => {
|
||||
let resolveHungPrompt: (() => void) | undefined;
|
||||
mockCreateFnAgent.mockImplementationOnce(async () => ({
|
||||
session: {
|
||||
state: { messages: [] },
|
||||
prompt: vi.fn(async () => {
|
||||
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||
}),
|
||||
dispose: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const sessionId = await createTargetInterviewSession(
|
||||
"10.0.1.11",
|
||||
"slice",
|
||||
"slice-stoppable",
|
||||
"Stoppable slice interview",
|
||||
undefined,
|
||||
"/tmp/project",
|
||||
);
|
||||
|
||||
let stopped = false;
|
||||
for (let i = 0; i < 50 && !stopped; i++) {
|
||||
stopped = stopMilestoneSliceInterviewGeneration(sessionId);
|
||||
if (!stopped) await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
expect(stopped).toBe(true);
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
const session = getTargetInterviewSession(sessionId);
|
||||
expect(session?.error).toMatch(/stopped by user/i);
|
||||
expect(stopMilestoneSliceInterviewGeneration(sessionId)).toBe(false);
|
||||
|
||||
resolveHungPrompt?.();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -27,7 +27,9 @@ import {
|
||||
setAiSessionStore,
|
||||
RateLimitError,
|
||||
SessionNotFoundError,
|
||||
stopMissionInterviewGeneration,
|
||||
submitMissionInterviewResponse,
|
||||
GENERATION_TIMEOUT_MS,
|
||||
} from "../mission-interview.js";
|
||||
import {
|
||||
setDiagnosticsSink,
|
||||
@@ -927,4 +929,85 @@ describe("mission-interview module", () => {
|
||||
expect(lastCall[0].systemPrompt).toMatch(/^You are a mission planning assistant/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("generation timeout / abort", () => {
|
||||
it("marks the session as error when initial generation exceeds GENERATION_TIMEOUT_MS", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
let resolveHungPrompt: (() => void) | undefined;
|
||||
mockCreateFnAgent.mockImplementationOnce(async () => ({
|
||||
session: {
|
||||
state: { messages: [] },
|
||||
prompt: vi.fn(async () => {
|
||||
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||
}),
|
||||
dispose: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const sessionId = await createMissionInterviewSession(
|
||||
"10.0.0.10",
|
||||
"Hung mission interview",
|
||||
"/tmp/project",
|
||||
);
|
||||
|
||||
// Yield so initializeAgent's async work registers the generation guard.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(GENERATION_TIMEOUT_MS);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
const session = getMissionInterviewSession(sessionId);
|
||||
expect(session?.error).toMatch(/timed out/i);
|
||||
|
||||
resolveHungPrompt?.();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("stopMissionInterviewGeneration aborts an in-flight session and marks it stopped", async () => {
|
||||
// Real timers — we want the guard.run() registration to actually happen
|
||||
// through normal microtask scheduling without us racing it.
|
||||
let resolveHungPrompt: (() => void) | undefined;
|
||||
mockCreateFnAgent.mockImplementationOnce(async () => ({
|
||||
session: {
|
||||
state: { messages: [] },
|
||||
prompt: vi.fn(async () => {
|
||||
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||
}),
|
||||
dispose: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const sessionId = await createMissionInterviewSession(
|
||||
"10.0.0.11",
|
||||
"Stoppable mission interview",
|
||||
"/tmp/project",
|
||||
);
|
||||
|
||||
// initializeAgent → createMissionInterviewAgent → continueAgentConversation
|
||||
// → generationGuard.run is several awaits deep; poll until the guard is
|
||||
// registered, then stop. Bounded so a regression doesn't hang the suite.
|
||||
let stopped = false;
|
||||
for (let i = 0; i < 50 && !stopped; i++) {
|
||||
stopped = stopMissionInterviewGeneration(sessionId);
|
||||
if (!stopped) await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
expect(stopped).toBe(true);
|
||||
|
||||
// Yield so the guard's catch/finally and onUserStop run.
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
const session = getMissionInterviewSession(sessionId);
|
||||
expect(session?.error).toMatch(/stopped by user/i);
|
||||
|
||||
// Stop is idempotent — no in-flight generation after first call.
|
||||
expect(stopMissionInterviewGeneration(sessionId)).toBe(false);
|
||||
|
||||
resolveHungPrompt?.();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -31,7 +31,9 @@ import {
|
||||
SessionNotFoundError,
|
||||
InvalidSessionStateError,
|
||||
setAiSessionStore,
|
||||
stopSubtaskGeneration,
|
||||
SubtaskStreamManager,
|
||||
GENERATION_TIMEOUT_MS,
|
||||
} from "../subtask-breakdown.js";
|
||||
|
||||
const UUID_REGEX =
|
||||
@@ -913,3 +915,85 @@ describe("SessionNotFoundError", () => {
|
||||
expect(error.message).toBe("Missing session");
|
||||
});
|
||||
});
|
||||
|
||||
describe("subtask generation timeout / abort", () => {
|
||||
it("marks the session as error and stops the prompt() promise when generation exceeds GENERATION_TIMEOUT_MS", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
let resolveHungPrompt: (() => void) | undefined;
|
||||
const hungPromptCallable = vi.fn(async () => {
|
||||
// Simulate a stalled provider stream that never terminates on its own.
|
||||
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||
return undefined;
|
||||
});
|
||||
|
||||
mockCreateFnAgent.mockImplementation(async () => ({
|
||||
session: {
|
||||
state: { messages: [] },
|
||||
prompt: hungPromptCallable,
|
||||
dispose: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const created = await createSubtaskSession(
|
||||
"Hung subtask generation",
|
||||
undefined,
|
||||
"/tmp/project",
|
||||
);
|
||||
|
||||
// Yield once so startSubtaskGeneration's microtasks run before we advance time.
|
||||
await Promise.resolve();
|
||||
expect(getSubtaskSession(created.sessionId)?.status).toBe("generating");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(GENERATION_TIMEOUT_MS);
|
||||
// Drain any remaining microtasks (the abort propagates through Promise.race).
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
const after = getSubtaskSession(created.sessionId);
|
||||
expect(after?.status).toBe("error");
|
||||
expect(after?.error).toMatch(/timed out/i);
|
||||
|
||||
// The hung prompt is still pending; release it so its microtask completes.
|
||||
resolveHungPrompt?.();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("stopSubtaskGeneration aborts an in-flight session and marks it stopped", async () => {
|
||||
vi.useFakeTimers();
|
||||
|
||||
let resolveHungPrompt: (() => void) | undefined;
|
||||
mockCreateFnAgent.mockImplementation(async () => ({
|
||||
session: {
|
||||
state: { messages: [] },
|
||||
prompt: vi.fn(async () => {
|
||||
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||
}),
|
||||
dispose: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
const created = await createSubtaskSession(
|
||||
"Stoppable subtask generation",
|
||||
undefined,
|
||||
"/tmp/project",
|
||||
);
|
||||
await Promise.resolve();
|
||||
|
||||
expect(stopSubtaskGeneration(created.sessionId)).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
const after = getSubtaskSession(created.sessionId);
|
||||
expect(after?.status).toBe("error");
|
||||
expect(after?.error).toMatch(/stopped by user/i);
|
||||
|
||||
// Stop is idempotent — no in-flight generation after first call.
|
||||
expect(stopSubtaskGeneration(created.sessionId)).toBe(false);
|
||||
|
||||
resolveHungPrompt?.();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
vi.useRealTimers();
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user