fix(dashboard,core): bound AI session prompt() with timeout + abort

Subtask, mission-interview, and milestone/slice-interview sessions could pin
their `generating` state forever when the underlying provider stream stalled
silently or a tool call hung. Wrap each `agent.session.prompt()` in a new
GenerationGuard helper (per-session AbortController + timer) so a stuck turn
becomes a bounded error users can retry. Adds matching `stop*Generation`
exports and threads abort through cleanup so dismissing a modal cancels the
in-flight call instead of leaking it.

Also closes the gh-cli tool hang vector: `runGhAsync` / `runGhJsonAsync` now
accept `{ signal, timeoutMs }` (default 30s). Github-touching extension tools
forward the AI tool's signal so an aborted agent kills the `gh` child instead
of orphaning it.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-28 08:05:12 -07:00
parent 596982dd75
commit 2f7ba29ead
11 changed files with 1164 additions and 266 deletions

View File

@@ -0,0 +1,132 @@
// @vitest-environment node
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { GenerationGuard, isAbortError, createAbortError } from "../ai-session-timeout.js";
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
describe("GenerationGuard", () => {
it("resolves with the operation result when it completes before the timeout", async () => {
const guard = new GenerationGuard();
const onTimeout = vi.fn();
const result = await guard.run("s1", 1_000, { onTimeout }, async () => "ok");
expect(result).toBe("ok");
expect(onTimeout).not.toHaveBeenCalled();
expect(guard.has("s1")).toBe(false);
});
it("fires onTimeout, aborts the operation, and rejects with AbortError when the timer fires", async () => {
const guard = new GenerationGuard();
const onTimeout = vi.fn();
let opSettled = false;
const promise = guard.run("s1", 1_000, { onTimeout }, async () => {
// Simulate a hung prompt() that never resolves on its own.
await new Promise<void>(() => { /* intentionally never resolves */ });
opSettled = true;
return "should-not-reach";
});
// Catch promise rejection now so it doesn't become unhandled when timers advance.
const settled = promise.then(
(v) => ({ ok: true as const, v }),
(err) => ({ ok: false as const, err }),
);
await vi.advanceTimersByTimeAsync(1_000);
const outcome = await settled;
expect(outcome.ok).toBe(false);
expect(outcome.ok ? null : outcome.err).toSatisfy((err: unknown) => isAbortError(err));
expect(onTimeout).toHaveBeenCalledTimes(1);
expect(opSettled).toBe(false);
expect(guard.has("s1")).toBe(false);
});
it("fires onUserStop (not onTimeout) when stop() aborts before the timer", async () => {
const guard = new GenerationGuard();
const onTimeout = vi.fn();
const onUserStop = vi.fn();
const promise = guard.run("s1", 10_000, { onTimeout, onUserStop }, async () => {
await new Promise<void>(() => { /* hang */ });
});
const settled = promise.then(
() => ({ ok: true as const }),
(err) => ({ ok: false as const, err }),
);
// Stop before the timer fires.
expect(guard.stop("s1")).toBe(true);
const outcome = await settled;
expect(outcome.ok).toBe(false);
expect(onTimeout).not.toHaveBeenCalled();
expect(onUserStop).toHaveBeenCalledTimes(1);
// Subsequent stop is a no-op.
expect(guard.stop("s1")).toBe(false);
});
it("re-entrant run for the same id aborts the prior generation", async () => {
const guard = new GenerationGuard();
const firstUserStop = vi.fn();
const first = guard.run(
"s1",
10_000,
{ onTimeout: vi.fn(), onUserStop: firstUserStop },
async () => { await new Promise<void>(() => { /* hang */ }); },
);
const firstSettled = first.then(
() => ({ ok: true as const }),
(err) => ({ ok: false as const, err }),
);
const second = guard.run("s1", 10_000, { onTimeout: vi.fn() }, async () => "fresh");
await expect(second).resolves.toBe("fresh");
const firstOutcome = await firstSettled;
expect(firstOutcome.ok).toBe(false);
// Re-entrant cancellation goes through the internal cancel path, not the
// user-facing stop, so onUserStop is intentionally not fired for the
// displaced generation. The displaced caller still observes AbortError.
expect(firstUserStop).not.toHaveBeenCalled();
expect(guard.has("s1")).toBe(false);
});
it("reset() aborts every active generation", async () => {
const guard = new GenerationGuard();
const a = guard.run("a", 10_000, { onTimeout: vi.fn() }, async () => {
await new Promise<void>(() => { /* hang */ });
});
const b = guard.run("b", 10_000, { onTimeout: vi.fn() }, async () => {
await new Promise<void>(() => { /* hang */ });
});
const aSettled = a.catch((err) => err);
const bSettled = b.catch((err) => err);
guard.reset();
expect(isAbortError(await aSettled)).toBe(true);
expect(isAbortError(await bSettled)).toBe(true);
expect(guard.has("a")).toBe(false);
expect(guard.has("b")).toBe(false);
});
it("createAbortError is identifiable via isAbortError", () => {
expect(isAbortError(createAbortError())).toBe(true);
expect(isAbortError(new Error("other"))).toBe(false);
expect(isAbortError("not an error")).toBe(false);
});
});

View File

@@ -33,6 +33,8 @@ import {
skipTargetInterview,
MILESTONE_INTERVIEW_SYSTEM_PROMPT,
SLICE_INTERVIEW_SYSTEM_PROMPT,
stopMilestoneSliceInterviewGeneration,
GENERATION_TIMEOUT_MS,
type MilestoneInterviewSummary,
type SliceInterviewSummary,
} from "../milestone-slice-interview.js";
@@ -769,4 +771,83 @@ describe("milestone-slice-interview module", () => {
expect(SLICE_INTERVIEW_SYSTEM_PROMPT).toContain("acceptanceCriteria");
});
});
describe("generation timeout / abort", () => {
it("marks the session as error when initial generation exceeds GENERATION_TIMEOUT_MS", async () => {
vi.useFakeTimers();
let resolveHungPrompt: (() => void) | undefined;
mockCreateFnAgent.mockImplementationOnce(async () => ({
session: {
state: { messages: [] },
prompt: vi.fn(async () => {
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
}),
dispose: vi.fn(),
},
}));
const sessionId = await createTargetInterviewSession(
"10.0.1.10",
"milestone",
"milestone-stuck",
"Hung milestone interview",
undefined,
"/tmp/project",
);
// Yield so the guard registration runs.
await Promise.resolve();
await Promise.resolve();
await vi.advanceTimersByTimeAsync(GENERATION_TIMEOUT_MS);
await vi.advanceTimersByTimeAsync(0);
const session = getTargetInterviewSession(sessionId);
expect(session?.error).toMatch(/timed out/i);
resolveHungPrompt?.();
await vi.advanceTimersByTimeAsync(0);
vi.useRealTimers();
});
it("stopMilestoneSliceInterviewGeneration aborts an in-flight session and marks it stopped", async () => {
let resolveHungPrompt: (() => void) | undefined;
mockCreateFnAgent.mockImplementationOnce(async () => ({
session: {
state: { messages: [] },
prompt: vi.fn(async () => {
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
}),
dispose: vi.fn(),
},
}));
const sessionId = await createTargetInterviewSession(
"10.0.1.11",
"slice",
"slice-stoppable",
"Stoppable slice interview",
undefined,
"/tmp/project",
);
let stopped = false;
for (let i = 0; i < 50 && !stopped; i++) {
stopped = stopMilestoneSliceInterviewGeneration(sessionId);
if (!stopped) await new Promise((resolve) => setTimeout(resolve, 5));
}
expect(stopped).toBe(true);
await new Promise((resolve) => setTimeout(resolve, 0));
const session = getTargetInterviewSession(sessionId);
expect(session?.error).toMatch(/stopped by user/i);
expect(stopMilestoneSliceInterviewGeneration(sessionId)).toBe(false);
resolveHungPrompt?.();
await new Promise((resolve) => setTimeout(resolve, 0));
});
});
});

View File

@@ -27,7 +27,9 @@ import {
setAiSessionStore,
RateLimitError,
SessionNotFoundError,
stopMissionInterviewGeneration,
submitMissionInterviewResponse,
GENERATION_TIMEOUT_MS,
} from "../mission-interview.js";
import {
setDiagnosticsSink,
@@ -927,4 +929,85 @@ describe("mission-interview module", () => {
expect(lastCall[0].systemPrompt).toMatch(/^You are a mission planning assistant/);
});
});
describe("generation timeout / abort", () => {
it("marks the session as error when initial generation exceeds GENERATION_TIMEOUT_MS", async () => {
vi.useFakeTimers();
let resolveHungPrompt: (() => void) | undefined;
mockCreateFnAgent.mockImplementationOnce(async () => ({
session: {
state: { messages: [] },
prompt: vi.fn(async () => {
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
}),
dispose: vi.fn(),
},
}));
const sessionId = await createMissionInterviewSession(
"10.0.0.10",
"Hung mission interview",
"/tmp/project",
);
// Yield so initializeAgent's async work registers the generation guard.
await Promise.resolve();
await Promise.resolve();
await vi.advanceTimersByTimeAsync(GENERATION_TIMEOUT_MS);
await vi.advanceTimersByTimeAsync(0);
const session = getMissionInterviewSession(sessionId);
expect(session?.error).toMatch(/timed out/i);
resolveHungPrompt?.();
await vi.advanceTimersByTimeAsync(0);
vi.useRealTimers();
});
it("stopMissionInterviewGeneration aborts an in-flight session and marks it stopped", async () => {
// Real timers — we want the guard.run() registration to actually happen
// through normal microtask scheduling without us racing it.
let resolveHungPrompt: (() => void) | undefined;
mockCreateFnAgent.mockImplementationOnce(async () => ({
session: {
state: { messages: [] },
prompt: vi.fn(async () => {
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
}),
dispose: vi.fn(),
},
}));
const sessionId = await createMissionInterviewSession(
"10.0.0.11",
"Stoppable mission interview",
"/tmp/project",
);
// initializeAgent → createMissionInterviewAgent → continueAgentConversation
// → generationGuard.run is several awaits deep; poll until the guard is
// registered, then stop. Bounded so a regression doesn't hang the suite.
let stopped = false;
for (let i = 0; i < 50 && !stopped; i++) {
stopped = stopMissionInterviewGeneration(sessionId);
if (!stopped) await new Promise((resolve) => setTimeout(resolve, 5));
}
expect(stopped).toBe(true);
// Yield so the guard's catch/finally and onUserStop run.
await new Promise((resolve) => setTimeout(resolve, 0));
const session = getMissionInterviewSession(sessionId);
expect(session?.error).toMatch(/stopped by user/i);
// Stop is idempotent — no in-flight generation after first call.
expect(stopMissionInterviewGeneration(sessionId)).toBe(false);
resolveHungPrompt?.();
await new Promise((resolve) => setTimeout(resolve, 0));
});
});
});

View File

@@ -31,7 +31,9 @@ import {
SessionNotFoundError,
InvalidSessionStateError,
setAiSessionStore,
stopSubtaskGeneration,
SubtaskStreamManager,
GENERATION_TIMEOUT_MS,
} from "../subtask-breakdown.js";
const UUID_REGEX =
@@ -913,3 +915,85 @@ describe("SessionNotFoundError", () => {
expect(error.message).toBe("Missing session");
});
});
describe("subtask generation timeout / abort", () => {
it("marks the session as error and stops the prompt() promise when generation exceeds GENERATION_TIMEOUT_MS", async () => {
vi.useFakeTimers();
let resolveHungPrompt: (() => void) | undefined;
const hungPromptCallable = vi.fn(async () => {
// Simulate a stalled provider stream that never terminates on its own.
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
return undefined;
});
mockCreateFnAgent.mockImplementation(async () => ({
session: {
state: { messages: [] },
prompt: hungPromptCallable,
dispose: vi.fn(),
},
}));
const created = await createSubtaskSession(
"Hung subtask generation",
undefined,
"/tmp/project",
);
// Yield once so startSubtaskGeneration's microtasks run before we advance time.
await Promise.resolve();
expect(getSubtaskSession(created.sessionId)?.status).toBe("generating");
await vi.advanceTimersByTimeAsync(GENERATION_TIMEOUT_MS);
// Drain any remaining microtasks (the abort propagates through Promise.race).
await vi.advanceTimersByTimeAsync(0);
const after = getSubtaskSession(created.sessionId);
expect(after?.status).toBe("error");
expect(after?.error).toMatch(/timed out/i);
// The hung prompt is still pending; release it so its microtask completes.
resolveHungPrompt?.();
await vi.advanceTimersByTimeAsync(0);
vi.useRealTimers();
});
it("stopSubtaskGeneration aborts an in-flight session and marks it stopped", async () => {
vi.useFakeTimers();
let resolveHungPrompt: (() => void) | undefined;
mockCreateFnAgent.mockImplementation(async () => ({
session: {
state: { messages: [] },
prompt: vi.fn(async () => {
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
}),
dispose: vi.fn(),
},
}));
const created = await createSubtaskSession(
"Stoppable subtask generation",
undefined,
"/tmp/project",
);
await Promise.resolve();
expect(stopSubtaskGeneration(created.sessionId)).toBe(true);
await vi.advanceTimersByTimeAsync(0);
const after = getSubtaskSession(created.sessionId);
expect(after?.status).toBe("error");
expect(after?.error).toMatch(/stopped by user/i);
// Stop is idempotent — no in-flight generation after first call.
expect(stopSubtaskGeneration(created.sessionId)).toBe(false);
resolveHungPrompt?.();
await vi.advanceTimersByTimeAsync(0);
vi.useRealTimers();
});
});