fix(dashboard,core): bound AI session prompt() with timeout + abort
Subtask, mission-interview, and milestone/slice-interview sessions could pin
their `generating` state forever when the underlying provider stream stalled
silently or a tool call hung. Wrap each `agent.session.prompt()` in a new
GenerationGuard helper (per-session AbortController + timer) so a stuck turn
becomes a bounded error users can retry. Adds matching `stop*Generation`
exports and threads abort through cleanup so dismissing a modal cancels the
in-flight call instead of leaking it.
Also closes the gh-cli tool hang vector: `runGhAsync` / `runGhJsonAsync` now
accept `{ signal, timeoutMs }` (default 30s). Github-touching extension tools
forward the AI tool's signal so an aborted agent kills the `gh` child instead
of orphaning it.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -96,7 +96,7 @@ function ensureGhCliAuth(): void {
|
|||||||
async function fetchGitHubIssuesViaGh(
|
async function fetchGitHubIssuesViaGh(
|
||||||
owner: string,
|
owner: string,
|
||||||
repo: string,
|
repo: string,
|
||||||
options: { limit?: number; labels?: string[] } = {},
|
options: { limit?: number; labels?: string[]; signal?: AbortSignal } = {},
|
||||||
): Promise<GitHubIssueApiResult[]> {
|
): Promise<GitHubIssueApiResult[]> {
|
||||||
ensureGhCliAuth();
|
ensureGhCliAuth();
|
||||||
|
|
||||||
@@ -110,20 +110,25 @@ async function fetchGitHubIssuesViaGh(
|
|||||||
const path = `repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues?${queryParams.toString()}`;
|
const path = `repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues?${queryParams.toString()}`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const issues = await runGhJsonAsync<GitHubIssueApiResult[]>(["api", path]);
|
const issues = await runGhJsonAsync<GitHubIssueApiResult[]>(["api", path], { signal: options.signal });
|
||||||
return issues.filter((issue) => !issue.pull_request);
|
return issues.filter((issue) => !issue.pull_request);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw new Error(getGhErrorMessage(error));
|
throw new Error(getGhErrorMessage(error));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchGitHubIssueViaGh(owner: string, repo: string, issueNumber: number): Promise<GitHubIssueApiResult> {
|
async function fetchGitHubIssueViaGh(
|
||||||
|
owner: string,
|
||||||
|
repo: string,
|
||||||
|
issueNumber: number,
|
||||||
|
options: { signal?: AbortSignal } = {},
|
||||||
|
): Promise<GitHubIssueApiResult> {
|
||||||
ensureGhCliAuth();
|
ensureGhCliAuth();
|
||||||
|
|
||||||
const path = `repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues/${issueNumber}`;
|
const path = `repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues/${issueNumber}`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return await runGhJsonAsync<GitHubIssueApiResult>(["api", path]);
|
return await runGhJsonAsync<GitHubIssueApiResult>(["api", path], { signal: options.signal });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throw new Error(getGhErrorMessage(error));
|
throw new Error(getGhErrorMessage(error));
|
||||||
}
|
}
|
||||||
@@ -786,12 +791,12 @@ export default function kbExtension(pi: ExtensionAPI) {
|
|||||||
),
|
),
|
||||||
}),
|
}),
|
||||||
|
|
||||||
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
||||||
const [owner, repo] = params.ownerRepo.split("/");
|
const [owner, repo] = params.ownerRepo.split("/");
|
||||||
const limit = params.limit ?? 30;
|
const limit = params.limit ?? 30;
|
||||||
const labels = params.labels;
|
const labels = params.labels;
|
||||||
|
|
||||||
const issues = await fetchGitHubIssuesViaGh(owner, repo, { limit, labels });
|
const issues = await fetchGitHubIssuesViaGh(owner, repo, { limit, labels, signal });
|
||||||
|
|
||||||
if (issues.length === 0) {
|
if (issues.length === 0) {
|
||||||
return {
|
return {
|
||||||
@@ -876,9 +881,9 @@ export default function kbExtension(pi: ExtensionAPI) {
|
|||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
|
|
||||||
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
||||||
const { owner, repo, issueNumber } = params;
|
const { owner, repo, issueNumber } = params;
|
||||||
const issue = await fetchGitHubIssueViaGh(owner, repo, issueNumber);
|
const issue = await fetchGitHubIssueViaGh(owner, repo, issueNumber, { signal });
|
||||||
|
|
||||||
if (issue.pull_request) {
|
if (issue.pull_request) {
|
||||||
throw new Error(`#${issueNumber} is a pull request, not an issue`);
|
throw new Error(`#${issueNumber} is a pull request, not an issue`);
|
||||||
@@ -975,9 +980,9 @@ export default function kbExtension(pi: ExtensionAPI) {
|
|||||||
),
|
),
|
||||||
}),
|
}),
|
||||||
|
|
||||||
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
||||||
const { owner, repo, limit = 30, labels } = params;
|
const { owner, repo, limit = 30, labels } = params;
|
||||||
const issues = await fetchGitHubIssuesViaGh(owner, repo, { limit, labels });
|
const issues = await fetchGitHubIssuesViaGh(owner, repo, { limit, labels, signal });
|
||||||
|
|
||||||
if (issues.length === 0) {
|
if (issues.length === 0) {
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -1,7 +1,21 @@
|
|||||||
import { describe, it, expect, vi } from "vitest";
|
import { describe, it, expect, vi } from "vitest";
|
||||||
|
|
||||||
|
const { mockExecFile } = vi.hoisted(() => ({
|
||||||
|
mockExecFile: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Mock child_process before importing gh-cli so runGhAsync's `execFile`
|
||||||
|
// reference uses our stub. We only mock execFile because the timeout path
|
||||||
|
// is what we need to exercise; the synchronous helpers don't go through it.
|
||||||
|
vi.mock("node:child_process", async () => {
|
||||||
|
const actual = await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||||
|
return { ...actual, execFile: mockExecFile };
|
||||||
|
});
|
||||||
|
|
||||||
import {
|
import {
|
||||||
getGhErrorMessage,
|
getGhErrorMessage,
|
||||||
parseRepoFromRemote,
|
parseRepoFromRemote,
|
||||||
|
runGhAsync,
|
||||||
} from "../gh-cli.js";
|
} from "../gh-cli.js";
|
||||||
|
|
||||||
// Tests for pure functions (no child_process dependency)
|
// Tests for pure functions (no child_process dependency)
|
||||||
@@ -241,3 +255,100 @@ describe("gh-cli functions (inline tests)", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("runGhAsync timeout / abort", () => {
|
||||||
|
// Each test wires execFile to capture the AbortSignal from gh-cli's
|
||||||
|
// internal controller and the user callback, then drives the abort path
|
||||||
|
// explicitly. This mirrors what real Node would do: when the signal fires,
|
||||||
|
// execFile invokes its callback with an AbortError-shaped failure.
|
||||||
|
type ExecFileCb = (
|
||||||
|
err: (Error & { code?: string | number; killed?: boolean }) | null,
|
||||||
|
stdout: string,
|
||||||
|
stderr: string,
|
||||||
|
) => void;
|
||||||
|
|
||||||
|
function captureExecFile(): { signalRef: { current?: AbortSignal }; cbRef: { current?: ExecFileCb } } {
|
||||||
|
const signalRef: { current?: AbortSignal } = {};
|
||||||
|
const cbRef: { current?: ExecFileCb } = {};
|
||||||
|
mockExecFile.mockImplementation((_bin, _args, options, callback) => {
|
||||||
|
signalRef.current = (options as { signal?: AbortSignal }).signal;
|
||||||
|
cbRef.current = callback as ExecFileCb;
|
||||||
|
// When the signal fires, invoke the callback with an AbortError-shaped
|
||||||
|
// failure — that's what Node's real execFile does on signal abort.
|
||||||
|
signalRef.current?.addEventListener("abort", () => {
|
||||||
|
const err = new Error("aborted") as Error & { code?: string };
|
||||||
|
err.name = "AbortError";
|
||||||
|
err.code = "ABORT_ERR";
|
||||||
|
callback?.(err as never, "", "");
|
||||||
|
}, { once: true });
|
||||||
|
return {} as ReturnType<typeof import("node:child_process").execFile>;
|
||||||
|
});
|
||||||
|
return { signalRef, cbRef };
|
||||||
|
}
|
||||||
|
|
||||||
|
it("rejects with timeout message after timeoutMs elapses and reports ABORT_ERR code", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
mockExecFile.mockReset();
|
||||||
|
captureExecFile();
|
||||||
|
|
||||||
|
const promise = runGhAsync(["api", "repos/owner/repo"], { timeoutMs: 1_000 });
|
||||||
|
const settled = promise.then(
|
||||||
|
(v) => ({ ok: true as const, v }),
|
||||||
|
(err) => ({ ok: false as const, err }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(1_000);
|
||||||
|
const outcome = await settled;
|
||||||
|
|
||||||
|
expect(outcome.ok).toBe(false);
|
||||||
|
if (!outcome.ok) {
|
||||||
|
expect(outcome.err.message).toContain("timed out after 1000ms");
|
||||||
|
expect(outcome.err.code).toBe("ABORT_ERR");
|
||||||
|
}
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("propagates an external AbortSignal and rejects with the abort reason", async () => {
|
||||||
|
mockExecFile.mockReset();
|
||||||
|
captureExecFile();
|
||||||
|
|
||||||
|
const ac = new AbortController();
|
||||||
|
const promise = runGhAsync(["api", "x"], { signal: ac.signal, timeoutMs: 0 });
|
||||||
|
const settled = promise.then(
|
||||||
|
() => ({ ok: true as const }),
|
||||||
|
(err) => ({ ok: false as const, err }),
|
||||||
|
);
|
||||||
|
|
||||||
|
ac.abort(new Error("user cancelled"));
|
||||||
|
const outcome = await settled;
|
||||||
|
|
||||||
|
expect(outcome.ok).toBe(false);
|
||||||
|
if (!outcome.ok) {
|
||||||
|
expect(outcome.err.message).toContain("user cancelled");
|
||||||
|
expect(outcome.err.code).toBe("ABORT_ERR");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects synchronously when the external signal is already aborted", async () => {
|
||||||
|
mockExecFile.mockReset();
|
||||||
|
// Should never be called — pre-aborted check happens before exec.
|
||||||
|
const ac = new AbortController();
|
||||||
|
ac.abort(new Error("pre-cancelled"));
|
||||||
|
|
||||||
|
await expect(runGhAsync(["api", "x"], { signal: ac.signal })).rejects.toMatchObject({
|
||||||
|
message: expect.stringContaining("pre-cancelled"),
|
||||||
|
code: "ABORT_ERR",
|
||||||
|
});
|
||||||
|
expect(mockExecFile).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("disables the timeout when timeoutMs <= 0", async () => {
|
||||||
|
mockExecFile.mockReset();
|
||||||
|
const { cbRef } = captureExecFile();
|
||||||
|
|
||||||
|
const promise = runGhAsync(["api", "x"], { timeoutMs: 0 });
|
||||||
|
// Resolve normally — verifies no implicit timeout fires and shorts the call.
|
||||||
|
cbRef.current?.(null, "ok\n", "");
|
||||||
|
await expect(promise).resolves.toBe("ok\n");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -7,6 +7,29 @@ export interface GhError extends Error {
|
|||||||
stdout: string;
|
stdout: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RunGhOptions {
|
||||||
|
cwd?: string;
|
||||||
|
/** External abort signal — propagated to the spawned `gh` process. */
|
||||||
|
signal?: AbortSignal;
|
||||||
|
/**
|
||||||
|
* Hard ceiling on `gh` runtime in milliseconds. The child process is killed
|
||||||
|
* when exceeded and the returned promise rejects with a `GhError`. Defaults
|
||||||
|
* to 30_000 (30 s); set to `0` or a negative value to disable.
|
||||||
|
*
|
||||||
|
* Without this, an upstream hang (network stall, hung credential helper,
|
||||||
|
* gh waiting on stdin) can leave the call pending forever — which in turn
|
||||||
|
* pins any AI session's `prompt()` that triggered the tool call.
|
||||||
|
*/
|
||||||
|
timeoutMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_GH_TIMEOUT_MS = 30_000;
|
||||||
|
|
||||||
|
function normalizeRunGhOptions(opts: string | RunGhOptions | undefined): RunGhOptions {
|
||||||
|
if (typeof opts === "string") return { cwd: opts };
|
||||||
|
return opts ?? {};
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if the `gh` CLI is installed and available.
|
* Check if the `gh` CLI is installed and available.
|
||||||
*/
|
*/
|
||||||
@@ -63,21 +86,76 @@ export function runGh(args: string[], cwd?: string): string {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Execute a gh CLI command asynchronously.
|
* Execute a gh CLI command asynchronously.
|
||||||
* Returns a promise that resolves with the output or rejects with GhError.
|
*
|
||||||
|
* Returns a promise that resolves with the output or rejects with `GhError`.
|
||||||
|
*
|
||||||
|
* `cwdOrOptions` accepts either a string (cwd, legacy form) or a `RunGhOptions`
|
||||||
|
* object. The options form supports an external `AbortSignal` and a
|
||||||
|
* `timeoutMs` ceiling — both default to safe values that prevent indefinite
|
||||||
|
* hangs when `gh` stalls on the network or a credential helper.
|
||||||
*/
|
*/
|
||||||
export function runGhAsync(args: string[], cwd?: string): Promise<string> {
|
export function runGhAsync(args: string[], cwdOrOptions?: string | RunGhOptions): Promise<string> {
|
||||||
|
const { cwd, signal: externalSignal, timeoutMs = DEFAULT_GH_TIMEOUT_MS } =
|
||||||
|
normalizeRunGhOptions(cwdOrOptions);
|
||||||
|
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
|
if (externalSignal?.aborted) {
|
||||||
|
reject(makeGhError(`gh command aborted: ${describeAbortReason(externalSignal.reason)}`, "ABORT_ERR"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compose the abort sources so we can distinguish timeout from external
|
||||||
|
// abort in the rejection. Using a private controller keeps signal
|
||||||
|
// ownership inside this function.
|
||||||
|
const controller = new AbortController();
|
||||||
|
let timedOut = false;
|
||||||
|
let externalAborted = false;
|
||||||
|
|
||||||
|
const onExternalAbort = () => {
|
||||||
|
externalAborted = true;
|
||||||
|
controller.abort();
|
||||||
|
};
|
||||||
|
if (externalSignal) {
|
||||||
|
externalSignal.addEventListener("abort", onExternalAbort, { once: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
const timer = timeoutMs > 0
|
||||||
|
? setTimeout(() => {
|
||||||
|
timedOut = true;
|
||||||
|
controller.abort();
|
||||||
|
}, timeoutMs)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
const cleanup = () => {
|
||||||
|
if (timer) clearTimeout(timer);
|
||||||
|
if (externalSignal) externalSignal.removeEventListener("abort", onExternalAbort);
|
||||||
|
};
|
||||||
|
|
||||||
execFile(
|
execFile(
|
||||||
"gh",
|
"gh",
|
||||||
args,
|
args,
|
||||||
{
|
{
|
||||||
encoding: "utf-8",
|
encoding: "utf-8",
|
||||||
cwd,
|
cwd,
|
||||||
|
signal: controller.signal,
|
||||||
},
|
},
|
||||||
(error, stdout, stderr) => {
|
(error, stdout, stderr) => {
|
||||||
|
cleanup();
|
||||||
if (error) {
|
if (error) {
|
||||||
const ghError = new Error(`gh command failed: ${error.message}`) as GhError;
|
const isAbort = (error as ExecFileException & { code?: string | number | null }).code === "ABORT_ERR"
|
||||||
ghError.code = error.code ?? null;
|
|| error.name === "AbortError";
|
||||||
|
let message: string;
|
||||||
|
if (timedOut) {
|
||||||
|
message = `gh command timed out after ${timeoutMs}ms`;
|
||||||
|
} else if (isAbort && externalAborted) {
|
||||||
|
message = `gh command aborted: ${describeAbortReason(externalSignal?.reason)}`;
|
||||||
|
} else if (isAbort) {
|
||||||
|
message = "gh command aborted";
|
||||||
|
} else {
|
||||||
|
message = `gh command failed: ${error.message}`;
|
||||||
|
}
|
||||||
|
const ghError = new Error(message) as GhError;
|
||||||
|
ghError.code = (error as ExecFileException).code ?? (isAbort ? "ABORT_ERR" : null);
|
||||||
ghError.stdout = stdout ?? "";
|
ghError.stdout = stdout ?? "";
|
||||||
ghError.stderr = stderr ?? "";
|
ghError.stderr = stderr ?? "";
|
||||||
reject(ghError);
|
reject(ghError);
|
||||||
@@ -89,6 +167,20 @@ export function runGhAsync(args: string[], cwd?: string): Promise<string> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function makeGhError(message: string, code: string | number | null): GhError {
|
||||||
|
const err = new Error(message) as GhError;
|
||||||
|
err.code = code;
|
||||||
|
err.stdout = "";
|
||||||
|
err.stderr = "";
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
|
||||||
|
function describeAbortReason(reason: unknown): string {
|
||||||
|
if (reason instanceof Error) return reason.message;
|
||||||
|
if (typeof reason === "string") return reason;
|
||||||
|
return "aborted";
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Execute a gh CLI command and parse the JSON output.
|
* Execute a gh CLI command and parse the JSON output.
|
||||||
* Requires the command to support --json flag.
|
* Requires the command to support --json flag.
|
||||||
@@ -106,10 +198,13 @@ export function runGhJson<T>(args: string[], cwd?: string): T {
|
|||||||
/**
|
/**
|
||||||
* Execute a gh CLI command asynchronously and parse the JSON output.
|
* Execute a gh CLI command asynchronously and parse the JSON output.
|
||||||
* Requires the command to support --json flag.
|
* Requires the command to support --json flag.
|
||||||
|
*
|
||||||
|
* Forwards `signal` / `timeoutMs` to the underlying `runGhAsync` so callers
|
||||||
|
* can bound the call from outside (e.g. an AI tool's `signal` argument).
|
||||||
*/
|
*/
|
||||||
export async function runGhJsonAsync<T>(args: string[], cwd?: string): Promise<T> {
|
export async function runGhJsonAsync<T>(args: string[], cwdOrOptions?: string | RunGhOptions): Promise<T> {
|
||||||
const jsonArgs = args.includes("--json") ? args : [...args, "--json"];
|
const jsonArgs = args.includes("--json") ? args : [...args, "--json"];
|
||||||
const output = await runGhAsync(jsonArgs, cwd);
|
const output = await runGhAsync(jsonArgs, cwdOrOptions);
|
||||||
try {
|
try {
|
||||||
return JSON.parse(output) as T;
|
return JSON.parse(output) as T;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
132
packages/dashboard/src/__tests__/ai-session-timeout.test.ts
Normal file
132
packages/dashboard/src/__tests__/ai-session-timeout.test.ts
Normal file
@@ -0,0 +1,132 @@
|
|||||||
|
// @vitest-environment node
|
||||||
|
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
import { GenerationGuard, isAbortError, createAbortError } from "../ai-session-timeout.js";
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("GenerationGuard", () => {
|
||||||
|
it("resolves with the operation result when it completes before the timeout", async () => {
|
||||||
|
const guard = new GenerationGuard();
|
||||||
|
const onTimeout = vi.fn();
|
||||||
|
|
||||||
|
const result = await guard.run("s1", 1_000, { onTimeout }, async () => "ok");
|
||||||
|
|
||||||
|
expect(result).toBe("ok");
|
||||||
|
expect(onTimeout).not.toHaveBeenCalled();
|
||||||
|
expect(guard.has("s1")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fires onTimeout, aborts the operation, and rejects with AbortError when the timer fires", async () => {
|
||||||
|
const guard = new GenerationGuard();
|
||||||
|
const onTimeout = vi.fn();
|
||||||
|
|
||||||
|
let opSettled = false;
|
||||||
|
const promise = guard.run("s1", 1_000, { onTimeout }, async () => {
|
||||||
|
// Simulate a hung prompt() that never resolves on its own.
|
||||||
|
await new Promise<void>(() => { /* intentionally never resolves */ });
|
||||||
|
opSettled = true;
|
||||||
|
return "should-not-reach";
|
||||||
|
});
|
||||||
|
|
||||||
|
// Catch promise rejection now so it doesn't become unhandled when timers advance.
|
||||||
|
const settled = promise.then(
|
||||||
|
(v) => ({ ok: true as const, v }),
|
||||||
|
(err) => ({ ok: false as const, err }),
|
||||||
|
);
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(1_000);
|
||||||
|
const outcome = await settled;
|
||||||
|
|
||||||
|
expect(outcome.ok).toBe(false);
|
||||||
|
expect(outcome.ok ? null : outcome.err).toSatisfy((err: unknown) => isAbortError(err));
|
||||||
|
expect(onTimeout).toHaveBeenCalledTimes(1);
|
||||||
|
expect(opSettled).toBe(false);
|
||||||
|
expect(guard.has("s1")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fires onUserStop (not onTimeout) when stop() aborts before the timer", async () => {
|
||||||
|
const guard = new GenerationGuard();
|
||||||
|
const onTimeout = vi.fn();
|
||||||
|
const onUserStop = vi.fn();
|
||||||
|
|
||||||
|
const promise = guard.run("s1", 10_000, { onTimeout, onUserStop }, async () => {
|
||||||
|
await new Promise<void>(() => { /* hang */ });
|
||||||
|
});
|
||||||
|
const settled = promise.then(
|
||||||
|
() => ({ ok: true as const }),
|
||||||
|
(err) => ({ ok: false as const, err }),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Stop before the timer fires.
|
||||||
|
expect(guard.stop("s1")).toBe(true);
|
||||||
|
|
||||||
|
const outcome = await settled;
|
||||||
|
expect(outcome.ok).toBe(false);
|
||||||
|
expect(onTimeout).not.toHaveBeenCalled();
|
||||||
|
expect(onUserStop).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
// Subsequent stop is a no-op.
|
||||||
|
expect(guard.stop("s1")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("re-entrant run for the same id aborts the prior generation", async () => {
|
||||||
|
const guard = new GenerationGuard();
|
||||||
|
const firstUserStop = vi.fn();
|
||||||
|
|
||||||
|
const first = guard.run(
|
||||||
|
"s1",
|
||||||
|
10_000,
|
||||||
|
{ onTimeout: vi.fn(), onUserStop: firstUserStop },
|
||||||
|
async () => { await new Promise<void>(() => { /* hang */ }); },
|
||||||
|
);
|
||||||
|
const firstSettled = first.then(
|
||||||
|
() => ({ ok: true as const }),
|
||||||
|
(err) => ({ ok: false as const, err }),
|
||||||
|
);
|
||||||
|
|
||||||
|
const second = guard.run("s1", 10_000, { onTimeout: vi.fn() }, async () => "fresh");
|
||||||
|
|
||||||
|
await expect(second).resolves.toBe("fresh");
|
||||||
|
const firstOutcome = await firstSettled;
|
||||||
|
expect(firstOutcome.ok).toBe(false);
|
||||||
|
// Re-entrant cancellation goes through the internal cancel path, not the
|
||||||
|
// user-facing stop, so onUserStop is intentionally not fired for the
|
||||||
|
// displaced generation. The displaced caller still observes AbortError.
|
||||||
|
expect(firstUserStop).not.toHaveBeenCalled();
|
||||||
|
expect(guard.has("s1")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reset() aborts every active generation", async () => {
|
||||||
|
const guard = new GenerationGuard();
|
||||||
|
|
||||||
|
const a = guard.run("a", 10_000, { onTimeout: vi.fn() }, async () => {
|
||||||
|
await new Promise<void>(() => { /* hang */ });
|
||||||
|
});
|
||||||
|
const b = guard.run("b", 10_000, { onTimeout: vi.fn() }, async () => {
|
||||||
|
await new Promise<void>(() => { /* hang */ });
|
||||||
|
});
|
||||||
|
const aSettled = a.catch((err) => err);
|
||||||
|
const bSettled = b.catch((err) => err);
|
||||||
|
|
||||||
|
guard.reset();
|
||||||
|
|
||||||
|
expect(isAbortError(await aSettled)).toBe(true);
|
||||||
|
expect(isAbortError(await bSettled)).toBe(true);
|
||||||
|
expect(guard.has("a")).toBe(false);
|
||||||
|
expect(guard.has("b")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("createAbortError is identifiable via isAbortError", () => {
|
||||||
|
expect(isAbortError(createAbortError())).toBe(true);
|
||||||
|
expect(isAbortError(new Error("other"))).toBe(false);
|
||||||
|
expect(isAbortError("not an error")).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -33,6 +33,8 @@ import {
|
|||||||
skipTargetInterview,
|
skipTargetInterview,
|
||||||
MILESTONE_INTERVIEW_SYSTEM_PROMPT,
|
MILESTONE_INTERVIEW_SYSTEM_PROMPT,
|
||||||
SLICE_INTERVIEW_SYSTEM_PROMPT,
|
SLICE_INTERVIEW_SYSTEM_PROMPT,
|
||||||
|
stopMilestoneSliceInterviewGeneration,
|
||||||
|
GENERATION_TIMEOUT_MS,
|
||||||
type MilestoneInterviewSummary,
|
type MilestoneInterviewSummary,
|
||||||
type SliceInterviewSummary,
|
type SliceInterviewSummary,
|
||||||
} from "../milestone-slice-interview.js";
|
} from "../milestone-slice-interview.js";
|
||||||
@@ -769,4 +771,83 @@ describe("milestone-slice-interview module", () => {
|
|||||||
expect(SLICE_INTERVIEW_SYSTEM_PROMPT).toContain("acceptanceCriteria");
|
expect(SLICE_INTERVIEW_SYSTEM_PROMPT).toContain("acceptanceCriteria");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("generation timeout / abort", () => {
|
||||||
|
it("marks the session as error when initial generation exceeds GENERATION_TIMEOUT_MS", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
|
||||||
|
let resolveHungPrompt: (() => void) | undefined;
|
||||||
|
mockCreateFnAgent.mockImplementationOnce(async () => ({
|
||||||
|
session: {
|
||||||
|
state: { messages: [] },
|
||||||
|
prompt: vi.fn(async () => {
|
||||||
|
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||||
|
}),
|
||||||
|
dispose: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
const sessionId = await createTargetInterviewSession(
|
||||||
|
"10.0.1.10",
|
||||||
|
"milestone",
|
||||||
|
"milestone-stuck",
|
||||||
|
"Hung milestone interview",
|
||||||
|
undefined,
|
||||||
|
"/tmp/project",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Yield so the guard registration runs.
|
||||||
|
await Promise.resolve();
|
||||||
|
await Promise.resolve();
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(GENERATION_TIMEOUT_MS);
|
||||||
|
await vi.advanceTimersByTimeAsync(0);
|
||||||
|
|
||||||
|
const session = getTargetInterviewSession(sessionId);
|
||||||
|
expect(session?.error).toMatch(/timed out/i);
|
||||||
|
|
||||||
|
resolveHungPrompt?.();
|
||||||
|
await vi.advanceTimersByTimeAsync(0);
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stopMilestoneSliceInterviewGeneration aborts an in-flight session and marks it stopped", async () => {
|
||||||
|
let resolveHungPrompt: (() => void) | undefined;
|
||||||
|
mockCreateFnAgent.mockImplementationOnce(async () => ({
|
||||||
|
session: {
|
||||||
|
state: { messages: [] },
|
||||||
|
prompt: vi.fn(async () => {
|
||||||
|
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||||
|
}),
|
||||||
|
dispose: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
const sessionId = await createTargetInterviewSession(
|
||||||
|
"10.0.1.11",
|
||||||
|
"slice",
|
||||||
|
"slice-stoppable",
|
||||||
|
"Stoppable slice interview",
|
||||||
|
undefined,
|
||||||
|
"/tmp/project",
|
||||||
|
);
|
||||||
|
|
||||||
|
let stopped = false;
|
||||||
|
for (let i = 0; i < 50 && !stopped; i++) {
|
||||||
|
stopped = stopMilestoneSliceInterviewGeneration(sessionId);
|
||||||
|
if (!stopped) await new Promise((resolve) => setTimeout(resolve, 5));
|
||||||
|
}
|
||||||
|
expect(stopped).toBe(true);
|
||||||
|
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
|
||||||
|
const session = getTargetInterviewSession(sessionId);
|
||||||
|
expect(session?.error).toMatch(/stopped by user/i);
|
||||||
|
expect(stopMilestoneSliceInterviewGeneration(sessionId)).toBe(false);
|
||||||
|
|
||||||
|
resolveHungPrompt?.();
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -27,7 +27,9 @@ import {
|
|||||||
setAiSessionStore,
|
setAiSessionStore,
|
||||||
RateLimitError,
|
RateLimitError,
|
||||||
SessionNotFoundError,
|
SessionNotFoundError,
|
||||||
|
stopMissionInterviewGeneration,
|
||||||
submitMissionInterviewResponse,
|
submitMissionInterviewResponse,
|
||||||
|
GENERATION_TIMEOUT_MS,
|
||||||
} from "../mission-interview.js";
|
} from "../mission-interview.js";
|
||||||
import {
|
import {
|
||||||
setDiagnosticsSink,
|
setDiagnosticsSink,
|
||||||
@@ -927,4 +929,85 @@ describe("mission-interview module", () => {
|
|||||||
expect(lastCall[0].systemPrompt).toMatch(/^You are a mission planning assistant/);
|
expect(lastCall[0].systemPrompt).toMatch(/^You are a mission planning assistant/);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("generation timeout / abort", () => {
|
||||||
|
it("marks the session as error when initial generation exceeds GENERATION_TIMEOUT_MS", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
|
||||||
|
let resolveHungPrompt: (() => void) | undefined;
|
||||||
|
mockCreateFnAgent.mockImplementationOnce(async () => ({
|
||||||
|
session: {
|
||||||
|
state: { messages: [] },
|
||||||
|
prompt: vi.fn(async () => {
|
||||||
|
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||||
|
}),
|
||||||
|
dispose: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
const sessionId = await createMissionInterviewSession(
|
||||||
|
"10.0.0.10",
|
||||||
|
"Hung mission interview",
|
||||||
|
"/tmp/project",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Yield so initializeAgent's async work registers the generation guard.
|
||||||
|
await Promise.resolve();
|
||||||
|
await Promise.resolve();
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(GENERATION_TIMEOUT_MS);
|
||||||
|
await vi.advanceTimersByTimeAsync(0);
|
||||||
|
|
||||||
|
const session = getMissionInterviewSession(sessionId);
|
||||||
|
expect(session?.error).toMatch(/timed out/i);
|
||||||
|
|
||||||
|
resolveHungPrompt?.();
|
||||||
|
await vi.advanceTimersByTimeAsync(0);
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stopMissionInterviewGeneration aborts an in-flight session and marks it stopped", async () => {
|
||||||
|
// Real timers — we want the guard.run() registration to actually happen
|
||||||
|
// through normal microtask scheduling without us racing it.
|
||||||
|
let resolveHungPrompt: (() => void) | undefined;
|
||||||
|
mockCreateFnAgent.mockImplementationOnce(async () => ({
|
||||||
|
session: {
|
||||||
|
state: { messages: [] },
|
||||||
|
prompt: vi.fn(async () => {
|
||||||
|
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||||
|
}),
|
||||||
|
dispose: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
const sessionId = await createMissionInterviewSession(
|
||||||
|
"10.0.0.11",
|
||||||
|
"Stoppable mission interview",
|
||||||
|
"/tmp/project",
|
||||||
|
);
|
||||||
|
|
||||||
|
// initializeAgent → createMissionInterviewAgent → continueAgentConversation
|
||||||
|
// → generationGuard.run is several awaits deep; poll until the guard is
|
||||||
|
// registered, then stop. Bounded so a regression doesn't hang the suite.
|
||||||
|
let stopped = false;
|
||||||
|
for (let i = 0; i < 50 && !stopped; i++) {
|
||||||
|
stopped = stopMissionInterviewGeneration(sessionId);
|
||||||
|
if (!stopped) await new Promise((resolve) => setTimeout(resolve, 5));
|
||||||
|
}
|
||||||
|
expect(stopped).toBe(true);
|
||||||
|
|
||||||
|
// Yield so the guard's catch/finally and onUserStop run.
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
|
||||||
|
const session = getMissionInterviewSession(sessionId);
|
||||||
|
expect(session?.error).toMatch(/stopped by user/i);
|
||||||
|
|
||||||
|
// Stop is idempotent — no in-flight generation after first call.
|
||||||
|
expect(stopMissionInterviewGeneration(sessionId)).toBe(false);
|
||||||
|
|
||||||
|
resolveHungPrompt?.();
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -31,7 +31,9 @@ import {
|
|||||||
SessionNotFoundError,
|
SessionNotFoundError,
|
||||||
InvalidSessionStateError,
|
InvalidSessionStateError,
|
||||||
setAiSessionStore,
|
setAiSessionStore,
|
||||||
|
stopSubtaskGeneration,
|
||||||
SubtaskStreamManager,
|
SubtaskStreamManager,
|
||||||
|
GENERATION_TIMEOUT_MS,
|
||||||
} from "../subtask-breakdown.js";
|
} from "../subtask-breakdown.js";
|
||||||
|
|
||||||
const UUID_REGEX =
|
const UUID_REGEX =
|
||||||
@@ -913,3 +915,85 @@ describe("SessionNotFoundError", () => {
|
|||||||
expect(error.message).toBe("Missing session");
|
expect(error.message).toBe("Missing session");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("subtask generation timeout / abort", () => {
|
||||||
|
it("marks the session as error and stops the prompt() promise when generation exceeds GENERATION_TIMEOUT_MS", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
|
||||||
|
let resolveHungPrompt: (() => void) | undefined;
|
||||||
|
const hungPromptCallable = vi.fn(async () => {
|
||||||
|
// Simulate a stalled provider stream that never terminates on its own.
|
||||||
|
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||||
|
return undefined;
|
||||||
|
});
|
||||||
|
|
||||||
|
mockCreateFnAgent.mockImplementation(async () => ({
|
||||||
|
session: {
|
||||||
|
state: { messages: [] },
|
||||||
|
prompt: hungPromptCallable,
|
||||||
|
dispose: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
const created = await createSubtaskSession(
|
||||||
|
"Hung subtask generation",
|
||||||
|
undefined,
|
||||||
|
"/tmp/project",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Yield once so startSubtaskGeneration's microtasks run before we advance time.
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(getSubtaskSession(created.sessionId)?.status).toBe("generating");
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(GENERATION_TIMEOUT_MS);
|
||||||
|
// Drain any remaining microtasks (the abort propagates through Promise.race).
|
||||||
|
await vi.advanceTimersByTimeAsync(0);
|
||||||
|
|
||||||
|
const after = getSubtaskSession(created.sessionId);
|
||||||
|
expect(after?.status).toBe("error");
|
||||||
|
expect(after?.error).toMatch(/timed out/i);
|
||||||
|
|
||||||
|
// The hung prompt is still pending; release it so its microtask completes.
|
||||||
|
resolveHungPrompt?.();
|
||||||
|
await vi.advanceTimersByTimeAsync(0);
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stopSubtaskGeneration aborts an in-flight session and marks it stopped", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
|
||||||
|
let resolveHungPrompt: (() => void) | undefined;
|
||||||
|
mockCreateFnAgent.mockImplementation(async () => ({
|
||||||
|
session: {
|
||||||
|
state: { messages: [] },
|
||||||
|
prompt: vi.fn(async () => {
|
||||||
|
await new Promise<void>((resolve) => { resolveHungPrompt = resolve; });
|
||||||
|
}),
|
||||||
|
dispose: vi.fn(),
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
|
||||||
|
const created = await createSubtaskSession(
|
||||||
|
"Stoppable subtask generation",
|
||||||
|
undefined,
|
||||||
|
"/tmp/project",
|
||||||
|
);
|
||||||
|
await Promise.resolve();
|
||||||
|
|
||||||
|
expect(stopSubtaskGeneration(created.sessionId)).toBe(true);
|
||||||
|
await vi.advanceTimersByTimeAsync(0);
|
||||||
|
|
||||||
|
const after = getSubtaskSession(created.sessionId);
|
||||||
|
expect(after?.status).toBe("error");
|
||||||
|
expect(after?.error).toMatch(/stopped by user/i);
|
||||||
|
|
||||||
|
// Stop is idempotent — no in-flight generation after first call.
|
||||||
|
expect(stopSubtaskGeneration(created.sessionId)).toBe(false);
|
||||||
|
|
||||||
|
resolveHungPrompt?.();
|
||||||
|
await vi.advanceTimersByTimeAsync(0);
|
||||||
|
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
138
packages/dashboard/src/ai-session-timeout.ts
Normal file
138
packages/dashboard/src/ai-session-timeout.ts
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
/**
|
||||||
|
* Per-session generation guard: bounds the runtime of an in-flight AI
|
||||||
|
* `prompt()` call with a timeout and an `AbortController`, so a silently-
|
||||||
|
* stalled model stream or hung tool call cannot leave a session pinned in
|
||||||
|
* `generating` forever.
|
||||||
|
*
|
||||||
|
* The guard is module-scoped: planning, subtask-breakdown, mission-interview,
|
||||||
|
* and milestone-slice-interview each instantiate their own. The session ID is
|
||||||
|
* the key, so concurrent generations across modules don't collide.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface TimeoutHandlers {
|
||||||
|
/** Fired exactly once when the timeout elapses, before the abort propagates. */
|
||||||
|
onTimeout: () => void;
|
||||||
|
/** Fired when abort happens for a non-timeout reason (e.g. manual stop). */
|
||||||
|
onUserStop?: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ActiveEntry {
|
||||||
|
abort: AbortController;
|
||||||
|
timer: ReturnType<typeof setTimeout>;
|
||||||
|
}
|
||||||
|
|
||||||
|
type AbortCause = "timeout" | "user-stop" | "displaced";
|
||||||
|
|
||||||
|
export class GenerationGuard {
|
||||||
|
private readonly active = new Map<string, ActiveEntry>();
|
||||||
|
/**
|
||||||
|
* Tracks the cause of a pending abort so the original `run()` can
|
||||||
|
* distinguish a user-initiated stop from re-entrant displacement. The flag
|
||||||
|
* is consumed (deleted) by the catch block of the displaced `run()`.
|
||||||
|
*/
|
||||||
|
private readonly abortCause = new Map<AbortController, AbortCause>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wrap `op` with a timeout + abort. If a previous generation is still
|
||||||
|
* registered for the same id, it is aborted first (the prior `run()`
|
||||||
|
* rejects with `AbortError`, marked as `displaced` so its `onUserStop`
|
||||||
|
* handler does NOT fire).
|
||||||
|
*/
|
||||||
|
async run<T>(
|
||||||
|
sessionId: string,
|
||||||
|
timeoutMs: number,
|
||||||
|
handlers: TimeoutHandlers,
|
||||||
|
op: () => Promise<T>,
|
||||||
|
): Promise<T> {
|
||||||
|
this.cancelInternal(sessionId, "displaced");
|
||||||
|
|
||||||
|
const abort = new AbortController();
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
this.abortCause.set(abort, "timeout");
|
||||||
|
try {
|
||||||
|
handlers.onTimeout();
|
||||||
|
} catch {
|
||||||
|
// swallow — handler errors must not prevent abort
|
||||||
|
}
|
||||||
|
abort.abort();
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
|
const entry: ActiveEntry = { abort, timer };
|
||||||
|
this.active.set(sessionId, entry);
|
||||||
|
|
||||||
|
const abortPromise = new Promise<never>((_, reject) => {
|
||||||
|
abort.signal.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => reject(createAbortError()),
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await Promise.race([op(), abortPromise]);
|
||||||
|
} catch (err) {
|
||||||
|
if (isAbortError(err)) {
|
||||||
|
const cause = this.abortCause.get(abort) ?? "user-stop";
|
||||||
|
if (cause === "user-stop") {
|
||||||
|
try {
|
||||||
|
handlers.onUserStop?.();
|
||||||
|
} catch {
|
||||||
|
// swallow
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timer);
|
||||||
|
this.abortCause.delete(abort);
|
||||||
|
if (this.active.get(sessionId) === entry) {
|
||||||
|
this.active.delete(sessionId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** AbortSignal of the in-flight generation, if any — for tools that honor it. */
|
||||||
|
signal(sessionId: string): AbortSignal | undefined {
|
||||||
|
return this.active.get(sessionId)?.abort.signal;
|
||||||
|
}
|
||||||
|
|
||||||
|
has(sessionId: string): boolean {
|
||||||
|
return this.active.has(sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manually abort the active generation. Returns true if there was one.
|
||||||
|
* The wrapped operation will reject with `AbortError`, and the `onUserStop`
|
||||||
|
* handler from the original `run()` call will fire.
|
||||||
|
*/
|
||||||
|
stop(sessionId: string): boolean {
|
||||||
|
return this.cancelInternal(sessionId, "user-stop");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Reset all in-flight generations (test/shutdown only). */
|
||||||
|
reset(): void {
|
||||||
|
for (const sessionId of [...this.active.keys()]) {
|
||||||
|
this.cancelInternal(sessionId, "user-stop");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private cancelInternal(sessionId: string, cause: AbortCause): boolean {
|
||||||
|
const entry = this.active.get(sessionId);
|
||||||
|
if (!entry) return false;
|
||||||
|
clearTimeout(entry.timer);
|
||||||
|
this.abortCause.set(entry.abort, cause);
|
||||||
|
entry.abort.abort();
|
||||||
|
this.active.delete(sessionId);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createAbortError(): Error {
|
||||||
|
const error = new Error("Generation aborted");
|
||||||
|
error.name = "AbortError";
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isAbortError(err: unknown): boolean {
|
||||||
|
return err instanceof Error && err.name === "AbortError";
|
||||||
|
}
|
||||||
@@ -29,6 +29,7 @@ import {
|
|||||||
resetDiagnosticsSink,
|
resetDiagnosticsSink,
|
||||||
nonfatal,
|
nonfatal,
|
||||||
} from "./ai-session-diagnostics.js";
|
} from "./ai-session-diagnostics.js";
|
||||||
|
import { GenerationGuard, isAbortError } from "./ai-session-timeout.js";
|
||||||
|
|
||||||
// Re-export JSON parsing utilities from mission-interview for external consumers
|
// Re-export JSON parsing utilities from mission-interview for external consumers
|
||||||
export {
|
export {
|
||||||
@@ -122,6 +123,14 @@ const RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000;
|
|||||||
/** Max number of retry attempts when AI returns unparseable output */
|
/** Max number of retry attempts when AI returns unparseable output */
|
||||||
const MAX_PARSE_RETRIES = 1;
|
const MAX_PARSE_RETRIES = 1;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Per-turn generation timeout. Bounds a stalled model stream or hung tool
|
||||||
|
* call so the session cannot stay pinned in `generating` indefinitely.
|
||||||
|
*/
|
||||||
|
export const GENERATION_TIMEOUT_MS = 120_000;
|
||||||
|
|
||||||
|
const generationGuard = new GenerationGuard();
|
||||||
|
|
||||||
/** Milestone interview system prompt */
|
/** Milestone interview system prompt */
|
||||||
export const MILESTONE_INTERVIEW_SYSTEM_PROMPT = `You are a milestone planning assistant for a project management system.
|
export const MILESTONE_INTERVIEW_SYSTEM_PROMPT = `You are a milestone planning assistant for a project management system.
|
||||||
|
|
||||||
@@ -346,6 +355,9 @@ function cleanupInMemorySession(sessionId: string): boolean {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Abort any in-flight generation so prompt() rejects promptly.
|
||||||
|
generationGuard.stop(sessionId);
|
||||||
|
|
||||||
if (session.agent) {
|
if (session.agent) {
|
||||||
try { session.agent.session.dispose?.(); } catch { /* ignore */ }
|
try { session.agent.session.dispose?.(); } catch { /* ignore */ }
|
||||||
session.agent = undefined;
|
session.agent = undefined;
|
||||||
@@ -356,6 +368,24 @@ function cleanupInMemorySession(sessionId: string): boolean {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function setTargetSessionError(session: TargetInterviewSession, message: string): void {
|
||||||
|
session.error = message;
|
||||||
|
session.updatedAt = new Date();
|
||||||
|
persistSession(session, "error", message);
|
||||||
|
milestoneSliceInterviewStreamManager.broadcast(session.id, {
|
||||||
|
type: "error",
|
||||||
|
data: message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manually abort an in-flight milestone/slice interview generation.
|
||||||
|
* Returns true if a generation was active and got aborted.
|
||||||
|
*/
|
||||||
|
export function stopMilestoneSliceInterviewGeneration(sessionId: string): boolean {
|
||||||
|
return generationGuard.stop(sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
function getSessionType(targetType: TargetType): "milestone_interview" | "slice_interview" {
|
function getSessionType(targetType: TargetType): "milestone_interview" | "slice_interview" {
|
||||||
return targetType === "milestone" ? "milestone_interview" : "slice_interview";
|
return targetType === "milestone" ? "milestone_interview" : "slice_interview";
|
||||||
}
|
}
|
||||||
@@ -744,12 +774,26 @@ async function ensureInterviewAgent(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await session.agent.session.prompt(
|
await generationGuard.run(
|
||||||
[
|
session.id,
|
||||||
"Previous conversation summary:",
|
GENERATION_TIMEOUT_MS,
|
||||||
historySummary,
|
{
|
||||||
"Use this context when handling the next user response.",
|
onTimeout: () => setTargetSessionError(
|
||||||
].join("\n\n"),
|
session,
|
||||||
|
"AI generation timed out while restoring context. You can retry or start a new session.",
|
||||||
|
),
|
||||||
|
onUserStop: () => setTargetSessionError(
|
||||||
|
session,
|
||||||
|
"Generation stopped by user. You can retry or start a new session.",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
() => session.agent!.session.prompt(
|
||||||
|
[
|
||||||
|
"Previous conversation summary:",
|
||||||
|
historySummary,
|
||||||
|
"Use this context when handling the next user response.",
|
||||||
|
].join("\n\n"),
|
||||||
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -791,128 +835,138 @@ async function continueAgentConversation(session: TargetInterviewSession, messag
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
session.thinkingOutput = "";
|
await generationGuard.run(
|
||||||
|
session.id,
|
||||||
|
GENERATION_TIMEOUT_MS,
|
||||||
|
{
|
||||||
|
onTimeout: () => setTargetSessionError(
|
||||||
|
session,
|
||||||
|
"AI generation timed out. You can retry or start a new session.",
|
||||||
|
),
|
||||||
|
onUserStop: () => setTargetSessionError(
|
||||||
|
session,
|
||||||
|
"Generation stopped by user. You can retry or start a new session.",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
const agent = session.agent!;
|
||||||
|
session.thinkingOutput = "";
|
||||||
|
|
||||||
await session.agent.session.prompt(message);
|
await agent.session.prompt(message);
|
||||||
|
|
||||||
// Get the response text from the agent's state
|
// Get the response text from the agent's state
|
||||||
interface AgentMessage {
|
interface AgentMessage {
|
||||||
role: string;
|
role: string;
|
||||||
content?: string | Array<{ type: string; text: string }>;
|
content?: string | Array<{ type: string; text: string }>;
|
||||||
}
|
}
|
||||||
const lastMessage = (session.agent.session.state.messages as AgentMessage[])
|
const lastMessage = (agent.session.state.messages as AgentMessage[])
|
||||||
.filter((m: AgentMessage) => m.role === "assistant")
|
.filter((m: AgentMessage) => m.role === "assistant")
|
||||||
.pop();
|
.pop();
|
||||||
|
|
||||||
let responseText = session.thinkingOutput;
|
let responseText = session.thinkingOutput;
|
||||||
if (lastMessage?.content) {
|
if (lastMessage?.content) {
|
||||||
if (typeof lastMessage.content === "string") {
|
if (typeof lastMessage.content === "string") {
|
||||||
responseText = lastMessage.content;
|
responseText = lastMessage.content;
|
||||||
} else if (Array.isArray(lastMessage.content)) {
|
} else if (Array.isArray(lastMessage.content)) {
|
||||||
responseText = lastMessage.content
|
responseText = lastMessage.content
|
||||||
.filter((c: { type: string; text: string }): c is { type: "text"; text: string } => c.type === "text")
|
.filter((c: { type: string; text: string }): c is { type: "text"; text: string } => c.type === "text")
|
||||||
.map((c: { type: string; text: string }) => c.text)
|
.map((c: { type: string; text: string }) => c.text)
|
||||||
.join("");
|
.join("");
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse with retry using the target interview parser
|
|
||||||
let parsed: TargetInterviewResponse | undefined;
|
|
||||||
let lastError: Error | undefined;
|
|
||||||
|
|
||||||
for (let attempt = 0; attempt <= MAX_PARSE_RETRIES; attempt++) {
|
|
||||||
try {
|
|
||||||
parsed = parseTargetInterviewResponseImpl(responseText);
|
|
||||||
break;
|
|
||||||
} catch (err) {
|
|
||||||
lastError = err instanceof Error ? err : new Error(String(err));
|
|
||||||
|
|
||||||
if (attempt < MAX_PARSE_RETRIES) {
|
|
||||||
diagnostics.warn(
|
|
||||||
"Parse attempt failed, requesting reformat",
|
|
||||||
{ sessionId: session.id, attempt: attempt + 1, operation: "parse-retry" }
|
|
||||||
);
|
|
||||||
try {
|
|
||||||
session.thinkingOutput = "";
|
|
||||||
await session.agent.session.prompt(
|
|
||||||
"Your previous response could not be parsed as JSON. " +
|
|
||||||
'Please respond with ONLY a valid JSON object: either {"type":"question","data":{...}} ' +
|
|
||||||
'or {"type":"complete","data":{"title":"...","description":"...","planningNotes":"...","verification":"..."}}' +
|
|
||||||
". No markdown, no explanation, just the JSON."
|
|
||||||
);
|
|
||||||
|
|
||||||
const retryMessage = (session.agent.session.state.messages as AgentMessage[])
|
|
||||||
.filter((m: AgentMessage) => m.role === "assistant")
|
|
||||||
.pop();
|
|
||||||
|
|
||||||
let retryText = session.thinkingOutput;
|
|
||||||
if (retryMessage?.content) {
|
|
||||||
if (typeof retryMessage.content === "string") {
|
|
||||||
retryText = retryMessage.content;
|
|
||||||
} else if (Array.isArray(retryMessage.content)) {
|
|
||||||
retryText = retryMessage.content
|
|
||||||
.filter((c: { type: string; text: string }): c is { type: "text"; text: string } => c.type === "text")
|
|
||||||
.map((c: { type: string; text: string }) => c.text)
|
|
||||||
.join("");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
responseText = retryText;
|
|
||||||
} catch (retryErr) {
|
|
||||||
diagnostics.errorFromException("Retry prompt failed for session", retryErr, { sessionId: session.id, operation: "retry-prompt" });
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!parsed) {
|
// Parse with retry using the target interview parser
|
||||||
const errorMsg = `${lastError?.message || "Failed to parse AI response"} You can try responding again or start a new session.`;
|
let parsed: TargetInterviewResponse | undefined;
|
||||||
diagnostics.error(
|
let lastError: Error | undefined;
|
||||||
"All parse attempts exhausted for session",
|
|
||||||
{ sessionId: session.id, message: errorMsg, operation: "parse-exhausted" }
|
for (let attempt = 0; attempt <= MAX_PARSE_RETRIES; attempt++) {
|
||||||
);
|
try {
|
||||||
session.error = errorMsg;
|
parsed = parseTargetInterviewResponseImpl(responseText);
|
||||||
session.updatedAt = new Date();
|
break;
|
||||||
persistSession(session, "error", errorMsg);
|
} catch (err) {
|
||||||
milestoneSliceInterviewStreamManager.broadcast(session.id, {
|
lastError = err instanceof Error ? err : new Error(String(err));
|
||||||
type: "error",
|
|
||||||
data: errorMsg,
|
if (attempt < MAX_PARSE_RETRIES) {
|
||||||
});
|
diagnostics.warn(
|
||||||
|
"Parse attempt failed, requesting reformat",
|
||||||
|
{ sessionId: session.id, attempt: attempt + 1, operation: "parse-retry" }
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
session.thinkingOutput = "";
|
||||||
|
await agent.session.prompt(
|
||||||
|
"Your previous response could not be parsed as JSON. " +
|
||||||
|
'Please respond with ONLY a valid JSON object: either {"type":"question","data":{...}} ' +
|
||||||
|
'or {"type":"complete","data":{"title":"...","description":"...","planningNotes":"...","verification":"..."}}' +
|
||||||
|
". No markdown, no explanation, just the JSON."
|
||||||
|
);
|
||||||
|
|
||||||
|
const retryMessage = (agent.session.state.messages as AgentMessage[])
|
||||||
|
.filter((m: AgentMessage) => m.role === "assistant")
|
||||||
|
.pop();
|
||||||
|
|
||||||
|
let retryText = session.thinkingOutput;
|
||||||
|
if (retryMessage?.content) {
|
||||||
|
if (typeof retryMessage.content === "string") {
|
||||||
|
retryText = retryMessage.content;
|
||||||
|
} else if (Array.isArray(retryMessage.content)) {
|
||||||
|
retryText = retryMessage.content
|
||||||
|
.filter((c: { type: string; text: string }): c is { type: "text"; text: string } => c.type === "text")
|
||||||
|
.map((c: { type: string; text: string }) => c.text)
|
||||||
|
.join("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
responseText = retryText;
|
||||||
|
} catch (retryErr) {
|
||||||
|
diagnostics.errorFromException("Retry prompt failed for session", retryErr, { sessionId: session.id, operation: "retry-prompt" });
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!parsed) {
|
||||||
|
const errorMsg = `${lastError?.message || "Failed to parse AI response"} You can try responding again or start a new session.`;
|
||||||
|
diagnostics.error(
|
||||||
|
"All parse attempts exhausted for session",
|
||||||
|
{ sessionId: session.id, message: errorMsg, operation: "parse-exhausted" }
|
||||||
|
);
|
||||||
|
setTargetSessionError(session, errorMsg);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.type === "question") {
|
||||||
|
session.currentQuestion = parsed.data;
|
||||||
|
session.error = undefined;
|
||||||
|
session.lastGeneratedThinking = session.thinkingOutput;
|
||||||
|
session.updatedAt = new Date();
|
||||||
|
persistSession(session, "awaiting_input");
|
||||||
|
milestoneSliceInterviewStreamManager.broadcast(session.id, {
|
||||||
|
type: "question",
|
||||||
|
data: parsed.data,
|
||||||
|
});
|
||||||
|
} else if (parsed.type === "complete") {
|
||||||
|
session.summary = parsed.data;
|
||||||
|
session.currentQuestion = undefined;
|
||||||
|
session.error = undefined;
|
||||||
|
session.updatedAt = new Date();
|
||||||
|
persistSession(session, "complete");
|
||||||
|
milestoneSliceInterviewStreamManager.broadcast(session.id, {
|
||||||
|
type: "summary",
|
||||||
|
data: parsed.data,
|
||||||
|
});
|
||||||
|
milestoneSliceInterviewStreamManager.broadcast(session.id, { type: "complete" });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
// Timeout / user-stop already published an error state via the guard
|
||||||
|
// handlers. Don't double-broadcast a generic AbortError.
|
||||||
|
if (isAbortError(err)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (parsed.type === "question") {
|
|
||||||
session.currentQuestion = parsed.data;
|
|
||||||
session.error = undefined;
|
|
||||||
session.lastGeneratedThinking = session.thinkingOutput;
|
|
||||||
session.updatedAt = new Date();
|
|
||||||
persistSession(session, "awaiting_input");
|
|
||||||
milestoneSliceInterviewStreamManager.broadcast(session.id, {
|
|
||||||
type: "question",
|
|
||||||
data: parsed.data,
|
|
||||||
});
|
|
||||||
} else if (parsed.type === "complete") {
|
|
||||||
session.summary = parsed.data;
|
|
||||||
session.currentQuestion = undefined;
|
|
||||||
session.error = undefined;
|
|
||||||
session.updatedAt = new Date();
|
|
||||||
persistSession(session, "complete");
|
|
||||||
milestoneSliceInterviewStreamManager.broadcast(session.id, {
|
|
||||||
type: "summary",
|
|
||||||
data: parsed.data,
|
|
||||||
});
|
|
||||||
milestoneSliceInterviewStreamManager.broadcast(session.id, { type: "complete" });
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
const errorMessage = err instanceof Error ? err.message : "AI processing failed";
|
const errorMessage = err instanceof Error ? err.message : "AI processing failed";
|
||||||
diagnostics.errorFromException("Agent conversation error for session", err, { sessionId: session.id, operation: "conversation" });
|
diagnostics.errorFromException("Agent conversation error for session", err, { sessionId: session.id, operation: "conversation" });
|
||||||
session.error = errorMessage;
|
setTargetSessionError(session, errorMessage);
|
||||||
session.updatedAt = new Date();
|
|
||||||
persistSession(session, "error", errorMessage);
|
|
||||||
milestoneSliceInterviewStreamManager.broadcast(session.id, {
|
|
||||||
type: "error",
|
|
||||||
data: errorMessage,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1265,6 +1319,7 @@ export function __resetMilestoneSliceInterviewState(): void {
|
|||||||
sessions.clear();
|
sessions.clear();
|
||||||
rateLimits.clear();
|
rateLimits.clear();
|
||||||
milestoneSliceInterviewStreamManager.reset();
|
milestoneSliceInterviewStreamManager.reset();
|
||||||
|
generationGuard.reset();
|
||||||
|
|
||||||
if (_aiSessionStore && _aiSessionDeletedListener) {
|
if (_aiSessionStore && _aiSessionDeletedListener) {
|
||||||
_aiSessionStore.off("ai_session:deleted", _aiSessionDeletedListener);
|
_aiSessionStore.off("ai_session:deleted", _aiSessionDeletedListener);
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import {
|
|||||||
resetDiagnosticsSink,
|
resetDiagnosticsSink,
|
||||||
nonfatal,
|
nonfatal,
|
||||||
} from "./ai-session-diagnostics.js";
|
} from "./ai-session-diagnostics.js";
|
||||||
|
import { GenerationGuard, isAbortError } from "./ai-session-timeout.js";
|
||||||
|
|
||||||
import { createFnAgent as engineCreateFnAgent } from "@fusion/engine";
|
import { createFnAgent as engineCreateFnAgent } from "@fusion/engine";
|
||||||
|
|
||||||
@@ -86,6 +87,15 @@ const RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000;
|
|||||||
/** Max number of retry attempts when AI returns unparseable output */
|
/** Max number of retry attempts when AI returns unparseable output */
|
||||||
const MAX_PARSE_RETRIES = 1;
|
const MAX_PARSE_RETRIES = 1;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Per-turn generation timeout. Mission interview turns produce larger plans
|
||||||
|
* than planning, so this is more generous than planning's 120 s. Bounds the
|
||||||
|
* worst case for a silently-stalled model stream or hung tool call.
|
||||||
|
*/
|
||||||
|
export const GENERATION_TIMEOUT_MS = 180_000;
|
||||||
|
|
||||||
|
const generationGuard = new GenerationGuard();
|
||||||
|
|
||||||
/** Mission interview system prompt */
|
/** Mission interview system prompt */
|
||||||
export const MISSION_INTERVIEW_SYSTEM_PROMPT = `You are a mission planning assistant for a project management system.
|
export const MISSION_INTERVIEW_SYSTEM_PROMPT = `You are a mission planning assistant for a project management system.
|
||||||
|
|
||||||
@@ -283,6 +293,9 @@ function cleanupInMemoryMissionSession(sessionId: string): boolean {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Abort any in-flight generation so prompt() rejects promptly.
|
||||||
|
generationGuard.stop(sessionId);
|
||||||
|
|
||||||
if (session.agent) {
|
if (session.agent) {
|
||||||
try { session.agent.session.dispose?.(); } catch { /* ignore */ }
|
try { session.agent.session.dispose?.(); } catch { /* ignore */ }
|
||||||
session.agent = undefined;
|
session.agent = undefined;
|
||||||
@@ -859,18 +872,54 @@ async function ensureMissionInterviewAgent(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await session.agent.session.prompt(
|
await generationGuard.run(
|
||||||
[
|
session.id,
|
||||||
"Previous conversation summary:",
|
GENERATION_TIMEOUT_MS,
|
||||||
historySummary,
|
{
|
||||||
"Use this context when handling the next user response.",
|
onTimeout: () => setMissionSessionError(
|
||||||
].join("\n\n"),
|
session,
|
||||||
|
"AI generation timed out while restoring context. You can retry or start a new session.",
|
||||||
|
),
|
||||||
|
onUserStop: () => setMissionSessionError(
|
||||||
|
session,
|
||||||
|
"Generation stopped by user. You can retry or start a new session.",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
() => session.agent!.session.prompt(
|
||||||
|
[
|
||||||
|
"Previous conversation summary:",
|
||||||
|
historySummary,
|
||||||
|
"Use this context when handling the next user response.",
|
||||||
|
].join("\n\n"),
|
||||||
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function setMissionSessionError(session: MissionInterviewSession, message: string): void {
|
||||||
|
session.error = message;
|
||||||
|
session.updatedAt = new Date();
|
||||||
|
persistMissionSession(session, "error", message);
|
||||||
|
missionInterviewStreamManager.broadcast(session.id, {
|
||||||
|
type: "error",
|
||||||
|
data: message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manually abort an in-flight mission interview generation. Returns true if
|
||||||
|
* a generation was active and got aborted.
|
||||||
|
*/
|
||||||
|
export function stopMissionInterviewGeneration(sessionId: string): boolean {
|
||||||
|
return generationGuard.stop(sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Continue the AI conversation with a user message.
|
* Continue the AI conversation with a user message.
|
||||||
* Includes bounded recovery: one retry on parse failure.
|
* Includes bounded recovery: one retry on parse failure.
|
||||||
|
*
|
||||||
|
* The entire body — including the parse-retry inner `prompt()` calls — runs
|
||||||
|
* inside `generationGuard.run`, so a stalled stream or hung tool call on
|
||||||
|
* either the first or retry attempt is bounded by `GENERATION_TIMEOUT_MS`.
|
||||||
*/
|
*/
|
||||||
async function continueAgentConversation(session: MissionInterviewSession, message: string): Promise<void> {
|
async function continueAgentConversation(session: MissionInterviewSession, message: string): Promise<void> {
|
||||||
if (!session.agent) {
|
if (!session.agent) {
|
||||||
@@ -878,128 +927,138 @@ async function continueAgentConversation(session: MissionInterviewSession, messa
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
session.thinkingOutput = "";
|
await generationGuard.run(
|
||||||
|
session.id,
|
||||||
|
GENERATION_TIMEOUT_MS,
|
||||||
|
{
|
||||||
|
onTimeout: () => setMissionSessionError(
|
||||||
|
session,
|
||||||
|
"AI generation timed out. You can retry or start a new session.",
|
||||||
|
),
|
||||||
|
onUserStop: () => setMissionSessionError(
|
||||||
|
session,
|
||||||
|
"Generation stopped by user. You can retry or start a new session.",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
const agent = session.agent!;
|
||||||
|
session.thinkingOutput = "";
|
||||||
|
|
||||||
await session.agent.session.prompt(message);
|
await agent.session.prompt(message);
|
||||||
|
|
||||||
// Get the response text from the agent's state
|
// Get the response text from the agent's state
|
||||||
interface AgentMessage {
|
interface AgentMessage {
|
||||||
role: string;
|
role: string;
|
||||||
content?: string | Array<{ type: string; text: string }>;
|
content?: string | Array<{ type: string; text: string }>;
|
||||||
}
|
}
|
||||||
const lastMessage = (session.agent.session.state.messages as AgentMessage[])
|
const lastMessage = (agent.session.state.messages as AgentMessage[])
|
||||||
.filter((m: AgentMessage) => m.role === "assistant")
|
.filter((m: AgentMessage) => m.role === "assistant")
|
||||||
.pop();
|
.pop();
|
||||||
|
|
||||||
let responseText = session.thinkingOutput;
|
let responseText = session.thinkingOutput;
|
||||||
if (lastMessage?.content) {
|
if (lastMessage?.content) {
|
||||||
if (typeof lastMessage.content === "string") {
|
if (typeof lastMessage.content === "string") {
|
||||||
responseText = lastMessage.content;
|
responseText = lastMessage.content;
|
||||||
} else if (Array.isArray(lastMessage.content)) {
|
} else if (Array.isArray(lastMessage.content)) {
|
||||||
responseText = lastMessage.content
|
responseText = lastMessage.content
|
||||||
.filter((c: { type: string; text: string }): c is { type: "text"; text: string } => c.type === "text")
|
.filter((c: { type: string; text: string }): c is { type: "text"; text: string } => c.type === "text")
|
||||||
.map((c: { type: string; text: string }) => c.text)
|
.map((c: { type: string; text: string }) => c.text)
|
||||||
.join("");
|
.join("");
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse with retry
|
|
||||||
let parsed: MissionInterviewResponse | undefined;
|
|
||||||
let lastError: Error | undefined;
|
|
||||||
|
|
||||||
for (let attempt = 0; attempt <= MAX_PARSE_RETRIES; attempt++) {
|
|
||||||
try {
|
|
||||||
parsed = parseMissionAgentResponse(responseText);
|
|
||||||
break;
|
|
||||||
} catch (err) {
|
|
||||||
lastError = err instanceof Error ? err : new Error(String(err));
|
|
||||||
|
|
||||||
if (attempt < MAX_PARSE_RETRIES) {
|
|
||||||
diagnostics.warn(
|
|
||||||
"Parse attempt failed, requesting reformat",
|
|
||||||
{ sessionId: session.id, attempt: attempt + 1, operation: "parse-retry" }
|
|
||||||
);
|
|
||||||
try {
|
|
||||||
session.thinkingOutput = "";
|
|
||||||
await session.agent.session.prompt(
|
|
||||||
"Your previous response could not be parsed as JSON. " +
|
|
||||||
'Please respond with ONLY a valid JSON object: either {"type":"question","data":{...}} ' +
|
|
||||||
'or {"type":"complete","data":{"missionTitle":"...","missionDescription":"...","milestones":[...]}}. ' +
|
|
||||||
"No markdown, no explanation, just the JSON."
|
|
||||||
);
|
|
||||||
|
|
||||||
const retryMessage = (session.agent.session.state.messages as AgentMessage[])
|
|
||||||
.filter((m: AgentMessage) => m.role === "assistant")
|
|
||||||
.pop();
|
|
||||||
|
|
||||||
let retryText = session.thinkingOutput;
|
|
||||||
if (retryMessage?.content) {
|
|
||||||
if (typeof retryMessage.content === "string") {
|
|
||||||
retryText = retryMessage.content;
|
|
||||||
} else if (Array.isArray(retryMessage.content)) {
|
|
||||||
retryText = retryMessage.content
|
|
||||||
.filter((c: { type: string; text: string }): c is { type: "text"; text: string } => c.type === "text")
|
|
||||||
.map((c: { type: string; text: string }) => c.text)
|
|
||||||
.join("");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
responseText = retryText;
|
|
||||||
} catch (retryErr) {
|
|
||||||
diagnostics.errorFromException("Retry prompt failed for session", retryErr, { sessionId: session.id, operation: "retry-prompt" });
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!parsed) {
|
// Parse with retry
|
||||||
const errorMsg = `${lastError?.message || "Failed to parse AI response"} You can try responding again or start a new session.`;
|
let parsed: MissionInterviewResponse | undefined;
|
||||||
diagnostics.error(
|
let lastError: Error | undefined;
|
||||||
"All parse attempts exhausted for session",
|
|
||||||
{ sessionId: session.id, message: errorMsg, operation: "parse-exhausted" }
|
for (let attempt = 0; attempt <= MAX_PARSE_RETRIES; attempt++) {
|
||||||
);
|
try {
|
||||||
session.error = errorMsg;
|
parsed = parseMissionAgentResponse(responseText);
|
||||||
session.updatedAt = new Date();
|
break;
|
||||||
persistMissionSession(session, "error", errorMsg);
|
} catch (err) {
|
||||||
missionInterviewStreamManager.broadcast(session.id, {
|
lastError = err instanceof Error ? err : new Error(String(err));
|
||||||
type: "error",
|
|
||||||
data: errorMsg,
|
if (attempt < MAX_PARSE_RETRIES) {
|
||||||
});
|
diagnostics.warn(
|
||||||
|
"Parse attempt failed, requesting reformat",
|
||||||
|
{ sessionId: session.id, attempt: attempt + 1, operation: "parse-retry" }
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
session.thinkingOutput = "";
|
||||||
|
await agent.session.prompt(
|
||||||
|
"Your previous response could not be parsed as JSON. " +
|
||||||
|
'Please respond with ONLY a valid JSON object: either {"type":"question","data":{...}} ' +
|
||||||
|
'or {"type":"complete","data":{"missionTitle":"...","missionDescription":"...","milestones":[...]}}. ' +
|
||||||
|
"No markdown, no explanation, just the JSON."
|
||||||
|
);
|
||||||
|
|
||||||
|
const retryMessage = (agent.session.state.messages as AgentMessage[])
|
||||||
|
.filter((m: AgentMessage) => m.role === "assistant")
|
||||||
|
.pop();
|
||||||
|
|
||||||
|
let retryText = session.thinkingOutput;
|
||||||
|
if (retryMessage?.content) {
|
||||||
|
if (typeof retryMessage.content === "string") {
|
||||||
|
retryText = retryMessage.content;
|
||||||
|
} else if (Array.isArray(retryMessage.content)) {
|
||||||
|
retryText = retryMessage.content
|
||||||
|
.filter((c: { type: string; text: string }): c is { type: "text"; text: string } => c.type === "text")
|
||||||
|
.map((c: { type: string; text: string }) => c.text)
|
||||||
|
.join("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
responseText = retryText;
|
||||||
|
} catch (retryErr) {
|
||||||
|
diagnostics.errorFromException("Retry prompt failed for session", retryErr, { sessionId: session.id, operation: "retry-prompt" });
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!parsed) {
|
||||||
|
const errorMsg = `${lastError?.message || "Failed to parse AI response"} You can try responding again or start a new session.`;
|
||||||
|
diagnostics.error(
|
||||||
|
"All parse attempts exhausted for session",
|
||||||
|
{ sessionId: session.id, message: errorMsg, operation: "parse-exhausted" }
|
||||||
|
);
|
||||||
|
setMissionSessionError(session, errorMsg);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.type === "question") {
|
||||||
|
session.currentQuestion = parsed.data;
|
||||||
|
session.error = undefined;
|
||||||
|
session.lastGeneratedThinking = session.thinkingOutput;
|
||||||
|
session.updatedAt = new Date();
|
||||||
|
persistMissionSession(session, "awaiting_input");
|
||||||
|
missionInterviewStreamManager.broadcast(session.id, {
|
||||||
|
type: "question",
|
||||||
|
data: parsed.data,
|
||||||
|
});
|
||||||
|
} else if (parsed.type === "complete") {
|
||||||
|
session.summary = parsed.data;
|
||||||
|
session.currentQuestion = undefined;
|
||||||
|
session.error = undefined;
|
||||||
|
session.updatedAt = new Date();
|
||||||
|
persistMissionSession(session, "complete");
|
||||||
|
missionInterviewStreamManager.broadcast(session.id, {
|
||||||
|
type: "summary",
|
||||||
|
data: parsed.data,
|
||||||
|
});
|
||||||
|
missionInterviewStreamManager.broadcast(session.id, { type: "complete" });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
// Timeout / user-stop already published an error state via the guard
|
||||||
|
// handlers. Don't double-broadcast a generic AbortError.
|
||||||
|
if (isAbortError(err)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (parsed.type === "question") {
|
|
||||||
session.currentQuestion = parsed.data;
|
|
||||||
session.error = undefined;
|
|
||||||
session.lastGeneratedThinking = session.thinkingOutput;
|
|
||||||
session.updatedAt = new Date();
|
|
||||||
persistMissionSession(session, "awaiting_input");
|
|
||||||
missionInterviewStreamManager.broadcast(session.id, {
|
|
||||||
type: "question",
|
|
||||||
data: parsed.data,
|
|
||||||
});
|
|
||||||
} else if (parsed.type === "complete") {
|
|
||||||
session.summary = parsed.data;
|
|
||||||
session.currentQuestion = undefined;
|
|
||||||
session.error = undefined;
|
|
||||||
session.updatedAt = new Date();
|
|
||||||
persistMissionSession(session, "complete");
|
|
||||||
missionInterviewStreamManager.broadcast(session.id, {
|
|
||||||
type: "summary",
|
|
||||||
data: parsed.data,
|
|
||||||
});
|
|
||||||
missionInterviewStreamManager.broadcast(session.id, { type: "complete" });
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
const errorMessage = err instanceof Error ? err.message : "AI processing failed";
|
const errorMessage = err instanceof Error ? err.message : "AI processing failed";
|
||||||
diagnostics.errorFromException("Agent conversation error for session", err, { sessionId: session.id, operation: "conversation" });
|
diagnostics.errorFromException("Agent conversation error for session", err, { sessionId: session.id, operation: "conversation" });
|
||||||
session.error = errorMessage;
|
setMissionSessionError(session, errorMessage);
|
||||||
session.updatedAt = new Date();
|
|
||||||
persistMissionSession(session, "error", errorMessage);
|
|
||||||
missionInterviewStreamManager.broadcast(session.id, {
|
|
||||||
type: "error",
|
|
||||||
data: errorMessage,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1211,6 +1270,7 @@ export function __resetMissionInterviewState(): void {
|
|||||||
sessions.clear();
|
sessions.clear();
|
||||||
rateLimits.clear();
|
rateLimits.clear();
|
||||||
missionInterviewStreamManager.reset();
|
missionInterviewStreamManager.reset();
|
||||||
|
generationGuard.reset();
|
||||||
|
|
||||||
if (_aiSessionStore && _aiSessionDeletedListener) {
|
if (_aiSessionStore && _aiSessionDeletedListener) {
|
||||||
_aiSessionStore.off("ai_session:deleted", _aiSessionDeletedListener);
|
_aiSessionStore.off("ai_session:deleted", _aiSessionDeletedListener);
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
createSessionDiagnostics,
|
createSessionDiagnostics,
|
||||||
resetDiagnosticsSink,
|
resetDiagnosticsSink,
|
||||||
} from "./ai-session-diagnostics.js";
|
} from "./ai-session-diagnostics.js";
|
||||||
|
import { GenerationGuard, isAbortError } from "./ai-session-timeout.js";
|
||||||
|
|
||||||
import { createFnAgent as engineCreateFnAgent } from "@fusion/engine";
|
import { createFnAgent as engineCreateFnAgent } from "@fusion/engine";
|
||||||
|
|
||||||
@@ -77,6 +78,14 @@ export type SubtaskStreamCallback = (event: SubtaskStreamEvent, eventId?: number
|
|||||||
|
|
||||||
const SESSION_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
const SESSION_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
||||||
const CLEANUP_INTERVAL_MS = 5 * 60 * 1000;
|
const CLEANUP_INTERVAL_MS = 5 * 60 * 1000;
|
||||||
|
/**
|
||||||
|
* Subtask breakdown is a single-turn flow with no follow-up questions, so
|
||||||
|
* a stuck `prompt()` (silent stream stall, hung tool call) blocks the whole
|
||||||
|
* UI. 90 s is generous for a small JSON response and bounds the worst case.
|
||||||
|
*/
|
||||||
|
export const GENERATION_TIMEOUT_MS = 90_000;
|
||||||
|
|
||||||
|
const generationGuard = new GenerationGuard();
|
||||||
|
|
||||||
/** Minimal interface for the agent object created by createFnAgent */
|
/** Minimal interface for the agent object created by createFnAgent */
|
||||||
interface SubtaskAgent {
|
interface SubtaskAgent {
|
||||||
@@ -208,6 +217,10 @@ function cleanupInMemorySubtaskSession(sessionId: string): boolean {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Abort any in-flight generation so the agent.session.prompt() rejects
|
||||||
|
// promptly and we don't leak the timer / AbortController.
|
||||||
|
generationGuard.stop(sessionId);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
session.agent?.session?.dispose?.();
|
session.agent?.session?.dispose?.();
|
||||||
} catch {
|
} catch {
|
||||||
@@ -407,6 +420,11 @@ async function startSubtaskGeneration(
|
|||||||
try {
|
try {
|
||||||
await generateSubtasks(sessionId, cwd, promptOverrides);
|
await generateSubtasks(sessionId, cwd, promptOverrides);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
// Timeout / user-stop already published an error state via the guard
|
||||||
|
// handlers. Don't overwrite it with a generic AbortError message.
|
||||||
|
if (isAbortError(err)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const existing = sessions.get(sessionId);
|
const existing = sessions.get(sessionId);
|
||||||
if (!existing) return;
|
if (!existing) return;
|
||||||
existing.status = "error";
|
existing.status = "error";
|
||||||
@@ -451,22 +469,39 @@ async function generateSubtasks(
|
|||||||
});
|
});
|
||||||
|
|
||||||
session.agent = agent;
|
session.agent = agent;
|
||||||
await agent.session.prompt(session.initialDescription);
|
|
||||||
|
|
||||||
const messages = agent.session.state.messages as Array<{ role: string; content?: string | Array<{ type: string; text: string }> }>;
|
await generationGuard.run(
|
||||||
const lastAssistant = messages.filter((m) => m.role === "assistant").pop();
|
sessionId,
|
||||||
let responseText = session.thinkingOutput;
|
GENERATION_TIMEOUT_MS,
|
||||||
if (typeof lastAssistant?.content === "string") {
|
{
|
||||||
responseText = lastAssistant.content;
|
onTimeout: () => setSubtaskError(
|
||||||
} else if (Array.isArray(lastAssistant?.content)) {
|
sessionId,
|
||||||
responseText = lastAssistant.content
|
"AI generation timed out. You can retry or start a new session.",
|
||||||
.filter((item): item is { type: "text"; text: string } => item.type === "text")
|
),
|
||||||
.map((item) => item.text)
|
onUserStop: () => setSubtaskError(
|
||||||
.join("");
|
sessionId,
|
||||||
}
|
"Generation stopped by user. You can retry or start a new session.",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
await agent.session.prompt(session.initialDescription);
|
||||||
|
|
||||||
const subtasks = parseSubtasks(responseText);
|
const messages = agent.session.state.messages as Array<{ role: string; content?: string | Array<{ type: string; text: string }> }>;
|
||||||
completeSession(sessionId, subtasks);
|
const lastAssistant = messages.filter((m) => m.role === "assistant").pop();
|
||||||
|
let responseText = session.thinkingOutput;
|
||||||
|
if (typeof lastAssistant?.content === "string") {
|
||||||
|
responseText = lastAssistant.content;
|
||||||
|
} else if (Array.isArray(lastAssistant?.content)) {
|
||||||
|
responseText = lastAssistant.content
|
||||||
|
.filter((item): item is { type: "text"; text: string } => item.type === "text")
|
||||||
|
.map((item) => item.text)
|
||||||
|
.join("");
|
||||||
|
}
|
||||||
|
|
||||||
|
const subtasks = parseSubtasks(responseText);
|
||||||
|
completeSession(sessionId, subtasks);
|
||||||
|
},
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -520,6 +555,24 @@ function generateFallbackSubtasks(initialDescription: string): SubtaskItem[] {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function setSubtaskError(sessionId: string, message: string): void {
|
||||||
|
const session = sessions.get(sessionId);
|
||||||
|
if (!session) return;
|
||||||
|
session.status = "error";
|
||||||
|
session.error = message;
|
||||||
|
session.updatedAt = new Date();
|
||||||
|
persistSubtaskSession(session, "error", message);
|
||||||
|
subtaskStreamManager.broadcast(sessionId, { type: "error", data: message });
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manually abort an in-flight subtask generation (UI "stop" button).
|
||||||
|
* Returns true if a generation was active and got aborted.
|
||||||
|
*/
|
||||||
|
export function stopSubtaskGeneration(sessionId: string): boolean {
|
||||||
|
return generationGuard.stop(sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
function completeSession(sessionId: string, subtasks: SubtaskItem[]): void {
|
function completeSession(sessionId: string, subtasks: SubtaskItem[]): void {
|
||||||
const session = sessions.get(sessionId);
|
const session = sessions.get(sessionId);
|
||||||
if (!session) return;
|
if (!session) return;
|
||||||
@@ -622,6 +675,7 @@ export function __resetSubtaskBreakdownState(): void {
|
|||||||
}
|
}
|
||||||
sessions.clear();
|
sessions.clear();
|
||||||
subtaskStreamManager.reset();
|
subtaskStreamManager.reset();
|
||||||
|
generationGuard.reset();
|
||||||
|
|
||||||
if (_aiSessionStore && _aiSessionDeletedListener) {
|
if (_aiSessionStore && _aiSessionDeletedListener) {
|
||||||
_aiSessionStore.off("ai_session:deleted", _aiSessionDeletedListener);
|
_aiSessionStore.off("ai_session:deleted", _aiSessionDeletedListener);
|
||||||
|
|||||||
Reference in New Issue
Block a user