feat(engine): cli-agent resume coordinator and self-healing integration (U8)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
24
.changeset/cli-agent-resume-coordinator.md
Normal file
24
.changeset/cli-agent-resume-coordinator.md
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": minor
|
||||||
|
---
|
||||||
|
|
||||||
|
Add the CLI agent resume coordinator and self-healing integration (U8). On
|
||||||
|
engine start, sessions persisted as live (starting / ready / busy /
|
||||||
|
waitingOnInput) are classified `engineDeath` and queued for resume respecting
|
||||||
|
the session-manager concurrency ceiling. Resume verifies the recorded worktree
|
||||||
|
still exists (missing → needsAttention, never a CLI spawned into a vanished
|
||||||
|
directory), detects a dirty worktree (logged + flagged on the session record,
|
||||||
|
resume proceeds), relaunches via the adapter's `buildResume` with the recorded
|
||||||
|
native session id in the recorded worktree, re-attaches telemetry, and
|
||||||
|
re-injects no prompt. Only `crashed`/`engineDeath` are resume-eligible
|
||||||
|
(`killed`/`userExited`/`authFailed`/`completed` never); attempts are capped at 2
|
||||||
|
with backoff; exhaustion, an unsupported adapter, a missing vendor session
|
||||||
|
store, or an immediate spawn error route to needsAttention (a permanent-failure
|
||||||
|
path, not a retry loop).
|
||||||
|
|
||||||
|
Self-healing idle-worktree sweeps (`enforceWorktreeCap`, `cleanupOrphans`,
|
||||||
|
unregistered-orphan reap) now skip a worktree backing a resume-eligible
|
||||||
|
`cli_sessions` record via a narrow `isWorktreeResumeReserved` seam, and the
|
||||||
|
stuck-task detector suppresses stuck/inactivity flagging while a task's CLI
|
||||||
|
session is `waitingOnInput` via a narrow `isCliSessionWaitingOnInput` seam — the
|
||||||
|
U3 stall backstop remains the only escalation while genuinely waiting.
|
||||||
161
packages/engine/src/__tests__/self-healing-cli-sessions.test.ts
Normal file
161
packages/engine/src/__tests__/self-healing-cli-sessions.test.ts
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
/**
|
||||||
|
* U8 — self-healing + stuck-detector CLI-session awareness.
|
||||||
|
*
|
||||||
|
* Idle-worktree sweeps (enforceWorktreeCap, cleanupOrphans, reapUnregisteredOrphans)
|
||||||
|
* must SKIP a worktree backing a resume-eligible cli_sessions record; the stuck
|
||||||
|
* detector must suppress stuck/inactivity flagging while a task's CLI session is
|
||||||
|
* waitingOnInput, yet still flag a genuinely-quiet session.
|
||||||
|
*
|
||||||
|
* The sweeps + module functions are exercised through the narrow seams
|
||||||
|
* (isWorktreeResumeReserved option / isCliSessionWaitingOnInput option) with the
|
||||||
|
* heavy git/FS dependencies mocked.
|
||||||
|
*/
|
||||||
|
import { EventEmitter } from "node:events";
|
||||||
|
import { mkdtempSync, mkdirSync, rmSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { beforeEach, afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type { TaskStore } from "@fusion/core";
|
||||||
|
|
||||||
|
import { SelfHealingManager } from "../self-healing.js";
|
||||||
|
import * as worktreePool from "../worktree-pool.js";
|
||||||
|
import { StuckTaskDetector, type DisposableSession } from "../stuck-task-detector.js";
|
||||||
|
|
||||||
|
function createStore(settings: Record<string, unknown>): TaskStore & EventEmitter {
|
||||||
|
const emitter = new EventEmitter() as TaskStore & EventEmitter;
|
||||||
|
(emitter as any).getSettings = vi.fn().mockResolvedValue(settings);
|
||||||
|
(emitter as any).listTasks = vi.fn().mockResolvedValue([]);
|
||||||
|
return emitter;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("self-healing idle-worktree sweeps skip resume-eligible CLI session worktrees (U8)", () => {
|
||||||
|
let rootDir: string;
|
||||||
|
let worktreesDir: string;
|
||||||
|
let reservedPath: string;
|
||||||
|
let freePath: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
rootDir = mkdtempSync(join(tmpdir(), "kb-selfheal-cli-"));
|
||||||
|
worktreesDir = join(rootDir, ".worktrees");
|
||||||
|
mkdirSync(worktreesDir, { recursive: true });
|
||||||
|
reservedPath = join(worktreesDir, "wt-reserved");
|
||||||
|
freePath = join(worktreesDir, "wt-free");
|
||||||
|
mkdirSync(reservedPath);
|
||||||
|
mkdirSync(freePath);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
rmSync(rootDir, { recursive: true, force: true });
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("enforceWorktreeCap skips the reserved worktree, reaps the free one", async () => {
|
||||||
|
// cap = maxWorktrees(1) * 2 = 2; we have 2 dirs → need 3 to exceed. Add one more.
|
||||||
|
mkdirSync(join(worktreesDir, "wt-extra"));
|
||||||
|
const store = createStore({ maxWorktrees: 1, recycleWorktrees: false });
|
||||||
|
vi.spyOn(worktreePool, "scanIdleWorktrees").mockResolvedValue([reservedPath, freePath, join(worktreesDir, "wt-extra")]);
|
||||||
|
const removeSpy = vi.spyOn(worktreePool, "removeWorktree").mockResolvedValue(undefined as never);
|
||||||
|
|
||||||
|
const manager = new SelfHealingManager(store, {
|
||||||
|
rootDir,
|
||||||
|
isWorktreeResumeReserved: (p) => p === reservedPath,
|
||||||
|
});
|
||||||
|
|
||||||
|
await (manager as any).enforceWorktreeCap();
|
||||||
|
|
||||||
|
const removed = removeSpy.mock.calls.map((c) => (c[0] as { worktreePath: string }).worktreePath);
|
||||||
|
expect(removed).not.toContain(reservedPath);
|
||||||
|
expect(removed).toContain(freePath);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cleanupOrphans (recycle off) skips the reserved worktree", async () => {
|
||||||
|
const store = createStore({ recycleWorktrees: false });
|
||||||
|
vi.spyOn(worktreePool, "scanIdleWorktrees").mockResolvedValue([reservedPath, freePath]);
|
||||||
|
const removeSpy = vi.spyOn(worktreePool, "removeWorktree").mockResolvedValue(undefined as never);
|
||||||
|
|
||||||
|
const manager = new SelfHealingManager(store, {
|
||||||
|
rootDir,
|
||||||
|
isWorktreeResumeReserved: (p) => p === reservedPath,
|
||||||
|
});
|
||||||
|
|
||||||
|
const cleaned = await (manager as any).cleanupOrphans();
|
||||||
|
|
||||||
|
const removed = removeSpy.mock.calls.map((c) => (c[0] as { worktreePath: string }).worktreePath);
|
||||||
|
expect(removed).toEqual([freePath]);
|
||||||
|
expect(cleaned).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("without the seam predicate, both worktrees are reaped (no behavior change)", async () => {
|
||||||
|
const store = createStore({ recycleWorktrees: false });
|
||||||
|
vi.spyOn(worktreePool, "scanIdleWorktrees").mockResolvedValue([reservedPath, freePath]);
|
||||||
|
const removeSpy = vi.spyOn(worktreePool, "removeWorktree").mockResolvedValue(undefined as never);
|
||||||
|
|
||||||
|
const manager = new SelfHealingManager(store, { rootDir });
|
||||||
|
await (manager as any).cleanupOrphans();
|
||||||
|
|
||||||
|
const removed = removeSpy.mock.calls.map((c) => (c[0] as { worktreePath: string }).worktreePath);
|
||||||
|
expect(removed).toEqual([reservedPath, freePath]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Stuck detector waitingOnInput suppression ────────────────────────────────
|
||||||
|
|
||||||
|
function fakeStore(settings: Record<string, unknown>): TaskStore {
|
||||||
|
return {
|
||||||
|
getSettings: vi.fn().mockResolvedValue(settings),
|
||||||
|
getTask: vi.fn(),
|
||||||
|
} as unknown as TaskStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
function fakeSession(): DisposableSession {
|
||||||
|
return { dispose: vi.fn() } as unknown as DisposableSession;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("stuck-task detector suppresses flagging while CLI session waitingOnInput (U8)", () => {
|
||||||
|
afterEach(() => vi.restoreAllMocks());
|
||||||
|
|
||||||
|
it("waitingOnInput session is NOT flagged; same session IS flagged once it stops waiting", async () => {
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
let waiting = true;
|
||||||
|
const store = fakeStore({ taskStuckTimeoutMs: 1000, globalPause: false, enginePaused: false });
|
||||||
|
const detector = new StuckTaskDetector(store, {
|
||||||
|
onStuck,
|
||||||
|
isCliSessionWaitingOnInput: () => waiting,
|
||||||
|
// Accept the requeue so killAndRetry proceeds to onStuck.
|
||||||
|
beforeRequeue: async () => true,
|
||||||
|
});
|
||||||
|
|
||||||
|
detector.trackTask("FN-1", fakeSession());
|
||||||
|
// Force the task to look inactive (past the 1s timeout).
|
||||||
|
(detector as any).tracked.get("FN-1").lastActivity = Date.now() - 10_000;
|
||||||
|
|
||||||
|
// While waitingOnInput: suppressed.
|
||||||
|
await (detector as any).checkStuckTasks();
|
||||||
|
expect(onStuck).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
// Once it stops waiting (genuinely quiet): the U3-style backstop equivalent
|
||||||
|
// (the detector) now flags it.
|
||||||
|
waiting = false;
|
||||||
|
// killAndRetry needs moveTask/logEntry; stub them on the store.
|
||||||
|
(store as any).moveTask = vi.fn().mockResolvedValue(undefined);
|
||||||
|
(store as any).logEntry = vi.fn().mockResolvedValue(undefined);
|
||||||
|
(store as any).getTask = vi.fn().mockResolvedValue({ id: "FN-1", status: "in-progress", steps: [], error: null });
|
||||||
|
await (detector as any).checkStuckTasks();
|
||||||
|
expect(onStuck).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onStuck.mock.calls[0][0].taskId).toBe("FN-1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("without the seam lookup, a waitingOnInput-shaped quiet task is flagged normally", async () => {
|
||||||
|
const onStuck = vi.fn();
|
||||||
|
const store = fakeStore({ taskStuckTimeoutMs: 1000, globalPause: false, enginePaused: false });
|
||||||
|
(store as any).moveTask = vi.fn().mockResolvedValue(undefined);
|
||||||
|
(store as any).logEntry = vi.fn().mockResolvedValue(undefined);
|
||||||
|
(store as any).getTask = vi.fn().mockResolvedValue({ id: "FN-2", status: "in-progress", steps: [], error: null });
|
||||||
|
const detector = new StuckTaskDetector(store, { onStuck, beforeRequeue: async () => true });
|
||||||
|
|
||||||
|
detector.trackTask("FN-2", fakeSession());
|
||||||
|
(detector as any).tracked.get("FN-2").lastActivity = Date.now() - 10_000;
|
||||||
|
await (detector as any).checkStuckTasks();
|
||||||
|
expect(onStuck).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,345 @@
|
|||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtempSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { rm } from "node:fs/promises";
|
||||||
|
import { Database, CliSessionStore, type CliSession } from "@fusion/core";
|
||||||
|
import type { IPty } from "node-pty";
|
||||||
|
import { CliSessionManager } from "../session-manager.js";
|
||||||
|
import { CliAdapterRegistry, type CliAgentAdapter } from "../adapter.js";
|
||||||
|
import { CliResumeCoordinator } from "../resume-coordinator.js";
|
||||||
|
|
||||||
|
// ── Mock PTY at the loadPtyModule seam ─────────────────────────────────────
|
||||||
|
|
||||||
|
interface MockPty extends IPty {
|
||||||
|
written: string[];
|
||||||
|
killed: boolean;
|
||||||
|
emitData(data: string): void;
|
||||||
|
emitExit(exitCode: number, signal?: number): void;
|
||||||
|
spawnArgs: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface MockState {
|
||||||
|
ptys: MockPty[];
|
||||||
|
spawnCount: number;
|
||||||
|
spawnThrows?: () => Error | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeMockPtyModule(state: MockState): typeof import("node-pty") {
|
||||||
|
return {
|
||||||
|
spawn(_file: string, args: string[] | string) {
|
||||||
|
state.spawnCount++;
|
||||||
|
const err = state.spawnThrows?.();
|
||||||
|
if (err) throw err;
|
||||||
|
let dataCb: ((d: string) => void) | undefined;
|
||||||
|
let exitCb: ((e: { exitCode: number; signal?: number }) => void) | undefined;
|
||||||
|
const mock: MockPty = {
|
||||||
|
pid: 1000 + state.ptys.length,
|
||||||
|
cols: 80,
|
||||||
|
rows: 24,
|
||||||
|
process: "mock",
|
||||||
|
handleFlowControl: false,
|
||||||
|
written: [],
|
||||||
|
killed: false,
|
||||||
|
spawnArgs: Array.isArray(args) ? args : [args],
|
||||||
|
onData: (cb: (d: string) => void) => {
|
||||||
|
dataCb = cb;
|
||||||
|
return { dispose() {} };
|
||||||
|
},
|
||||||
|
onExit: (cb: (e: { exitCode: number; signal?: number }) => void) => {
|
||||||
|
exitCb = cb;
|
||||||
|
return { dispose() {} };
|
||||||
|
},
|
||||||
|
on() {},
|
||||||
|
write() {},
|
||||||
|
resize() {},
|
||||||
|
clear() {},
|
||||||
|
kill() {
|
||||||
|
mock.killed = true;
|
||||||
|
},
|
||||||
|
pause() {},
|
||||||
|
resume() {},
|
||||||
|
emitData(d: string) {
|
||||||
|
dataCb?.(d);
|
||||||
|
},
|
||||||
|
emitExit(exitCode: number, signal?: number) {
|
||||||
|
exitCb?.({ exitCode, signal });
|
||||||
|
},
|
||||||
|
} as unknown as MockPty;
|
||||||
|
state.ptys.push(mock);
|
||||||
|
return mock as unknown as IPty;
|
||||||
|
},
|
||||||
|
} as unknown as typeof import("node-pty");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Test adapter ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
function makeAdapter(overrides: Partial<CliAgentAdapter> = {}): CliAgentAdapter {
|
||||||
|
return {
|
||||||
|
id: "test-cli",
|
||||||
|
name: "Test CLI",
|
||||||
|
capabilities: {
|
||||||
|
nativeDone: true,
|
||||||
|
nativeWaiting: true,
|
||||||
|
transcriptSource: "hooks",
|
||||||
|
supportsResume: true,
|
||||||
|
},
|
||||||
|
buildLaunch: () => ({ command: "test-cli", args: ["--interactive"] }),
|
||||||
|
buildEnvAllowlist: () => ["PATH", "HOME"],
|
||||||
|
createReadinessDetector: () => ({ observe: () => true }),
|
||||||
|
formatInjection: (text) => ({ payload: `${text}\r` }),
|
||||||
|
buildResume: (ctx) => ({ command: "test-cli", args: ["--resume", ctx.nativeSessionId] }),
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Harness ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
interface Harness {
|
||||||
|
manager: CliSessionManager;
|
||||||
|
registry: CliAdapterRegistry;
|
||||||
|
store: CliSessionStore;
|
||||||
|
state: MockState;
|
||||||
|
db: Database;
|
||||||
|
tmpDir: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeHarness(opts?: { adapter?: CliAgentAdapter; ceiling?: number }): Harness {
|
||||||
|
const tmpDir = mkdtempSync(join(tmpdir(), "kb-cli-resume-test-"));
|
||||||
|
const fusionDir = join(tmpDir, ".fusion");
|
||||||
|
const db = new Database(fusionDir, { inMemory: true });
|
||||||
|
db.init();
|
||||||
|
const store = new CliSessionStore(fusionDir, db);
|
||||||
|
const registry = new CliAdapterRegistry();
|
||||||
|
registry.register(opts?.adapter ?? makeAdapter());
|
||||||
|
const state: MockState = { ptys: [], spawnCount: 0 };
|
||||||
|
const manager = new CliSessionManager({
|
||||||
|
registry,
|
||||||
|
store,
|
||||||
|
concurrencyCeiling: opts?.ceiling ?? 8,
|
||||||
|
loadPty: async () => makeMockPtyModule(state),
|
||||||
|
});
|
||||||
|
return { manager, registry, store, state, db, tmpDir };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Seed a record as the engine would persist a live session before dying. */
|
||||||
|
function seedSession(
|
||||||
|
store: CliSessionStore,
|
||||||
|
worktreePath: string,
|
||||||
|
over: Partial<CliSession> = {},
|
||||||
|
): CliSession {
|
||||||
|
const rec = store.createSession({
|
||||||
|
adapterId: over.adapterId ?? "test-cli",
|
||||||
|
projectId: "proj-1",
|
||||||
|
purpose: "execute",
|
||||||
|
taskId: over.taskId ?? "FN-1",
|
||||||
|
worktreePath,
|
||||||
|
nativeSessionId: "nativeSessionId" in over ? over.nativeSessionId : "native-abc",
|
||||||
|
agentState: over.agentState ?? "busy",
|
||||||
|
terminationReason: over.terminationReason ?? null,
|
||||||
|
resumeAttempts: over.resumeAttempts ?? 0,
|
||||||
|
autonomyPosture: over.autonomyPosture ?? null,
|
||||||
|
});
|
||||||
|
return rec;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeCoordinator(h: Harness, over?: Partial<ConstructorParameters<typeof CliResumeCoordinator>[0]>) {
|
||||||
|
return new CliResumeCoordinator({
|
||||||
|
store: h.store,
|
||||||
|
manager: h.manager,
|
||||||
|
registry: h.registry,
|
||||||
|
worktreeExists: () => true,
|
||||||
|
isWorktreeDirty: async () => false,
|
||||||
|
...over,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("CliResumeCoordinator (U8)", () => {
|
||||||
|
const harnesses: Harness[] = [];
|
||||||
|
function track(h: Harness): Harness {
|
||||||
|
harnesses.push(h);
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
beforeEach(() => {});
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const h of harnesses) {
|
||||||
|
h.manager.dispose?.();
|
||||||
|
h.db.close?.();
|
||||||
|
await rm(h.tmpDir, { recursive: true, force: true }).catch(() => {});
|
||||||
|
}
|
||||||
|
harnesses.length = 0;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("AE3/F4: resumes a live-on-restart session via buildResume with the recorded native id; record intact; no duplicate on a second run", async () => {
|
||||||
|
const h = track(makeHarness());
|
||||||
|
const rec = seedSession(h.store, h.tmpDir, { agentState: "busy", nativeSessionId: "native-xyz" });
|
||||||
|
const coord = makeCoordinator(h);
|
||||||
|
|
||||||
|
const results = await coord.recoverOnStart();
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].disposition).toBe("resumed");
|
||||||
|
|
||||||
|
// Spawned via buildResume with the recorded native id.
|
||||||
|
expect(h.state.spawnCount).toBe(1);
|
||||||
|
expect(h.state.ptys[0].spawnArgs).toEqual(["--resume", "native-xyz"]);
|
||||||
|
|
||||||
|
// Manager owns the session; record reused (state back to starting), not duplicated.
|
||||||
|
expect(h.manager.isLive(rec.id)).toBe(true);
|
||||||
|
expect(h.store.listSessions()).toHaveLength(1);
|
||||||
|
const after = h.store.getSession(rec.id)!;
|
||||||
|
expect(after.taskId).toBe("FN-1");
|
||||||
|
|
||||||
|
// Second sweep: session is live → no duplicate spawn.
|
||||||
|
const second = await coord.recoverOnStart();
|
||||||
|
expect(second).toHaveLength(0); // already live, filtered out
|
||||||
|
expect(h.state.spawnCount).toBe(1);
|
||||||
|
expect(h.store.listSessions()).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never resumes killed or userExited records across sweeps", async () => {
|
||||||
|
const h = track(makeHarness());
|
||||||
|
// A dead record carrying killed/userExited is not in the orphaned-live set,
|
||||||
|
// so recoverOnStart never touches it; and resumeOne routes it to attention.
|
||||||
|
const killed = seedSession(h.store, h.tmpDir, { agentState: "dead", terminationReason: "killed", taskId: "FN-K" });
|
||||||
|
const userExited = seedSession(h.store, h.tmpDir, { agentState: "dead", terminationReason: "userExited", taskId: "FN-U" });
|
||||||
|
const coord = makeCoordinator(h);
|
||||||
|
|
||||||
|
const results = await coord.recoverOnStart();
|
||||||
|
expect(results).toHaveLength(0); // neither is orphaned-live
|
||||||
|
expect(h.state.spawnCount).toBe(0);
|
||||||
|
|
||||||
|
// Direct disposition is ineligible (no spawn).
|
||||||
|
expect((await coord.resumeOne(killed)).disposition).toBe("needsAttention-ineligible");
|
||||||
|
expect((await coord.resumeOne(userExited)).disposition).toBe("needsAttention-ineligible");
|
||||||
|
expect(h.state.spawnCount).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("authFailed → needsAttention without a resume attempt", async () => {
|
||||||
|
const h = track(makeHarness());
|
||||||
|
const rec = seedSession(h.store, h.tmpDir, { agentState: "dead", terminationReason: "authFailed" });
|
||||||
|
const coord = makeCoordinator(h);
|
||||||
|
const res = await coord.resumeOne(rec);
|
||||||
|
expect(res.disposition).toBe("needsAttention-ineligible");
|
||||||
|
expect(h.state.spawnCount).toBe(0);
|
||||||
|
expect(h.store.getSession(rec.id)!.agentState).toBe("needsAttention");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("cap: two failures → needsAttention, no third spawn across cycles", async () => {
|
||||||
|
const h = track(makeHarness());
|
||||||
|
// Spawn always throws (vendor store / spawn error).
|
||||||
|
h.state.spawnThrows = () => new Error("spawn failed");
|
||||||
|
const rec = seedSession(h.store, h.tmpDir, { agentState: "busy" });
|
||||||
|
const coord = makeCoordinator(h, { maxResumeAttempts: 2 });
|
||||||
|
|
||||||
|
// First sweep: spawn throws → immediate permanent-failure path → needsAttention.
|
||||||
|
const r1 = await coord.recoverOnStart();
|
||||||
|
expect(r1[0].disposition).toBe("needsAttention-spawnError");
|
||||||
|
expect(h.store.getSession(rec.id)!.agentState).toBe("needsAttention");
|
||||||
|
const spawnsAfter1 = h.state.spawnCount;
|
||||||
|
|
||||||
|
// Subsequent sweeps: record is no longer orphaned-live → never spawned again.
|
||||||
|
await coord.recoverOnStart();
|
||||||
|
await coord.recoverOnStart();
|
||||||
|
expect(h.state.spawnCount).toBe(spawnsAfter1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("missing vendor store (no native id) → permanent-failure path, not retry loop", async () => {
|
||||||
|
const h = track(makeHarness());
|
||||||
|
const rec = seedSession(h.store, h.tmpDir, { agentState: "busy", nativeSessionId: null });
|
||||||
|
const coord = makeCoordinator(h);
|
||||||
|
const res = await coord.resumeOne(rec);
|
||||||
|
expect(res.disposition).toBe("needsAttention-spawnError");
|
||||||
|
expect(h.state.spawnCount).toBe(0);
|
||||||
|
expect(h.store.getSession(rec.id)!.agentState).toBe("needsAttention");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("missing worktree → needsAttention without spawning", async () => {
|
||||||
|
const h = track(makeHarness());
|
||||||
|
const rec = seedSession(h.store, h.tmpDir, { agentState: "busy" });
|
||||||
|
const coord = makeCoordinator(h, { worktreeExists: () => false });
|
||||||
|
const res = await coord.resumeOne(rec);
|
||||||
|
expect(res.disposition).toBe("needsAttention-missingWorktree");
|
||||||
|
expect(h.state.spawnCount).toBe(0);
|
||||||
|
expect(h.store.getSession(rec.id)!.agentState).toBe("needsAttention");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("adapter without resume support → needsAttention with clear reason, no spawn", async () => {
|
||||||
|
const h = track(
|
||||||
|
makeHarness({
|
||||||
|
adapter: makeAdapter({
|
||||||
|
capabilities: {
|
||||||
|
nativeDone: true,
|
||||||
|
nativeWaiting: true,
|
||||||
|
transcriptSource: "hooks",
|
||||||
|
supportsResume: false,
|
||||||
|
},
|
||||||
|
buildResume: undefined,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const rec = seedSession(h.store, h.tmpDir, { agentState: "busy" });
|
||||||
|
const coord = makeCoordinator(h);
|
||||||
|
const res = await coord.resumeOne(rec);
|
||||||
|
expect(res.disposition).toBe("needsAttention-resumeUnsupported");
|
||||||
|
expect(h.state.spawnCount).toBe(0);
|
||||||
|
expect(h.store.getSession(rec.id)!.agentState).toBe("needsAttention");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("dirty worktree → flagged on the record, resume proceeds", async () => {
|
||||||
|
const h = track(makeHarness());
|
||||||
|
const rec = seedSession(h.store, h.tmpDir, { agentState: "busy" });
|
||||||
|
const coord = makeCoordinator(h, { isWorktreeDirty: async () => true });
|
||||||
|
const res = await coord.resumeOne(rec);
|
||||||
|
expect(res.disposition).toBe("resumed");
|
||||||
|
expect(res.dirtyWorktree).toBe(true);
|
||||||
|
expect(h.state.spawnCount).toBe(1);
|
||||||
|
const after = h.store.getSession(rec.id)!;
|
||||||
|
expect(after.autonomyPosture?.resumeDirtyWorktree).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("re-attaches telemetry on resume and injects no prompt", async () => {
|
||||||
|
const h = track(makeHarness());
|
||||||
|
seedSession(h.store, h.tmpDir, { agentState: "busy" });
|
||||||
|
const reattached: string[] = [];
|
||||||
|
const coord = makeCoordinator(h, {
|
||||||
|
reattachTelemetry: (s) => {
|
||||||
|
reattached.push(s.id);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await coord.recoverOnStart();
|
||||||
|
expect(reattached).toHaveLength(1);
|
||||||
|
// No prompt injected: the resume PTY received no writes.
|
||||||
|
expect(h.state.ptys[0].written.join("")).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("respects the concurrency ceiling: queues remaining sessions for the next sweep", async () => {
|
||||||
|
const h = track(makeHarness({ ceiling: 1 }));
|
||||||
|
seedSession(h.store, h.tmpDir, { agentState: "busy", taskId: "FN-A", nativeSessionId: "n-a" });
|
||||||
|
seedSession(h.store, h.tmpDir, { agentState: "busy", taskId: "FN-B", nativeSessionId: "n-b" });
|
||||||
|
const coord = makeCoordinator(h);
|
||||||
|
const results = await coord.recoverOnStart();
|
||||||
|
const resumed = results.filter((r) => r.disposition === "resumed");
|
||||||
|
const skipped = results.filter((r) => r.disposition === "skipped-noCapacity");
|
||||||
|
expect(resumed).toHaveLength(1);
|
||||||
|
expect(skipped).toHaveLength(1);
|
||||||
|
expect(h.state.spawnCount).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resumeReservedWorktrees: reports worktrees backing resume-eligible records", () => {
|
||||||
|
const h = track(makeHarness());
|
||||||
|
seedSession(h.store, h.tmpDir, { agentState: "busy", taskId: "FN-live" });
|
||||||
|
seedSession(h.store, h.tmpDir, { agentState: "dead", terminationReason: "killed", taskId: "FN-killed", worktreePath: h.tmpDir } as Partial<CliSession>);
|
||||||
|
const coord = makeCoordinator(h);
|
||||||
|
const reserved = coord.resumeReservedWorktrees();
|
||||||
|
expect(reserved.has(h.tmpDir)).toBe(true);
|
||||||
|
|
||||||
|
// An exhausted record is NOT reserved.
|
||||||
|
const exhausted = seedSession(h.store, h.tmpDir, {
|
||||||
|
agentState: "dead",
|
||||||
|
terminationReason: "crashed",
|
||||||
|
resumeAttempts: 2,
|
||||||
|
taskId: "FN-exh",
|
||||||
|
});
|
||||||
|
expect(coord.isRecordResumeEligible(exhausted)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
354
packages/engine/src/cli-agent/resume-coordinator.ts
Normal file
354
packages/engine/src/cli-agent/resume-coordinator.ts
Normal file
@@ -0,0 +1,354 @@
|
|||||||
|
/**
|
||||||
|
* CliResumeCoordinator — engine-restart recovery for CLI agent sessions
|
||||||
|
* (CLI Agent Executor, U8).
|
||||||
|
*
|
||||||
|
* On engine start, sessions persisted as live (starting / ready / busy /
|
||||||
|
* waitingOnInput) were orphaned by the engine's death — there is no live PTY
|
||||||
|
* behind them. This coordinator finds those records, classifies them as
|
||||||
|
* `engineDeath`, and queues a resume that respects the session-manager
|
||||||
|
* concurrency ceiling.
|
||||||
|
*
|
||||||
|
* Resume semantics (KTD — termination taxonomy, resume-the-CLI):
|
||||||
|
* - Eligibility: ONLY `crashed` and `engineDeath` are resume-eligible
|
||||||
|
* (`isResumeEligible`). `killed` / `userExited` are never auto-resumed;
|
||||||
|
* `authFailed` and `completed` are never resumed. A record found live on
|
||||||
|
* restart is reclassified to `engineDeath` (it had no chance to record a
|
||||||
|
* terminal reason), making it eligible.
|
||||||
|
* - Worktree-existence precondition: the recorded worktree MUST still exist —
|
||||||
|
* a missing worktree routes the session to `needsAttention`, NEVER a CLI
|
||||||
|
* spawned into a vanished directory.
|
||||||
|
* - Dirty-tree detection: a dirty `git status` is logged and flagged on the
|
||||||
|
* session record (under `autonomyPosture.resumeDirtyWorktree`), then resume
|
||||||
|
* PROCEEDS — the flag surfaces to the UI.
|
||||||
|
* - Relaunch: via the manager's resume path (adapter `buildResume` with the
|
||||||
|
* recorded `nativeSessionId`, in the recorded worktree). Telemetry is
|
||||||
|
* re-attached (a fresh hook token + scripts via `wireTelemetry`/the hub).
|
||||||
|
* NO prompt is re-injected — scrollback replays to viewers, the agent
|
||||||
|
* continues from its own native transcript.
|
||||||
|
* - Attempt cap: 2 attempts with backoff (tracked on `resumeAttempts`).
|
||||||
|
* Exhaustion, an adapter without resume support, a missing vendor session
|
||||||
|
* store, or an immediate spawn error route to `needsAttention` (a permanent
|
||||||
|
* failure path, NOT an infinite retry loop).
|
||||||
|
*
|
||||||
|
* The coordinator NEVER imports dashboard code. The worktree-existence check
|
||||||
|
* and the dirty-tree probe are injected seams so tests need no real git/FS.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import { execFile } from "node:child_process";
|
||||||
|
import { promisify } from "node:util";
|
||||||
|
import type { CliSession, CliSessionStore, CliTerminationReason } from "@fusion/core";
|
||||||
|
import type { CliSessionManager } from "./session-manager.js";
|
||||||
|
import { CliConcurrencyLimitError, CliResumeUnsupportedError } from "./session-manager.js";
|
||||||
|
import type { CliAdapterRegistry } from "./adapter.js";
|
||||||
|
import { isResumeEligible } from "./state-machine.js";
|
||||||
|
|
||||||
|
const execFileAsync = promisify(execFile);
|
||||||
|
|
||||||
|
/** Persisted-as-live states that, found on restart, imply an orphaned PTY. */
|
||||||
|
const ORPHANED_LIVE_STATES = new Set<CliSession["agentState"]>([
|
||||||
|
"starting",
|
||||||
|
"ready",
|
||||||
|
"busy",
|
||||||
|
"waitingOnInput",
|
||||||
|
]);
|
||||||
|
|
||||||
|
/** Default resume attempt cap (KTD = 2). */
|
||||||
|
export const DEFAULT_MAX_RESUME_ATTEMPTS = 2;
|
||||||
|
/** Default base backoff (ms) between resume attempts; doubled per attempt. */
|
||||||
|
export const DEFAULT_RESUME_BACKOFF_BASE_MS = 1000;
|
||||||
|
|
||||||
|
/** Outcome of a single session's resume disposition. */
|
||||||
|
export type ResumeDisposition =
|
||||||
|
| "resumed"
|
||||||
|
| "needsAttention-missingWorktree"
|
||||||
|
| "needsAttention-ineligible"
|
||||||
|
| "needsAttention-exhausted"
|
||||||
|
| "needsAttention-resumeUnsupported"
|
||||||
|
| "needsAttention-spawnError"
|
||||||
|
| "skipped-noCapacity";
|
||||||
|
|
||||||
|
export interface ResumeResult {
|
||||||
|
sessionId: string;
|
||||||
|
taskId: string | null;
|
||||||
|
disposition: ResumeDisposition;
|
||||||
|
/** Whether the worktree was dirty at resume (flag also persisted on the record). */
|
||||||
|
dirtyWorktree?: boolean;
|
||||||
|
/** Reason string for needsAttention dispositions. */
|
||||||
|
reason?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CliResumeCoordinatorOptions {
|
||||||
|
store: CliSessionStore;
|
||||||
|
manager: CliSessionManager;
|
||||||
|
registry: CliAdapterRegistry;
|
||||||
|
/**
|
||||||
|
* Re-attach telemetry for a resumed session — typically wires a fresh hook
|
||||||
|
* token + scripts via the TelemetryHub. Called AFTER a successful relaunch and
|
||||||
|
* BEFORE returning. Best-effort; a throw is logged, never fatal to the sweep.
|
||||||
|
*/
|
||||||
|
reattachTelemetry?: (session: CliSession) => void | Promise<void>;
|
||||||
|
/** Max resume attempts before needsAttention. Default 2 (KTD). */
|
||||||
|
maxResumeAttempts?: number;
|
||||||
|
/** Base backoff (ms); doubled per prior attempt. Default 1000. */
|
||||||
|
resumeBackoffBaseMs?: number;
|
||||||
|
/** Worktree-existence probe (injected for tests). Default `fs.existsSync`. */
|
||||||
|
worktreeExists?: (worktreePath: string) => boolean;
|
||||||
|
/**
|
||||||
|
* Dirty-tree probe (injected for tests). Returns true when `git status` shows
|
||||||
|
* uncommitted changes. Default: runs `git status --porcelain` in the worktree.
|
||||||
|
*/
|
||||||
|
isWorktreeDirty?: (worktreePath: string) => Promise<boolean>;
|
||||||
|
/** Best-effort logger. */
|
||||||
|
log?: (msg: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Default dirty-tree probe: `git status --porcelain` is non-empty. */
|
||||||
|
async function defaultIsWorktreeDirty(worktreePath: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
const { stdout } = await execFileAsync("git", ["status", "--porcelain"], {
|
||||||
|
cwd: worktreePath,
|
||||||
|
});
|
||||||
|
return stdout.trim().length > 0;
|
||||||
|
} catch {
|
||||||
|
// Not a git worktree / git unavailable: treat as not-dirty (don't block resume).
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class CliResumeCoordinator {
|
||||||
|
private readonly store: CliSessionStore;
|
||||||
|
private readonly manager: CliSessionManager;
|
||||||
|
private readonly registry: CliAdapterRegistry;
|
||||||
|
private readonly reattachTelemetry?: (session: CliSession) => void | Promise<void>;
|
||||||
|
private readonly maxResumeAttempts: number;
|
||||||
|
private readonly resumeBackoffBaseMs: number;
|
||||||
|
private readonly worktreeExists: (worktreePath: string) => boolean;
|
||||||
|
private readonly isWorktreeDirty: (worktreePath: string) => Promise<boolean>;
|
||||||
|
private readonly log: (msg: string) => void;
|
||||||
|
|
||||||
|
constructor(opts: CliResumeCoordinatorOptions) {
|
||||||
|
this.store = opts.store;
|
||||||
|
this.manager = opts.manager;
|
||||||
|
this.registry = opts.registry;
|
||||||
|
this.reattachTelemetry = opts.reattachTelemetry;
|
||||||
|
this.maxResumeAttempts = opts.maxResumeAttempts ?? DEFAULT_MAX_RESUME_ATTEMPTS;
|
||||||
|
this.resumeBackoffBaseMs = opts.resumeBackoffBaseMs ?? DEFAULT_RESUME_BACKOFF_BASE_MS;
|
||||||
|
this.worktreeExists = opts.worktreeExists ?? ((p) => existsSync(p));
|
||||||
|
this.isWorktreeDirty = opts.isWorktreeDirty ?? defaultIsWorktreeDirty;
|
||||||
|
this.log = opts.log ?? (() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The set of worktree paths backing resume-eligible session records. Exposed
|
||||||
|
* for the self-healing seam so idle-worktree sweeps treat them as in-use. A
|
||||||
|
* record is resume-eligible if it is found live-on-restart (→ engineDeath) or
|
||||||
|
* already carries a resume-eligible termination reason AND has not exhausted
|
||||||
|
* its attempt cap. The path is `resolve`d-free (raw recorded path); callers
|
||||||
|
* normalize as needed.
|
||||||
|
*/
|
||||||
|
resumeReservedWorktrees(): Set<string> {
|
||||||
|
const reserved = new Set<string>();
|
||||||
|
for (const session of this.store.listSessions()) {
|
||||||
|
if (!session.worktreePath) continue;
|
||||||
|
if (!this.isRecordResumeEligible(session)) continue;
|
||||||
|
reserved.add(session.worktreePath);
|
||||||
|
}
|
||||||
|
return reserved;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether a recorded session is currently resume-eligible (for sweep skipping). */
|
||||||
|
isRecordResumeEligible(session: CliSession): boolean {
|
||||||
|
if (session.resumeAttempts >= this.maxResumeAttempts) return false;
|
||||||
|
// Found-live-on-restart → engineDeath (eligible).
|
||||||
|
if (ORPHANED_LIVE_STATES.has(session.agentState)) return true;
|
||||||
|
// Reaped-but-resumable: a dead record whose recorded reason is resume-eligible.
|
||||||
|
if (
|
||||||
|
session.agentState === "dead" &&
|
||||||
|
session.terminationReason != null &&
|
||||||
|
isResumeEligible(session.terminationReason)
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Engine-start sweep. Finds orphaned-live sessions, classifies engineDeath,
|
||||||
|
* and resumes each (respecting the manager's concurrency ceiling). Returns a
|
||||||
|
* per-session disposition list. Idempotent: a second run after a successful
|
||||||
|
* resume finds the session live (re-spawned record is `starting`/`ready`) but
|
||||||
|
* the manager's `isLive` guard prevents a duplicate spawn — see `resumeOne`.
|
||||||
|
*/
|
||||||
|
async recoverOnStart(): Promise<ResumeResult[]> {
|
||||||
|
const candidates = this.store
|
||||||
|
.listSessions()
|
||||||
|
.filter((s) => ORPHANED_LIVE_STATES.has(s.agentState))
|
||||||
|
// Never reclaim a session the manager already owns (idempotent re-run).
|
||||||
|
.filter((s) => !this.manager.isLive(s.id));
|
||||||
|
|
||||||
|
const results: ResumeResult[] = [];
|
||||||
|
for (const session of candidates) {
|
||||||
|
// Concurrency ceiling: stop queuing once slots are exhausted. The
|
||||||
|
// remaining records stay persisted-live and are picked up next sweep.
|
||||||
|
if (this.manager.availableSlots() <= 0) {
|
||||||
|
results.push({
|
||||||
|
sessionId: session.id,
|
||||||
|
taskId: session.taskId,
|
||||||
|
disposition: "skipped-noCapacity",
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
results.push(await this.resumeOne(session));
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resume a single orphaned session through the eligibility predicate and
|
||||||
|
* worktree precondition. Public for targeted tests.
|
||||||
|
*/
|
||||||
|
async resumeOne(session: CliSession): Promise<ResumeResult> {
|
||||||
|
const base = { sessionId: session.id, taskId: session.taskId };
|
||||||
|
|
||||||
|
// Idempotency: the manager already owns a live PTY for this id → no-op.
|
||||||
|
if (this.manager.isLive(session.id)) {
|
||||||
|
return { ...base, disposition: "resumed" };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reclassify a found-live record to engineDeath (it never recorded a reason).
|
||||||
|
// A dead record keeps its recorded reason (crashed / killed / userExited / …).
|
||||||
|
const reason: CliTerminationReason = ORPHANED_LIVE_STATES.has(session.agentState)
|
||||||
|
? "engineDeath"
|
||||||
|
: session.terminationReason ?? "engineDeath";
|
||||||
|
|
||||||
|
// Eligibility predicate: only crashed / engineDeath ever resume.
|
||||||
|
if (!isResumeEligible(reason)) {
|
||||||
|
this.toNeedsAttention(session, reason, `ineligible termination reason: ${reason}`);
|
||||||
|
return { ...base, disposition: "needsAttention-ineligible", reason };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt-cap exhaustion → permanent needsAttention (never a third spawn).
|
||||||
|
if (session.resumeAttempts >= this.maxResumeAttempts) {
|
||||||
|
this.toNeedsAttention(session, reason, `resume attempts exhausted (${session.resumeAttempts})`);
|
||||||
|
return { ...base, disposition: "needsAttention-exhausted", reason };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Worktree-existence precondition: never spawn into a vanished directory.
|
||||||
|
const worktreePath = session.worktreePath;
|
||||||
|
if (!worktreePath || !this.worktreeExists(worktreePath)) {
|
||||||
|
this.toNeedsAttention(session, reason, `recorded worktree missing: ${worktreePath ?? "<none>"}`);
|
||||||
|
return { ...base, disposition: "needsAttention-missingWorktree", reason };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adapter must support resume.
|
||||||
|
const adapter = (() => {
|
||||||
|
try {
|
||||||
|
return this.registry.get(session.adapterId);
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
if (!adapter || !adapter.capabilities.supportsResume || typeof adapter.buildResume !== "function") {
|
||||||
|
this.toNeedsAttention(session, reason, `adapter does not support resume: ${session.adapterId}`);
|
||||||
|
return { ...base, disposition: "needsAttention-resumeUnsupported", reason };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Vendor session store precondition: a captured native id is required.
|
||||||
|
if (!session.nativeSessionId) {
|
||||||
|
this.toNeedsAttention(session, reason, "missing native session id (no vendor session store)");
|
||||||
|
return { ...base, disposition: "needsAttention-spawnError", reason };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dirty-tree detection: log + flag, then PROCEED.
|
||||||
|
let dirty = false;
|
||||||
|
try {
|
||||||
|
dirty = await this.isWorktreeDirty(worktreePath);
|
||||||
|
} catch {
|
||||||
|
dirty = false;
|
||||||
|
}
|
||||||
|
if (dirty) {
|
||||||
|
this.log(`[cli-resume] session ${session.id}: worktree dirty at resume — flagged, proceeding`);
|
||||||
|
this.flagDirty(session);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Relaunch via the manager's resume path (adapter buildResume + native id),
|
||||||
|
// reusing the existing record so no duplicate session row is created.
|
||||||
|
try {
|
||||||
|
await this.manager.spawn({
|
||||||
|
adapterId: session.adapterId,
|
||||||
|
projectId: session.projectId,
|
||||||
|
purpose: session.purpose,
|
||||||
|
taskId: session.taskId,
|
||||||
|
chatSessionId: session.chatSessionId,
|
||||||
|
worktreePath,
|
||||||
|
posture: session.autonomyPosture,
|
||||||
|
resume: { sessionId: session.id, nativeSessionId: session.nativeSessionId },
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
// Immediate spawn failure / unsupported resume / missing vendor store →
|
||||||
|
// permanent-failure path. Record the attempt and route to needsAttention
|
||||||
|
// once the cap is reached; otherwise leave it for the next sweep (backoff).
|
||||||
|
const attempts = session.resumeAttempts + 1;
|
||||||
|
this.store.updateSession(session.id, { resumeAttempts: attempts });
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
const isUnsupported = err instanceof CliResumeUnsupportedError;
|
||||||
|
const isCeiling = err instanceof CliConcurrencyLimitError;
|
||||||
|
if (isCeiling) {
|
||||||
|
// Capacity raced away — leave persisted-live for the next sweep, no attempt charge.
|
||||||
|
this.store.updateSession(session.id, { resumeAttempts: session.resumeAttempts });
|
||||||
|
return { ...base, disposition: "skipped-noCapacity" };
|
||||||
|
}
|
||||||
|
this.log(`[cli-resume] session ${session.id}: resume spawn failed (${msg})`);
|
||||||
|
if (isUnsupported || attempts >= this.maxResumeAttempts) {
|
||||||
|
this.toNeedsAttention(session, reason, `resume spawn failed: ${msg}`);
|
||||||
|
return {
|
||||||
|
...base,
|
||||||
|
disposition: isUnsupported
|
||||||
|
? "needsAttention-resumeUnsupported"
|
||||||
|
: "needsAttention-spawnError",
|
||||||
|
reason: msg,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
// Under the cap: needsAttention is the permanent floor only at exhaustion;
|
||||||
|
// a single immediate failure (missing vendor store / spawn error) is also
|
||||||
|
// permanent per the KTD — do NOT loop. Route to needsAttention now.
|
||||||
|
this.toNeedsAttention(session, reason, `resume spawn failed: ${msg}`);
|
||||||
|
return { ...base, disposition: "needsAttention-spawnError", reason: msg };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-attach telemetry (fresh hook token + scripts). Best-effort.
|
||||||
|
if (this.reattachTelemetry) {
|
||||||
|
try {
|
||||||
|
const fresh = this.store.getSession(session.id) ?? session;
|
||||||
|
await this.reattachTelemetry(fresh);
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
this.log(`[cli-resume] session ${session.id}: telemetry re-attach failed (${msg}) — non-fatal`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.log(`[cli-resume] session ${session.id}: resumed (native ${session.nativeSessionId}) in ${worktreePath}`);
|
||||||
|
return { ...base, disposition: "resumed", dirtyWorktree: dirty };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Backoff (ms) before the next resume attempt for a given attempt count. */
|
||||||
|
backoffForAttempt(attemptsSoFar: number): number {
|
||||||
|
return this.resumeBackoffBaseMs * 2 ** attemptsSoFar;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Route a session to needsAttention, preserving the precise termination reason. */
|
||||||
|
private toNeedsAttention(session: CliSession, reason: CliTerminationReason, why: string): void {
|
||||||
|
this.log(`[cli-resume] session ${session.id} → needsAttention: ${why}`);
|
||||||
|
this.store.updateSession(session.id, {
|
||||||
|
agentState: "needsAttention",
|
||||||
|
terminationReason: reason,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Persist the dirty-worktree flag on the session record (extensible posture). */
|
||||||
|
private flagDirty(session: CliSession): void {
|
||||||
|
const posture = { ...(session.autonomyPosture ?? {}), resumeDirtyWorktree: true };
|
||||||
|
this.store.updateSession(session.id, { autonomyPosture: posture });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -37,7 +37,7 @@ import {
|
|||||||
} from "@fusion/core";
|
} from "@fusion/core";
|
||||||
import { loadPtyModule } from "../pty-native.js";
|
import { loadPtyModule } from "../pty-native.js";
|
||||||
import type { IPty } from "node-pty";
|
import type { IPty } from "node-pty";
|
||||||
import type { CliAdapterRegistry, CliAgentAdapter, CliReadinessDetector } from "./adapter.js";
|
import type { CliAdapterRegistry, CliAgentAdapter, CliLaunchSpec, CliReadinessDetector } from "./adapter.js";
|
||||||
|
|
||||||
// ── Constants ──────────────────────────────────────────────────────────────
|
// ── Constants ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -81,6 +81,15 @@ export class UnknownCliSessionError extends Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Thrown when a resume is requested for an adapter that cannot resume. */
|
||||||
|
export class CliResumeUnsupportedError extends Error {
|
||||||
|
readonly code = "CLI_RESUME_UNSUPPORTED";
|
||||||
|
constructor(public readonly adapterId: string) {
|
||||||
|
super(`CLI adapter does not support resume: ${adapterId}`);
|
||||||
|
this.name = "CliResumeUnsupportedError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── Injection neutralization (security-critical) ───────────────────────────
|
// ── Injection neutralization (security-critical) ───────────────────────────
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -292,6 +301,19 @@ export interface SpawnCliSessionOptions {
|
|||||||
/** Initial PTY size. */
|
/** Initial PTY size. */
|
||||||
cols?: number;
|
cols?: number;
|
||||||
rows?: number;
|
rows?: number;
|
||||||
|
/**
|
||||||
|
* Resume an existing session record instead of creating a new one. When set,
|
||||||
|
* spawn builds the launch invocation via the adapter's `buildResume` (carrying
|
||||||
|
* the recorded `nativeSessionId`) and REUSES the supplied record id rather than
|
||||||
|
* minting a fresh `cli_sessions` row — so a recovered session never produces a
|
||||||
|
* duplicate record. The adapter MUST advertise `supportsResume`/`buildResume`.
|
||||||
|
*/
|
||||||
|
resume?: {
|
||||||
|
/** The existing session record id to relaunch in place. */
|
||||||
|
sessionId: string;
|
||||||
|
/** The recorded native (vendor) session id handed to `buildResume`. */
|
||||||
|
nativeSessionId: string;
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Internal live-session state ─────────────────────────────────────────────
|
// ── Internal live-session state ─────────────────────────────────────────────
|
||||||
@@ -379,6 +401,16 @@ export class CliSessionManager {
|
|||||||
return this.sessions.size;
|
return this.sessions.size;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Configured ceiling on concurrently live PTY sessions. */
|
||||||
|
capacity(): number {
|
||||||
|
return this.concurrencyCeiling;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Free concurrency slots remaining before the ceiling (never negative). */
|
||||||
|
availableSlots(): number {
|
||||||
|
return Math.max(0, this.concurrencyCeiling - this.sessions.size);
|
||||||
|
}
|
||||||
|
|
||||||
/** Whether a session id is currently live. */
|
/** Whether a session id is currently live. */
|
||||||
isLive(sessionId: string): boolean {
|
isLive(sessionId: string): boolean {
|
||||||
return this.sessions.has(sessionId);
|
return this.sessions.has(sessionId);
|
||||||
@@ -403,23 +435,43 @@ export class CliSessionManager {
|
|||||||
settings: (options.settings ?? {}) as Record<string, unknown>,
|
settings: (options.settings ?? {}) as Record<string, unknown>,
|
||||||
posture,
|
posture,
|
||||||
};
|
};
|
||||||
const launch = adapter.buildLaunch(launchCtx);
|
|
||||||
|
// Resume vs fresh launch. A resume relaunches the recorded native session id
|
||||||
|
// via the adapter's `buildResume` and REUSES the existing record (no
|
||||||
|
// duplicate row); a fresh launch uses `buildLaunch` and mints a new record.
|
||||||
|
let launch: CliLaunchSpec;
|
||||||
|
let record: CliSession;
|
||||||
|
if (options.resume) {
|
||||||
|
if (!adapter.capabilities.supportsResume || typeof adapter.buildResume !== "function") {
|
||||||
|
throw new CliResumeUnsupportedError(options.adapterId);
|
||||||
|
}
|
||||||
|
launch = adapter.buildResume({ ...launchCtx, nativeSessionId: options.resume.nativeSessionId });
|
||||||
|
const existing = this.store.getSession(options.resume.sessionId);
|
||||||
|
if (!existing) throw new UnknownCliSessionError(options.resume.sessionId);
|
||||||
|
// Move the reused record back to "starting" for the relaunch.
|
||||||
|
record = this.store.updateSession(options.resume.sessionId, {
|
||||||
|
agentState: "starting",
|
||||||
|
worktreePath: options.worktreePath ?? existing.worktreePath ?? null,
|
||||||
|
}) ?? existing;
|
||||||
|
} else {
|
||||||
|
launch = adapter.buildLaunch(launchCtx);
|
||||||
|
// Persist the session record BEFORE spawning so a crash mid-spawn still has
|
||||||
|
// a durable record to reason about.
|
||||||
|
record = this.store.createSession({
|
||||||
|
adapterId: options.adapterId,
|
||||||
|
projectId: options.projectId,
|
||||||
|
purpose: options.purpose,
|
||||||
|
taskId: options.taskId ?? null,
|
||||||
|
chatSessionId: options.chatSessionId ?? null,
|
||||||
|
worktreePath: options.worktreePath ?? null,
|
||||||
|
autonomyPosture: posture,
|
||||||
|
agentState: "starting",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const allowlist = adapter.buildEnvAllowlist(launchCtx);
|
const allowlist = adapter.buildEnvAllowlist(launchCtx);
|
||||||
const env = this.buildEnv(allowlist);
|
const env = this.buildEnv(allowlist);
|
||||||
|
|
||||||
// Persist the session record BEFORE spawning so a crash mid-spawn still has
|
|
||||||
// a durable record to reason about.
|
|
||||||
const record = this.store.createSession({
|
|
||||||
adapterId: options.adapterId,
|
|
||||||
projectId: options.projectId,
|
|
||||||
purpose: options.purpose,
|
|
||||||
taskId: options.taskId ?? null,
|
|
||||||
chatSessionId: options.chatSessionId ?? null,
|
|
||||||
worktreePath: options.worktreePath ?? null,
|
|
||||||
autonomyPosture: posture,
|
|
||||||
agentState: "starting",
|
|
||||||
});
|
|
||||||
|
|
||||||
const pty = await this.loadPty();
|
const pty = await this.loadPty();
|
||||||
let child: IPty;
|
let child: IPty;
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -628,6 +628,7 @@ export {
|
|||||||
export {
|
export {
|
||||||
CliSessionManager,
|
CliSessionManager,
|
||||||
CliConcurrencyLimitError,
|
CliConcurrencyLimitError,
|
||||||
|
CliResumeUnsupportedError,
|
||||||
UnknownCliSessionError,
|
UnknownCliSessionError,
|
||||||
neutralizeInjection,
|
neutralizeInjection,
|
||||||
DEFAULT_SCROLLBACK_BYTES,
|
DEFAULT_SCROLLBACK_BYTES,
|
||||||
@@ -636,6 +637,15 @@ export {
|
|||||||
type CliSessionManagerOptions,
|
type CliSessionManagerOptions,
|
||||||
type SpawnCliSessionOptions,
|
type SpawnCliSessionOptions,
|
||||||
} from "./cli-agent/session-manager.js";
|
} from "./cli-agent/session-manager.js";
|
||||||
|
// CLI Agent Executor — resume coordinator + self-healing/stuck integration (U8).
|
||||||
|
export {
|
||||||
|
CliResumeCoordinator,
|
||||||
|
DEFAULT_MAX_RESUME_ATTEMPTS,
|
||||||
|
DEFAULT_RESUME_BACKOFF_BASE_MS,
|
||||||
|
type CliResumeCoordinatorOptions,
|
||||||
|
type ResumeResult,
|
||||||
|
type ResumeDisposition,
|
||||||
|
} from "./cli-agent/resume-coordinator.js";
|
||||||
export {
|
export {
|
||||||
TelemetryHub,
|
TelemetryHub,
|
||||||
stripAnsiControl,
|
stripAnsiControl,
|
||||||
|
|||||||
@@ -293,6 +293,16 @@ export interface SelfHealingOptions {
|
|||||||
recoverActiveMissionValidations?: () => Promise<{ recoveredCount: number }>;
|
recoverActiveMissionValidations?: () => Promise<{ recoveredCount: number }>;
|
||||||
/** Optional callback to reap stale mission validator runs during startup and maintenance. */
|
/** Optional callback to reap stale mission validator runs during startup and maintenance. */
|
||||||
reapStaleMissionValidatorRuns?: () => Promise<{ reapedCount: number }>;
|
reapStaleMissionValidatorRuns?: () => Promise<{ reapedCount: number }>;
|
||||||
|
/**
|
||||||
|
* U8 (CLI Agent Executor): returns true when a worktree path backs a
|
||||||
|
* resume-eligible `cli_sessions` record. Idle-worktree sweeps
|
||||||
|
* (`enforceWorktreeCap`, `cleanupOrphans`, `reapUnregisteredOrphans`) MUST
|
||||||
|
* treat such a worktree as in-use, so a reaped-but-resumable session cannot
|
||||||
|
* have its worktree reclaimed out from under it before the resume coordinator
|
||||||
|
* relaunches the CLI. Narrow seam: a single predicate; absence (undefined)
|
||||||
|
* preserves the prior behavior. The path passed is the absolute worktree dir.
|
||||||
|
*/
|
||||||
|
isWorktreeResumeReserved?: (worktreePath: string) => boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const APPROVED_TRIAGE_RECOVERY_GRACE_MS = 60_000;
|
const APPROVED_TRIAGE_RECOVERY_GRACE_MS = 60_000;
|
||||||
@@ -1551,6 +1561,27 @@ export class SelfHealingManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* U8 seam: whether a worktree path backs a resume-eligible CLI agent session
|
||||||
|
* and must therefore be treated as in-use by idle sweeps. Defensive: a throw
|
||||||
|
* in the injected predicate is treated as "reserved" (conservative — never
|
||||||
|
* reclaim a worktree we can't prove is free).
|
||||||
|
*/
|
||||||
|
private isWorktreeResumeReserved(worktreePath: string): boolean {
|
||||||
|
const predicate = this.options.isWorktreeResumeReserved;
|
||||||
|
if (!predicate) return false;
|
||||||
|
try {
|
||||||
|
return predicate(resolve(worktreePath));
|
||||||
|
} catch (err: unknown) {
|
||||||
|
log.warn(
|
||||||
|
`[self-healing] resume-reserved check threw for ${worktreePath}: ${
|
||||||
|
err instanceof Error ? err.message : String(err)
|
||||||
|
} — treating as reserved (conservative)`,
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async cleanupWorktreeOnly(task: Task): Promise<void> {
|
private async cleanupWorktreeOnly(task: Task): Promise<void> {
|
||||||
if (task.worktree && existsSync(task.worktree)) {
|
if (task.worktree && existsSync(task.worktree)) {
|
||||||
try {
|
try {
|
||||||
@@ -8434,6 +8465,11 @@ export class SelfHealingManager {
|
|||||||
|
|
||||||
let cleaned = 0;
|
let cleaned = 0;
|
||||||
for (const worktreePath of orphaned) {
|
for (const worktreePath of orphaned) {
|
||||||
|
// U8: never reclaim a worktree backing a resume-eligible CLI session.
|
||||||
|
if (this.isWorktreeResumeReserved(worktreePath)) {
|
||||||
|
log.log(`[self-healing] deferring idle-sweep for ${worktreePath}: resume-eligible CLI session present`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
await removeWorktree({
|
await removeWorktree({
|
||||||
rootDir: this.options.rootDir,
|
rootDir: this.options.rootDir,
|
||||||
@@ -8506,6 +8542,11 @@ export class SelfHealingManager {
|
|||||||
log.log(`[self-healing] deferring unregistered-orphan reap for ${path}: active session present`);
|
log.log(`[self-healing] deferring unregistered-orphan reap for ${path}: active session present`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
// U8: never reclaim a worktree backing a resume-eligible CLI session.
|
||||||
|
if (this.isWorktreeResumeReserved(path)) {
|
||||||
|
log.log(`[self-healing] deferring unregistered-orphan reap for ${path}: resume-eligible CLI session present`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
rmSync(path, { recursive: true, force: true });
|
rmSync(path, { recursive: true, force: true });
|
||||||
log.log(`Cleaned unregistered worktree dir: ${path}`);
|
log.log(`Cleaned unregistered worktree dir: ${path}`);
|
||||||
@@ -8639,6 +8680,11 @@ export class SelfHealingManager {
|
|||||||
|
|
||||||
for (const { path: worktreePath } of withMtime) {
|
for (const { path: worktreePath } of withMtime) {
|
||||||
if (removed >= excess) break;
|
if (removed >= excess) break;
|
||||||
|
// U8: never reclaim a worktree backing a resume-eligible CLI session.
|
||||||
|
if (this.isWorktreeResumeReserved(worktreePath)) {
|
||||||
|
log.log(`[self-healing] cap-enforcement skipping ${worktreePath}: resume-eligible CLI session present`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
await removeWorktree({
|
await removeWorktree({
|
||||||
rootDir: this.options.rootDir,
|
rootDir: this.options.rootDir,
|
||||||
|
|||||||
@@ -109,6 +109,16 @@ export interface StuckTaskDetectorOptions {
|
|||||||
*
|
*
|
||||||
* Errors in this callback fall through to the normal kill path (treated as `false`). */
|
* Errors in this callback fall through to the normal kill path (treated as `false`). */
|
||||||
onLoopDetected?: (event: StuckTaskEvent) => Promise<boolean>;
|
onLoopDetected?: (event: StuckTaskEvent) => Promise<boolean>;
|
||||||
|
/**
|
||||||
|
* U8 (CLI Agent Executor): returns true when a task's live CLI agent session
|
||||||
|
* is `waitingOnInput` — expected idleness (a permission/question prompt). When
|
||||||
|
* true, stuck/inactivity flagging is SUPPRESSED for that task this cycle: the
|
||||||
|
* agent is intentionally quiet waiting for a human, not stalled. The U3 stall
|
||||||
|
* backstop (the CLI session state machine's own watchdog) remains the only
|
||||||
|
* escalation path while waiting. Narrow seam: a single lookup; absence
|
||||||
|
* (undefined) preserves the prior behavior. Errors are treated as "not
|
||||||
|
* waiting" (fail toward the normal stuck path, never silently suppress). */
|
||||||
|
isCliSessionWaitingOnInput?: (taskId: string) => boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class StuckTaskDetector {
|
export class StuckTaskDetector {
|
||||||
@@ -118,6 +128,7 @@ export class StuckTaskDetector {
|
|||||||
private onStuck?: (event: StuckTaskEvent) => void;
|
private onStuck?: (event: StuckTaskEvent) => void;
|
||||||
private beforeRequeue?: (taskId: string, reason: "inactivity" | "loop" | "no-progress-churn", event: StuckTaskEvent) => Promise<boolean>;
|
private beforeRequeue?: (taskId: string, reason: "inactivity" | "loop" | "no-progress-churn", event: StuckTaskEvent) => Promise<boolean>;
|
||||||
private onLoopDetected?: (event: StuckTaskEvent) => Promise<boolean>;
|
private onLoopDetected?: (event: StuckTaskEvent) => Promise<boolean>;
|
||||||
|
private isCliSessionWaitingOnInput?: (taskId: string) => boolean;
|
||||||
private paused = false;
|
private paused = false;
|
||||||
private exhaustedTasks = new Set<string>();
|
private exhaustedTasks = new Set<string>();
|
||||||
|
|
||||||
@@ -129,6 +140,23 @@ export class StuckTaskDetector {
|
|||||||
this.onStuck = options.onStuck;
|
this.onStuck = options.onStuck;
|
||||||
this.beforeRequeue = options.beforeRequeue;
|
this.beforeRequeue = options.beforeRequeue;
|
||||||
this.onLoopDetected = options.onLoopDetected;
|
this.onLoopDetected = options.onLoopDetected;
|
||||||
|
this.isCliSessionWaitingOnInput = options.isCliSessionWaitingOnInput;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* U8: whether stuck flagging should be suppressed for a task because its live
|
||||||
|
* CLI agent session is `waitingOnInput` (expected idleness). Defensive: a
|
||||||
|
* throw in the injected lookup is treated as "not waiting" so a broken seam
|
||||||
|
* never silently disables stuck detection.
|
||||||
|
*/
|
||||||
|
private isWaitingOnInput(canonicalTaskId: string): boolean {
|
||||||
|
if (!this.isCliSessionWaitingOnInput) return false;
|
||||||
|
try {
|
||||||
|
return this.isCliSessionWaitingOnInput(canonicalTaskId);
|
||||||
|
} catch (err) {
|
||||||
|
stuckLog.error(`waitingOnInput lookup threw for ${canonicalTaskId}; treating as not waiting:`, err);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -617,6 +645,13 @@ export class StuckTaskDetector {
|
|||||||
}
|
}
|
||||||
const reason = this.classifyStuckReason(taskId, timeoutMs);
|
const reason = this.classifyStuckReason(taskId, timeoutMs);
|
||||||
if (reason !== null) {
|
if (reason !== null) {
|
||||||
|
// U8: suppress flagging while the CLI session is waitingOnInput
|
||||||
|
// (expected idleness). The U3 stall backstop remains the only escalation
|
||||||
|
// path while genuinely waiting.
|
||||||
|
if (this.isWaitingOnInput(entry.canonicalTaskId)) {
|
||||||
|
stuckLog.log(`Suppressing stuck flag for ${taskId} (canonical=${entry.canonicalTaskId}) — CLI session waitingOnInput`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
stuckTasks.push(taskId);
|
stuckTasks.push(taskId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user