fix(FN-1473): recover no-progress task_done failures
This commit is contained in:
11
.changeset/add-quick-chat-hide-option.md
Normal file
11
.changeset/add-quick-chat-hide-option.md
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
---
|
||||||
|
"@gsxdsm/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
Add quick chat FAB hide option and fix mobile nav overlap
|
||||||
|
|
||||||
|
- Add project-scoped `showQuickChatFAB` setting to control Quick Chat FAB visibility (default: true)
|
||||||
|
- When disabled, the FAB is hidden but chat remains accessible from the More menu
|
||||||
|
- Fix mobile Quick Chat FAB/panel positioning to properly account for mobile nav bar height and safe-area insets
|
||||||
|
- Add regression tests for QuickChatFAB visibility behavior
|
||||||
|
- Update mobile CSS tests to properly verify Quick Chat offset rules
|
||||||
117
packages/dashboard/src/__tests__/debug-execfile.test.ts
Normal file
117
packages/dashboard/src/__tests__/debug-execfile.test.ts
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
import { describe, it, expect, vi, beforeEach, afterEach, beforeAll, afterAll } from "vitest";
|
||||||
|
import { EventEmitter } from "node:events";
|
||||||
|
import { execSync } from "node:child_process";
|
||||||
|
import { mkdirSync, writeFileSync, rmSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import type { Task } from "@fusion/core";
|
||||||
|
|
||||||
|
import { createApiRoutes } from "../routes.js";
|
||||||
|
|
||||||
|
class MockStore extends EventEmitter {
|
||||||
|
private tasks = new Map<string, Task>();
|
||||||
|
|
||||||
|
getRootDir(): string { return process.cwd(); }
|
||||||
|
async getTask(id: string): Promise<Task> {
|
||||||
|
const task = this.tasks.get(id);
|
||||||
|
if (!task) throw Object.assign(new Error("Task not found"), { code: "ENOENT" });
|
||||||
|
return task;
|
||||||
|
}
|
||||||
|
addTask(task: Task): void { this.tasks.set(task.id, task); }
|
||||||
|
getMissionStore() { return new EventEmitter(); }
|
||||||
|
async listTasks(): Promise<Task[]> { return []; }
|
||||||
|
}
|
||||||
|
|
||||||
|
function createTask(overrides: Partial<Task> = {}): Task {
|
||||||
|
return {
|
||||||
|
id: "FN-675", title: "Test task", description: "Test description",
|
||||||
|
column: "in-progress", dependencies: [], steps: [], currentStep: 0, log: [],
|
||||||
|
createdAt: "2026-04-01T00:00:00.000Z", updatedAt: "2026-04-01T00:00:00.000Z",
|
||||||
|
columnMovedAt: "2026-04-01T00:00:00.000Z", worktree: "/tmp/fn-675",
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("Session files endpoint", () => {
|
||||||
|
let testWorktree: string;
|
||||||
|
let firstCommit: string;
|
||||||
|
|
||||||
|
beforeAll(() => {
|
||||||
|
// Create a test git repository
|
||||||
|
testWorktree = join(tmpdir(), `fn-test-${Date.now()}`);
|
||||||
|
mkdirSync(testWorktree, { recursive: true });
|
||||||
|
|
||||||
|
// Initialize git repo
|
||||||
|
execSync("git init", { cwd: testWorktree });
|
||||||
|
execSync("git config user.email test@test.com", { cwd: testWorktree });
|
||||||
|
execSync("git config user.name Test", { cwd: testWorktree });
|
||||||
|
|
||||||
|
// Create a commit
|
||||||
|
writeFileSync(join(testWorktree, "test.txt"), "initial content");
|
||||||
|
execSync("git add .", { cwd: testWorktree });
|
||||||
|
execSync("git commit -m 'initial'", { cwd: testWorktree });
|
||||||
|
|
||||||
|
// Get the first commit SHA
|
||||||
|
firstCommit = execSync("git rev-parse HEAD", { cwd: testWorktree }).toString().trim();
|
||||||
|
|
||||||
|
// Create a second commit with changes
|
||||||
|
writeFileSync(join(testWorktree, "changed.txt"), "new content");
|
||||||
|
execSync("git add .", { cwd: testWorktree });
|
||||||
|
execSync("git commit -m 'second commit'", { cwd: testWorktree });
|
||||||
|
|
||||||
|
console.log("Test worktree:", testWorktree);
|
||||||
|
console.log("First commit:", firstCommit);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(() => {
|
||||||
|
// Clean up
|
||||||
|
try {
|
||||||
|
rmSync(testWorktree, { recursive: true, force: true });
|
||||||
|
} catch (e) {
|
||||||
|
// Ignore cleanup errors
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"] });
|
||||||
|
vi.setSystemTime(new Date("2026-04-01T12:00:00.000Z"));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("endpoint should return files from git using real baseCommitSha", async () => {
|
||||||
|
const store = new MockStore();
|
||||||
|
|
||||||
|
store.addTask(createTask({
|
||||||
|
id: "FN-TEST", title: "Test", description: "Test",
|
||||||
|
column: "in-progress", dependencies: [], steps: [], currentStep: 0, log: [],
|
||||||
|
createdAt: "2026-04-01T00:00:00.000Z", updatedAt: "2026-04-01T00:00:00.000Z",
|
||||||
|
columnMovedAt: "2026-04-01T00:00:00.000Z", worktree: testWorktree, baseCommitSha: firstCommit,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const router = createApiRoutes(store);
|
||||||
|
const layer = (router as any).stack.find(
|
||||||
|
(c: any) => c.route?.path === "/tasks/:id/session-files" && c.route?.methods?.get,
|
||||||
|
);
|
||||||
|
|
||||||
|
const handler = layer.route.stack[layer.route.stack.length - 1].handle;
|
||||||
|
|
||||||
|
const res: any = {
|
||||||
|
statusCode: 200,
|
||||||
|
body: undefined,
|
||||||
|
status(code: number) { this.statusCode = code; return this; },
|
||||||
|
json(payload: any) { this.body = payload; return this; }
|
||||||
|
};
|
||||||
|
|
||||||
|
await handler({ params: { id: "FN-TEST" } }, res);
|
||||||
|
|
||||||
|
console.log("Response body:", res.body);
|
||||||
|
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
expect(Array.isArray(res.body)).toBe(true);
|
||||||
|
// Should have changed.txt as a changed file
|
||||||
|
expect(res.body).toContain("changed.txt");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -606,6 +606,12 @@ export class TaskExecutor {
|
|||||||
return task.steps.every((s) => s.status === "done" || s.status === "skipped");
|
return task.steps.every((s) => s.status === "done" || s.status === "skipped");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private isNoProgressNoTaskDoneFailure(task: Task): boolean {
|
||||||
|
return task.status === "failed" &&
|
||||||
|
task.error?.includes("without calling task_done") === true &&
|
||||||
|
task.steps.every((step) => step.status === "pending");
|
||||||
|
}
|
||||||
|
|
||||||
private async clearResumeFailureState(task: Task): Promise<void> {
|
private async clearResumeFailureState(task: Task): Promise<void> {
|
||||||
if (task.status === "failed" || task.error) {
|
if (task.status === "failed" || task.error) {
|
||||||
await this.store.updateTask(task.id, { status: null, error: null });
|
await this.store.updateTask(task.id, { status: null, error: null });
|
||||||
@@ -736,6 +742,11 @@ export class TaskExecutor {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (this.isNoProgressNoTaskDoneFailure(task)) {
|
||||||
|
executorLog.log(`${task.id} failed without task_done and has no step progress — leaving for self-healing requeue`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
executorLog.log(`Resuming ${task.id}: ${task.title || task.description.slice(0, 60)}`);
|
executorLog.log(`Resuming ${task.id}: ${task.title || task.description.slice(0, 60)}`);
|
||||||
try {
|
try {
|
||||||
await this.clearResumeFailureState(task);
|
await this.clearResumeFailureState(task);
|
||||||
|
|||||||
@@ -427,6 +427,25 @@ describe("In-progress task resume after restart", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("resumeOrphaned() leaves no-progress no-task_done failures for self-healing", async () => {
|
||||||
|
const store = createMockStore();
|
||||||
|
const failedTask = makeTask("FN-1473", "in-progress", {
|
||||||
|
status: "failed",
|
||||||
|
error: "Agent finished without calling task_done (after retry)",
|
||||||
|
steps: [],
|
||||||
|
});
|
||||||
|
store.listTasks.mockResolvedValue([failedTask]);
|
||||||
|
|
||||||
|
const executor = new TaskExecutor(store, "/tmp/test");
|
||||||
|
const executeSpy = vi.spyOn(executor, "execute");
|
||||||
|
|
||||||
|
await executor.resumeOrphaned();
|
||||||
|
|
||||||
|
expect(executeSpy).not.toHaveBeenCalled();
|
||||||
|
expect(mockedCreateHaiAgent).not.toHaveBeenCalled();
|
||||||
|
expect(store.logEntry).not.toHaveBeenCalledWith("FN-1473", "Resumed after engine restart");
|
||||||
|
});
|
||||||
|
|
||||||
it("recoverCompletedTask() marks task failed then moves to in-review when workflow fails", async () => {
|
it("recoverCompletedTask() marks task failed then moves to in-review when workflow fails", async () => {
|
||||||
const store = createMockStore({
|
const store = createMockStore({
|
||||||
getTask: vi.fn().mockResolvedValue(makeTaskDetail("FN-963", "in-progress", {
|
getTask: vi.fn().mockResolvedValue(makeTaskDetail("FN-963", "in-progress", {
|
||||||
|
|||||||
@@ -11,12 +11,14 @@ const {
|
|||||||
mockSelfHealingStart,
|
mockSelfHealingStart,
|
||||||
mockSelfHealingStop,
|
mockSelfHealingStop,
|
||||||
mockSelfHealingCtor,
|
mockSelfHealingCtor,
|
||||||
|
mockRecoverNoProgressNoTaskDoneFailures,
|
||||||
mockRunStartupRecovery,
|
mockRunStartupRecovery,
|
||||||
mockExecutorCtor,
|
mockExecutorCtor,
|
||||||
} = vi.hoisted(() => ({
|
} = vi.hoisted(() => ({
|
||||||
mockSelfHealingStart: vi.fn(),
|
mockSelfHealingStart: vi.fn(),
|
||||||
mockSelfHealingStop: vi.fn(),
|
mockSelfHealingStop: vi.fn(),
|
||||||
mockSelfHealingCtor: vi.fn(),
|
mockSelfHealingCtor: vi.fn(),
|
||||||
|
mockRecoverNoProgressNoTaskDoneFailures: vi.fn().mockResolvedValue(0),
|
||||||
mockRunStartupRecovery: vi.fn().mockResolvedValue(undefined),
|
mockRunStartupRecovery: vi.fn().mockResolvedValue(undefined),
|
||||||
mockExecutorCtor: vi.fn(),
|
mockExecutorCtor: vi.fn(),
|
||||||
}));
|
}));
|
||||||
@@ -97,6 +99,7 @@ vi.mock("../self-healing.js", async () => {
|
|||||||
return {
|
return {
|
||||||
start: mockSelfHealingStart,
|
start: mockSelfHealingStart,
|
||||||
stop: mockSelfHealingStop,
|
stop: mockSelfHealingStop,
|
||||||
|
recoverNoProgressNoTaskDoneFailures: mockRecoverNoProgressNoTaskDoneFailures,
|
||||||
runStartupRecovery: mockRunStartupRecovery,
|
runStartupRecovery: mockRunStartupRecovery,
|
||||||
};
|
};
|
||||||
}),
|
}),
|
||||||
@@ -207,6 +210,7 @@ describe("InProcessRuntime", () => {
|
|||||||
it("runs self-healing startup recovery immediately after orphan resume on startup", async () => {
|
it("runs self-healing startup recovery immediately after orphan resume on startup", async () => {
|
||||||
await runtime.start();
|
await runtime.start();
|
||||||
|
|
||||||
|
expect(mockRecoverNoProgressNoTaskDoneFailures).toHaveBeenCalledTimes(1);
|
||||||
expect(mockRunStartupRecovery).toHaveBeenCalledTimes(1);
|
expect(mockRunStartupRecovery).toHaveBeenCalledTimes(1);
|
||||||
}, 30000);
|
}, 30000);
|
||||||
|
|
||||||
|
|||||||
@@ -417,7 +417,11 @@ export class InProcessRuntime
|
|||||||
// 8. Set up event forwarding from TaskStore
|
// 8. Set up event forwarding from TaskStore
|
||||||
this.setupEventForwarding();
|
this.setupEventForwarding();
|
||||||
|
|
||||||
// 9. Resume orphaned in-progress tasks
|
// 9. Requeue no-progress no-task_done failures before resumeOrphaned
|
||||||
|
// can restart them.
|
||||||
|
await this.selfHealingManager.recoverNoProgressNoTaskDoneFailures();
|
||||||
|
|
||||||
|
// 10. Resume orphaned in-progress tasks
|
||||||
await this.executor.resumeOrphaned();
|
await this.executor.resumeOrphaned();
|
||||||
|
|
||||||
// Some "stuck" tasks are already orphaned by the time the runtime boots:
|
// Some "stuck" tasks are already orphaned by the time the runtime boots:
|
||||||
@@ -426,10 +430,10 @@ export class InProcessRuntime
|
|||||||
// SelfHealingManager so the policy lives in one place.
|
// SelfHealingManager so the policy lives in one place.
|
||||||
await this.selfHealingManager.runStartupRecovery();
|
await this.selfHealingManager.runStartupRecovery();
|
||||||
|
|
||||||
// 10. Start scheduler
|
// 11. Start scheduler
|
||||||
this.scheduler.start();
|
this.scheduler.start();
|
||||||
|
|
||||||
// 11. Start MissionExecutionLoop for validation cycle handling
|
// 12. Start MissionExecutionLoop for validation cycle handling
|
||||||
this.missionExecutionLoop = missionExecutionLoop;
|
this.missionExecutionLoop = missionExecutionLoop;
|
||||||
if (missionExecutionLoop) {
|
if (missionExecutionLoop) {
|
||||||
missionExecutionLoop.start();
|
missionExecutionLoop.start();
|
||||||
@@ -446,7 +450,7 @@ export class InProcessRuntime
|
|||||||
void activeMissionAutopilot.recoverMissions(activeMissionStore);
|
void activeMissionAutopilot.recoverMissions(activeMissionStore);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 12. Reconcile feature status for all active missions (not just autopilot)
|
// 13. Reconcile feature status for all active missions (not just autopilot)
|
||||||
if (activeMissionStore) {
|
if (activeMissionStore) {
|
||||||
void this.scheduler.reconcileAllMissionFeatures();
|
void this.scheduler.reconcileAllMissionFeatures();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ import { existsSync } from "node:fs";
|
|||||||
import { scanOrphanedBranches } from "./worktree-pool.js";
|
import { scanOrphanedBranches } from "./worktree-pool.js";
|
||||||
|
|
||||||
const mockedExecSync = vi.mocked(execSync);
|
const mockedExecSync = vi.mocked(execSync);
|
||||||
|
const mockedExistsSync = vi.mocked(existsSync);
|
||||||
const mockedScanOrphanedBranches = vi.mocked(scanOrphanedBranches);
|
const mockedScanOrphanedBranches = vi.mocked(scanOrphanedBranches);
|
||||||
|
|
||||||
// ── Mock helpers ────────────────────────────────────────────────────
|
// ── Mock helpers ────────────────────────────────────────────────────
|
||||||
@@ -322,6 +323,7 @@ describe("SelfHealingManager", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("runStartupRecovery invokes the startup recovery subset", async () => {
|
it("runStartupRecovery invokes the startup recovery subset", async () => {
|
||||||
|
const recoverNoProgressNoTaskDoneFailures = vi.spyOn(manager, "recoverNoProgressNoTaskDoneFailures").mockResolvedValue(1);
|
||||||
const recoverCompletedTasks = vi.spyOn(manager, "recoverCompletedTasks").mockResolvedValue(1);
|
const recoverCompletedTasks = vi.spyOn(manager, "recoverCompletedTasks").mockResolvedValue(1);
|
||||||
const recoverMisclassifiedFailures = vi.spyOn(manager, "recoverMisclassifiedFailures").mockResolvedValue(1);
|
const recoverMisclassifiedFailures = vi.spyOn(manager, "recoverMisclassifiedFailures").mockResolvedValue(1);
|
||||||
const recoverOrphanedExecutions = vi.spyOn(manager, "recoverOrphanedExecutions").mockResolvedValue(1);
|
const recoverOrphanedExecutions = vi.spyOn(manager, "recoverOrphanedExecutions").mockResolvedValue(1);
|
||||||
@@ -329,6 +331,7 @@ describe("SelfHealingManager", () => {
|
|||||||
|
|
||||||
await manager.runStartupRecovery();
|
await manager.runStartupRecovery();
|
||||||
|
|
||||||
|
expect(recoverNoProgressNoTaskDoneFailures).toHaveBeenCalledTimes(1);
|
||||||
expect(recoverCompletedTasks).toHaveBeenCalledTimes(1);
|
expect(recoverCompletedTasks).toHaveBeenCalledTimes(1);
|
||||||
expect(recoverMisclassifiedFailures).toHaveBeenCalledTimes(1);
|
expect(recoverMisclassifiedFailures).toHaveBeenCalledTimes(1);
|
||||||
expect(recoverOrphanedExecutions).toHaveBeenCalledTimes(1);
|
expect(recoverOrphanedExecutions).toHaveBeenCalledTimes(1);
|
||||||
@@ -336,6 +339,115 @@ describe("SelfHealingManager", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("recoverNoProgressNoTaskDoneFailures", () => {
|
||||||
|
it("requeues clean in-progress no-task_done failures with no step progress", async () => {
|
||||||
|
const managerWithRecovery = new SelfHealingManager(store, {
|
||||||
|
rootDir: "/tmp/test-project",
|
||||||
|
getExecutingTaskIds: () => new Set<string>(),
|
||||||
|
});
|
||||||
|
vi.spyOn(managerWithRecovery as any, "hasRecoverableGitWork").mockReturnValue(false);
|
||||||
|
|
||||||
|
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
|
||||||
|
{
|
||||||
|
id: "FN-1473",
|
||||||
|
column: "in-progress",
|
||||||
|
status: "failed",
|
||||||
|
error: "Agent finished without calling task_done (after retry)",
|
||||||
|
paused: false,
|
||||||
|
steps: [],
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const result = await managerWithRecovery.recoverNoProgressNoTaskDoneFailures();
|
||||||
|
|
||||||
|
expect(result).toBe(1);
|
||||||
|
expect(store.updateTask).toHaveBeenCalledWith("FN-1473", {
|
||||||
|
status: "stuck-killed",
|
||||||
|
worktree: null,
|
||||||
|
branch: null,
|
||||||
|
});
|
||||||
|
expect(store.logEntry).toHaveBeenCalledWith(
|
||||||
|
"FN-1473",
|
||||||
|
expect.stringContaining("no-progress no-task_done failure"),
|
||||||
|
);
|
||||||
|
expect(store.moveTask).toHaveBeenCalledWith("FN-1473", "todo");
|
||||||
|
|
||||||
|
managerWithRecovery.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips no-task_done failures with step progress", async () => {
|
||||||
|
const managerWithRecovery = new SelfHealingManager(store, {
|
||||||
|
rootDir: "/tmp/test-project",
|
||||||
|
getExecutingTaskIds: () => new Set<string>(),
|
||||||
|
});
|
||||||
|
vi.spyOn(managerWithRecovery as any, "hasRecoverableGitWork").mockReturnValue(false);
|
||||||
|
|
||||||
|
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
|
||||||
|
{
|
||||||
|
id: "FN-1473",
|
||||||
|
column: "in-progress",
|
||||||
|
status: "failed",
|
||||||
|
error: "Agent finished without calling task_done (after retry)",
|
||||||
|
paused: false,
|
||||||
|
steps: [{ status: "done" }, { status: "pending" }],
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const result = await managerWithRecovery.recoverNoProgressNoTaskDoneFailures();
|
||||||
|
|
||||||
|
expect(result).toBe(0);
|
||||||
|
expect(store.updateTask).not.toHaveBeenCalledWith("FN-1473", expect.anything());
|
||||||
|
expect(store.moveTask).not.toHaveBeenCalledWith("FN-1473", "todo");
|
||||||
|
|
||||||
|
managerWithRecovery.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips when git work should be preserved", async () => {
|
||||||
|
const managerWithRecovery = new SelfHealingManager(store, {
|
||||||
|
rootDir: "/tmp/test-project",
|
||||||
|
getExecutingTaskIds: () => new Set<string>(),
|
||||||
|
});
|
||||||
|
vi.spyOn(managerWithRecovery as any, "hasRecoverableGitWork").mockReturnValue(true);
|
||||||
|
|
||||||
|
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
|
||||||
|
{
|
||||||
|
id: "FN-1473",
|
||||||
|
column: "in-progress",
|
||||||
|
status: "failed",
|
||||||
|
error: "Agent finished without calling task_done (after retry)",
|
||||||
|
paused: false,
|
||||||
|
steps: [{ status: "pending" }],
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const result = await managerWithRecovery.recoverNoProgressNoTaskDoneFailures();
|
||||||
|
|
||||||
|
expect(result).toBe(0);
|
||||||
|
expect(store.updateTask).not.toHaveBeenCalledWith("FN-1473", expect.anything());
|
||||||
|
expect(store.moveTask).not.toHaveBeenCalledWith("FN-1473", "todo");
|
||||||
|
|
||||||
|
managerWithRecovery.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats dirty worktrees as recoverable git work", () => {
|
||||||
|
const task = {
|
||||||
|
id: "FN-1473",
|
||||||
|
worktree: "/tmp/test-project/.worktrees/fn-1473",
|
||||||
|
branch: "fusion/fn-1473",
|
||||||
|
} as Task;
|
||||||
|
mockedExistsSync.mockReturnValue(true);
|
||||||
|
mockedExecSync.mockImplementation((command) => {
|
||||||
|
if (String(command) === "git status --porcelain") {
|
||||||
|
return " M packages/engine/src/executor.ts\n" as any;
|
||||||
|
}
|
||||||
|
return "" as any;
|
||||||
|
});
|
||||||
|
|
||||||
|
expect((manager as any).hasRecoverableGitWork(task)).toBe(true);
|
||||||
|
mockedExecSync.mockClear();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
// ── cleanupOrphanedBranches ────────────────────────────────────────
|
// ── cleanupOrphanedBranches ────────────────────────────────────────
|
||||||
|
|
||||||
describe("cleanupOrphanedBranches", () => {
|
describe("cleanupOrphanedBranches", () => {
|
||||||
|
|||||||
@@ -103,6 +103,7 @@ export class SelfHealingManager {
|
|||||||
* stale in-progress/specifying tasks that no longer have a live worker.
|
* stale in-progress/specifying tasks that no longer have a live worker.
|
||||||
*/
|
*/
|
||||||
async runStartupRecovery(): Promise<void> {
|
async runStartupRecovery(): Promise<void> {
|
||||||
|
await this.recoverNoProgressNoTaskDoneFailures();
|
||||||
await this.recoverCompletedTasks();
|
await this.recoverCompletedTasks();
|
||||||
await this.recoverMisclassifiedFailures();
|
await this.recoverMisclassifiedFailures();
|
||||||
await this.recoverOrphanedExecutions();
|
await this.recoverOrphanedExecutions();
|
||||||
@@ -342,6 +343,7 @@ export class SelfHealingManager {
|
|||||||
await this.recoverMergeableReviewTasks();
|
await this.recoverMergeableReviewTasks();
|
||||||
await this.recoverMergedReviewTasks();
|
await this.recoverMergedReviewTasks();
|
||||||
await this.recoverMisclassifiedFailures();
|
await this.recoverMisclassifiedFailures();
|
||||||
|
await this.recoverNoProgressNoTaskDoneFailures();
|
||||||
await this.recoverOrphanedExecutions();
|
await this.recoverOrphanedExecutions();
|
||||||
await this.recoverApprovedTriageTasks();
|
await this.recoverApprovedTriageTasks();
|
||||||
await this.archiveStaleDoneTasks();
|
await this.archiveStaleDoneTasks();
|
||||||
@@ -675,6 +677,109 @@ export class SelfHealingManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recover `in-progress` tasks that failed only because the agent exited
|
||||||
|
* without calling task_done, and where there is no sign of work to preserve.
|
||||||
|
*
|
||||||
|
* These are safe to requeue automatically when no steps progressed and git
|
||||||
|
* has neither worktree changes nor branch commits. Cases with any evidence
|
||||||
|
* of work are left alone for manual inspection or the normal orphan recovery
|
||||||
|
* path.
|
||||||
|
*/
|
||||||
|
async recoverNoProgressNoTaskDoneFailures(): Promise<number> {
|
||||||
|
try {
|
||||||
|
const tasks = await this.store.listTasks({ slim: true, column: "in-progress" });
|
||||||
|
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||||||
|
|
||||||
|
const candidates = tasks.filter((task) =>
|
||||||
|
task.column === "in-progress" &&
|
||||||
|
task.status === "failed" &&
|
||||||
|
isNoTaskDoneFailure(task) &&
|
||||||
|
!task.paused &&
|
||||||
|
!executingIds.has(task.id) &&
|
||||||
|
!isTaskWorkComplete(task) &&
|
||||||
|
!hasStepProgress(task),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (candidates.length === 0) return 0;
|
||||||
|
|
||||||
|
log.warn(`Found ${candidates.length} no-progress no-task_done failure(s) in in-progress`);
|
||||||
|
|
||||||
|
let recovered = 0;
|
||||||
|
for (const task of candidates) {
|
||||||
|
try {
|
||||||
|
if (this.hasRecoverableGitWork(task)) {
|
||||||
|
log.log(`${task.id} has recoverable git work — leaving in-progress for inspection`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.store.updateTask(task.id, {
|
||||||
|
status: "stuck-killed",
|
||||||
|
worktree: null,
|
||||||
|
branch: null,
|
||||||
|
});
|
||||||
|
await this.store.logEntry(
|
||||||
|
task.id,
|
||||||
|
"Auto-recovered no-progress no-task_done failure — clean worktree, moved back to todo",
|
||||||
|
);
|
||||||
|
await this.store.moveTask(task.id, "todo");
|
||||||
|
recovered++;
|
||||||
|
} catch (err: any) {
|
||||||
|
log.error(`Failed to recover no-progress no-task_done failure ${task.id}: ${err.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (recovered > 0) {
|
||||||
|
log.log(`Recovered ${recovered} no-progress no-task_done failure(s) → todo`);
|
||||||
|
}
|
||||||
|
return recovered;
|
||||||
|
} catch (err: any) {
|
||||||
|
log.error(`No-progress no-task_done recovery failed: ${err.message}`);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private hasRecoverableGitWork(task: Task): boolean {
|
||||||
|
if (task.worktree && existsSync(task.worktree)) {
|
||||||
|
try {
|
||||||
|
const status = execSync("git status --porcelain", {
|
||||||
|
cwd: task.worktree,
|
||||||
|
stdio: "pipe",
|
||||||
|
encoding: "utf-8",
|
||||||
|
timeout: 30_000,
|
||||||
|
}).trim();
|
||||||
|
if (status.length > 0) return true;
|
||||||
|
} catch {
|
||||||
|
// If we cannot inspect an existing worktree, preserve it.
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const branchName = task.branch || `fusion/${task.id.toLowerCase()}`;
|
||||||
|
try {
|
||||||
|
execSync(`git rev-parse --verify "${branchName}"`, {
|
||||||
|
cwd: this.options.rootDir,
|
||||||
|
stdio: "pipe",
|
||||||
|
timeout: 30_000,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const uniqueCommits = execSync(`git rev-list --count HEAD.."${branchName}"`, {
|
||||||
|
cwd: this.options.rootDir,
|
||||||
|
stdio: "pipe",
|
||||||
|
encoding: "utf-8",
|
||||||
|
timeout: 30_000,
|
||||||
|
}).trim();
|
||||||
|
return Number.parseInt(uniqueCommits, 10) > 0;
|
||||||
|
} catch {
|
||||||
|
// If the branch exists but cannot be compared, preserve it.
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Recover triage tasks that already have an approved specification but were
|
* Recover triage tasks that already have an approved specification but were
|
||||||
* left stuck in `status: "specifying"` without an active triage session.
|
* left stuck in `status: "specifying"` without an active triage session.
|
||||||
@@ -907,3 +1012,11 @@ function isTaskWorkComplete(task: Task): boolean {
|
|||||||
if (task.steps.length === 0) return false;
|
if (task.steps.length === 0) return false;
|
||||||
return task.steps.every((step) => step.status === "done" || step.status === "skipped");
|
return task.steps.every((step) => step.status === "done" || step.status === "skipped");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isNoTaskDoneFailure(task: Task): boolean {
|
||||||
|
return task.error?.includes("without calling task_done") === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasStepProgress(task: Task): boolean {
|
||||||
|
return task.steps.some((step) => step.status !== "pending");
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user