feat(FN-5219): detect and recover in-progress limbo tasks
The merge adds a new self-healing recovery path for tasks stuck in `in-progress` limbo (no pending step updates but not marked done), hardening `resetTask` and `recoverInProgressLimboTasks` with proper worker binding cleanup, audit event coverage, and integration tests validating the invariant acros Fusion-Task-Id: FN-5219
This commit is contained in:
Fusion (runfusion.ai)
committed by
gsxdsm
gsxdsm
parent
646e546ef5
commit
1a4525e0b9
@@ -177,6 +177,7 @@ function createMockStore(overrides: Partial<TaskStore> = {}): TaskStore {
|
||||
createTask: vi.fn(),
|
||||
moveTask: vi.fn(),
|
||||
updateTask: vi.fn(),
|
||||
updateStep: vi.fn(),
|
||||
deleteTask: vi.fn(),
|
||||
mergeTask: vi.fn(),
|
||||
archiveTask: vi.fn(),
|
||||
@@ -206,6 +207,7 @@ function createMockStore(overrides: Partial<TaskStore> = {}): TaskStore {
|
||||
updateIssueInfo: vi.fn().mockResolvedValue(undefined),
|
||||
linkGithubIssue: vi.fn().mockResolvedValue(undefined),
|
||||
recordActivity: vi.fn().mockResolvedValue(undefined),
|
||||
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
|
||||
getRootDir: vi.fn().mockReturnValue("/fake/root"),
|
||||
listWorkflowSteps: vi.fn().mockResolvedValue([]),
|
||||
createWorkflowStep: vi.fn(),
|
||||
@@ -962,6 +964,173 @@ describe("POST /tasks/:id/refine", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /tasks/:id/reset", () => {
|
||||
let store: TaskStore;
|
||||
|
||||
beforeEach(() => {
|
||||
store = createMockStore({
|
||||
updateStep: vi.fn(),
|
||||
updateTask: vi.fn(),
|
||||
moveTask: vi.fn(),
|
||||
logEntry: vi.fn(),
|
||||
getTask: vi.fn(),
|
||||
});
|
||||
});
|
||||
|
||||
function buildApp(engine?: { getTaskStore: () => TaskStore; getAgentStore: () => { listAgents: ReturnType<typeof vi.fn>; syncExecutionTaskLink: ReturnType<typeof vi.fn>; deleteAgent: ReturnType<typeof vi.fn> } }) {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use("/api", createApiRoutes(store, engine ? { engine: engine as any } as any : undefined));
|
||||
return app;
|
||||
}
|
||||
|
||||
it("resets cleanly when moveTask already returns todo with cleared bindings", async () => {
|
||||
const agentStore = {
|
||||
listAgents: vi.fn().mockResolvedValue([
|
||||
{ id: "agent-durable", taskId: "FN-5200" },
|
||||
{ id: "agent-ephemeral", taskId: "FN-5200", name: "executor-FN-5200", role: "executor", reportsTo: null },
|
||||
]),
|
||||
syncExecutionTaskLink: vi.fn().mockResolvedValue(undefined),
|
||||
deleteAgent: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const engine = {
|
||||
getTaskStore: () => store,
|
||||
getAgentStore: () => agentStore,
|
||||
};
|
||||
const staleTask = {
|
||||
...FAKE_TASK_DETAIL,
|
||||
id: "FN-5200",
|
||||
column: "in-progress" as const,
|
||||
branch: "fusion/fn-5200",
|
||||
worktree: "/tmp/missing-worktree",
|
||||
steps: [{ title: "one", status: "done" }],
|
||||
};
|
||||
const cleanResetTask = {
|
||||
...staleTask,
|
||||
column: "todo" as const,
|
||||
branch: null,
|
||||
worktree: null,
|
||||
checkedOutBy: null,
|
||||
executionStartedAt: null,
|
||||
sessionFile: null,
|
||||
steps: [{ title: "one", status: "pending" }],
|
||||
};
|
||||
|
||||
(store.getTask as ReturnType<typeof vi.fn>)
|
||||
.mockResolvedValueOnce(staleTask)
|
||||
.mockResolvedValueOnce(cleanResetTask);
|
||||
(store.moveTask as ReturnType<typeof vi.fn>).mockResolvedValue(cleanResetTask);
|
||||
|
||||
const res = await REQUEST(buildApp(engine), "POST", "/api/tasks/FN-5200/reset", JSON.stringify({ confirm: true }), {
|
||||
"Content-Type": "application/json",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(store.updateTask).toHaveBeenCalledTimes(1);
|
||||
expect(store.logEntry).toHaveBeenCalledTimes(1);
|
||||
expect(store.recordRunAuditEvent).not.toHaveBeenCalled();
|
||||
expect(agentStore.syncExecutionTaskLink).toHaveBeenCalledWith("agent-durable", undefined);
|
||||
expect(agentStore.deleteAgent).toHaveBeenCalledWith("agent-ephemeral");
|
||||
expect(res.body.column).toBe("todo");
|
||||
expect(res.body.branch ?? null).toBeNull();
|
||||
expect(res.body.worktree ?? null).toBeNull();
|
||||
});
|
||||
|
||||
it("FN-5149: reset with stale missing worktree should not return in-progress limbo state", async () => {
|
||||
const agentStore = {
|
||||
listAgents: vi.fn().mockResolvedValue([
|
||||
{ id: "agent-durable", taskId: "FN-5149" },
|
||||
{ id: "agent-ephemeral", taskId: "FN-5149", name: "executor-FN-5149", role: "executor", reportsTo: null },
|
||||
]),
|
||||
syncExecutionTaskLink: vi.fn().mockResolvedValue(undefined),
|
||||
deleteAgent: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const engine = {
|
||||
getTaskStore: () => store,
|
||||
getAgentStore: () => agentStore,
|
||||
};
|
||||
const staleTask = {
|
||||
...FAKE_TASK_DETAIL,
|
||||
id: "FN-5149",
|
||||
column: "in-progress" as const,
|
||||
branch: "fusion/fn-5149",
|
||||
worktree: "/tmp/missing-worktree",
|
||||
checkedOutBy: "agent-reset",
|
||||
checkedOutAt: "2026-05-19T00:00:00.000Z",
|
||||
checkoutNodeId: "node-1",
|
||||
checkoutRunId: "run-1",
|
||||
checkoutLeaseRenewedAt: "2026-05-19T00:01:00.000Z",
|
||||
checkoutLeaseEpoch: 2,
|
||||
executionStartedAt: "2026-05-19T00:00:00.000Z",
|
||||
sessionFile: "/tmp/session.json",
|
||||
taskDoneRetryCount: 2,
|
||||
worktreeSessionRetryCount: 1,
|
||||
steps: [
|
||||
{ title: "one", status: "done" },
|
||||
{ title: "two", status: "in-progress" },
|
||||
],
|
||||
};
|
||||
const driftedAfterReset = {
|
||||
...staleTask,
|
||||
branch: null,
|
||||
worktree: "/tmp/missing-worktree",
|
||||
column: "in-progress" as const,
|
||||
steps: staleTask.steps.map((step) => ({ ...step, status: "pending" })),
|
||||
};
|
||||
const correctedAfterReset = {
|
||||
...driftedAfterReset,
|
||||
column: "todo" as const,
|
||||
worktree: null,
|
||||
checkedOutBy: null,
|
||||
checkedOutAt: null,
|
||||
checkoutNodeId: null,
|
||||
checkoutRunId: null,
|
||||
checkoutLeaseRenewedAt: null,
|
||||
checkoutLeaseEpoch: null,
|
||||
executionStartedAt: null,
|
||||
taskDoneRetryCount: null,
|
||||
worktreeSessionRetryCount: null,
|
||||
sessionFile: null,
|
||||
};
|
||||
|
||||
(store.getTask as ReturnType<typeof vi.fn>)
|
||||
.mockResolvedValueOnce(staleTask)
|
||||
.mockResolvedValueOnce(driftedAfterReset)
|
||||
.mockResolvedValueOnce(correctedAfterReset);
|
||||
(store.moveTask as ReturnType<typeof vi.fn>).mockResolvedValue(driftedAfterReset);
|
||||
(store.updateTask as ReturnType<typeof vi.fn>)
|
||||
.mockResolvedValueOnce(staleTask)
|
||||
.mockResolvedValueOnce(correctedAfterReset);
|
||||
|
||||
const res = await REQUEST(buildApp(engine), "POST", "/api/tasks/FN-5149/reset", JSON.stringify({ confirm: true }), {
|
||||
"Content-Type": "application/json",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(store.updateStep).toHaveBeenNthCalledWith(1, "FN-5149", 0, "pending");
|
||||
expect(store.updateStep).toHaveBeenNthCalledWith(2, "FN-5149", 1, "pending");
|
||||
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
domain: "database",
|
||||
mutationType: "task:auto-recover-reset-drift",
|
||||
target: "FN-5149",
|
||||
}));
|
||||
expect(agentStore.syncExecutionTaskLink).toHaveBeenCalledWith("agent-durable", undefined);
|
||||
expect(agentStore.deleteAgent).toHaveBeenCalledWith("agent-ephemeral");
|
||||
expect(store.logEntry).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
"FN-5149",
|
||||
"Auto-corrected reset drift after moveTask — normalized task back to todo with cleared worktree/branch bindings",
|
||||
expect.any(String),
|
||||
);
|
||||
expect(res.body.column).toBe("todo");
|
||||
expect(res.body.branch ?? null).toBeNull();
|
||||
expect(res.body.worktree ?? null).toBeNull();
|
||||
expect(res.body.steps.map((step: { status: string }) => step.status)).toEqual(["pending", "pending"]);
|
||||
expect(res.body.checkedOutBy).toBeFalsy();
|
||||
expect(res.body.executionStartedAt).toBeFalsy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("DELETE /tasks/:id", () => {
|
||||
let store: TaskStore;
|
||||
|
||||
@@ -971,18 +1140,30 @@ describe("DELETE /tasks/:id", () => {
|
||||
});
|
||||
});
|
||||
|
||||
function buildApp() {
|
||||
function buildApp(engine?: { getTaskStore: () => TaskStore; getAgentStore: () => { listAgents: ReturnType<typeof vi.fn>; syncExecutionTaskLink: ReturnType<typeof vi.fn>; deleteAgent: ReturnType<typeof vi.fn> } }) {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use("/api", createApiRoutes(store));
|
||||
app.use("/api", createApiRoutes(store, engine ? { engine: engine as any } as any : undefined));
|
||||
return app;
|
||||
}
|
||||
|
||||
it("deletes a task with the default safe mode", async () => {
|
||||
const agentStore = {
|
||||
listAgents: vi.fn().mockResolvedValue([
|
||||
{ id: "agent-durable", taskId: "KB-001" },
|
||||
{ id: "agent-ephemeral", taskId: "KB-001", name: "executor-KB-001", role: "executor", reportsTo: null },
|
||||
]),
|
||||
syncExecutionTaskLink: vi.fn().mockResolvedValue(undefined),
|
||||
deleteAgent: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const engine = {
|
||||
getTaskStore: () => store,
|
||||
getAgentStore: () => agentStore,
|
||||
};
|
||||
const deletedTask = { ...FAKE_TASK_DETAIL, id: "KB-001" };
|
||||
(store.deleteTask as ReturnType<typeof vi.fn>).mockResolvedValue(deletedTask);
|
||||
|
||||
const res = await REQUEST(buildApp(), "DELETE", "/api/tasks/KB-001");
|
||||
const res = await REQUEST(buildApp(engine), "DELETE", "/api/tasks/KB-001");
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.id).toBe("KB-001");
|
||||
@@ -995,6 +1176,8 @@ describe("DELETE /tasks/:id", () => {
|
||||
runId: expect.stringMatching(/^synthetic-dashboard-delete-KB-001-/),
|
||||
}),
|
||||
}));
|
||||
expect(agentStore.syncExecutionTaskLink).toHaveBeenCalledWith("agent-durable", undefined);
|
||||
expect(agentStore.deleteAgent).toHaveBeenCalledWith("agent-ephemeral");
|
||||
});
|
||||
|
||||
it("returns structured 409 conflict when delete is blocked by dependents", async () => {
|
||||
|
||||
@@ -30,12 +30,14 @@ import {
|
||||
reconcileDeterministicDuplicate,
|
||||
extractIntentSignature,
|
||||
findNearDuplicates,
|
||||
isEphemeralAgent,
|
||||
type NearDuplicateCandidate,
|
||||
} from "@fusion/core";
|
||||
import { GitHubClient } from "../github.js";
|
||||
import { createTrackingIssueForTask } from "../github-tracking-hook.js";
|
||||
import { parseGitHubBadgeUrl } from "./register-git-github.js";
|
||||
import { planTaskWorktreePath } from "@fusion/engine";
|
||||
import type { RunAuditEventInput } from "@fusion/core";
|
||||
import { ApiError, badRequest, conflict, notFound } from "../api-error.js";
|
||||
import type { ApiRoutesContext } from "./types.js";
|
||||
import { resolveBranchSelection } from "./branch-selection.js";
|
||||
@@ -47,6 +49,82 @@ const DUPLICATE_STOPWORDS = new Set(["a", "an", "the", "and", "or", "of", "to",
|
||||
|
||||
export const __fingerprintCreateLocksForTests = deterministicGuardLocks;
|
||||
|
||||
const RESET_TASK_FIELDS = {
|
||||
worktree: null,
|
||||
branch: null,
|
||||
currentStep: 0,
|
||||
status: null,
|
||||
error: null,
|
||||
stuckKillCount: 0,
|
||||
taskDoneRetryCount: null,
|
||||
worktreeSessionRetryCount: null,
|
||||
workflowStepRetries: undefined,
|
||||
recoveryRetryCount: null,
|
||||
nextRecoveryAt: null,
|
||||
postReviewFixCount: 0,
|
||||
verificationFailureCount: 0,
|
||||
mergeConflictBounceCount: 0,
|
||||
checkedOutBy: null,
|
||||
executionStartedAt: null,
|
||||
sessionFile: null,
|
||||
} as const;
|
||||
|
||||
const RESET_DRIFT_CORRECTION_FIELDS = {
|
||||
column: "todo" as const,
|
||||
worktree: null,
|
||||
branch: null,
|
||||
status: null,
|
||||
error: null,
|
||||
checkedOutBy: null,
|
||||
executionStartedAt: null,
|
||||
taskDoneRetryCount: null,
|
||||
worktreeSessionRetryCount: null,
|
||||
sessionFile: null,
|
||||
} as const;
|
||||
|
||||
async function emitResetDriftAudit(
|
||||
scopedStore: TaskStore,
|
||||
taskId: string,
|
||||
metadata: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
const recordRunAuditEvent = (scopedStore as TaskStore & {
|
||||
recordRunAuditEvent?: (input: RunAuditEventInput) => Promise<void>;
|
||||
}).recordRunAuditEvent;
|
||||
if (typeof recordRunAuditEvent !== "function") {
|
||||
return;
|
||||
}
|
||||
await recordRunAuditEvent({
|
||||
taskId,
|
||||
agentId: "system",
|
||||
runId: `synthetic-dashboard-reset-${taskId}-${Date.now()}`,
|
||||
domain: "database",
|
||||
mutationType: "task:auto-recover-reset-drift",
|
||||
target: taskId,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
async function releaseExecutionAgentBindings(
|
||||
engine: { getAgentStore?: () => { listAgents: (input: { includeEphemeral?: boolean }) => Promise<Array<{ id: string; taskId?: string }>>; syncExecutionTaskLink: (agentId: string, taskId: string | undefined) => Promise<unknown>; deleteAgent: (agentId: string) => Promise<unknown>; getAgent?: (agentId: string) => Promise<unknown>; } | undefined } | undefined,
|
||||
taskId: string,
|
||||
): Promise<void> {
|
||||
const agentStore = engine?.getAgentStore?.();
|
||||
if (!agentStore) {
|
||||
return;
|
||||
}
|
||||
|
||||
const linkedAgents = (await agentStore.listAgents({ includeEphemeral: true }))
|
||||
.filter((agent) => agent.taskId === taskId);
|
||||
|
||||
for (const agent of linkedAgents) {
|
||||
if (isEphemeralAgent(agent as never)) {
|
||||
await agentStore.deleteAgent(agent.id);
|
||||
continue;
|
||||
}
|
||||
await agentStore.syncExecutionTaskLink(agent.id, undefined);
|
||||
}
|
||||
}
|
||||
|
||||
function buildDuplicateQuery(title: string | undefined, description: string): string {
|
||||
const tokens = `${title ?? ""} ${description}`
|
||||
.toLowerCase()
|
||||
@@ -858,7 +936,7 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork
|
||||
// Nuclear reset — erase all progress and allocate a fresh worktree+branch on next run
|
||||
router.post("/tasks/:id/reset", async (req, res) => {
|
||||
try {
|
||||
const { store: scopedStore } = await getProjectContext(req);
|
||||
const { store: scopedStore, engine } = await getProjectContext(req);
|
||||
|
||||
const { confirm: confirmed } = (req.body ?? {}) as { confirm?: boolean };
|
||||
if (!confirmed) {
|
||||
@@ -876,28 +954,56 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork
|
||||
}
|
||||
}
|
||||
|
||||
await scopedStore.updateTask(req.params.id, {
|
||||
worktree: null,
|
||||
branch: null,
|
||||
currentStep: 0,
|
||||
status: null,
|
||||
error: null,
|
||||
stuckKillCount: 0,
|
||||
taskDoneRetryCount: null,
|
||||
workflowStepRetries: undefined,
|
||||
recoveryRetryCount: null,
|
||||
nextRecoveryAt: null,
|
||||
postReviewFixCount: 0,
|
||||
verificationFailureCount: 0,
|
||||
mergeConflictBounceCount: 0,
|
||||
});
|
||||
await scopedStore.updateTask(req.params.id, RESET_TASK_FIELDS);
|
||||
|
||||
await scopedStore.logEntry(
|
||||
req.params.id,
|
||||
"Task reset by user — all progress cleared, fresh worktree and branch will be allocated",
|
||||
);
|
||||
|
||||
const updated = await scopedStore.moveTask(req.params.id, "todo");
|
||||
await scopedStore.moveTask(req.params.id, "todo");
|
||||
await releaseExecutionAgentBindings(engine, req.params.id);
|
||||
let updated = await scopedStore.getTask(req.params.id);
|
||||
if (!updated) {
|
||||
throw notFound(`Task ${req.params.id} not found after reset`);
|
||||
}
|
||||
|
||||
const needsDriftCorrection = updated.column !== "todo"
|
||||
|| (updated.worktree ?? null) !== null
|
||||
|| (updated.branch ?? null) !== null
|
||||
|| (updated.checkedOutBy ?? null) !== null
|
||||
|| (updated.executionStartedAt ?? null) !== null;
|
||||
|
||||
if (needsDriftCorrection) {
|
||||
const offendingSnapshot = {
|
||||
column: updated.column,
|
||||
worktree: updated.worktree ?? null,
|
||||
branch: updated.branch ?? null,
|
||||
checkedOutBy: updated.checkedOutBy ?? null,
|
||||
executionStartedAt: updated.executionStartedAt ?? null,
|
||||
taskDoneRetryCount: updated.taskDoneRetryCount ?? null,
|
||||
worktreeSessionRetryCount: updated.worktreeSessionRetryCount ?? null,
|
||||
sessionFile: updated.sessionFile ?? null,
|
||||
};
|
||||
await scopedStore.updateTask(req.params.id, RESET_DRIFT_CORRECTION_FIELDS);
|
||||
await scopedStore.logEntry(
|
||||
req.params.id,
|
||||
"Auto-corrected reset drift after moveTask — normalized task back to todo with cleared worktree/branch bindings",
|
||||
JSON.stringify(offendingSnapshot),
|
||||
);
|
||||
await emitResetDriftAudit(scopedStore, req.params.id, offendingSnapshot);
|
||||
updated = await scopedStore.getTask(req.params.id);
|
||||
if (!updated) {
|
||||
throw notFound(`Task ${req.params.id} not found after reset drift correction`);
|
||||
}
|
||||
}
|
||||
|
||||
if (updated.column !== "todo" || (updated.worktree ?? null) !== null || (updated.branch ?? null) !== null) {
|
||||
throw conflict(
|
||||
`Reset refused to return task ${req.params.id} in limbo state (${updated.column}, branch=${updated.branch ?? "null"}, worktree=${updated.worktree ?? "null"})`,
|
||||
);
|
||||
}
|
||||
|
||||
res.json(updated);
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof ApiError) {
|
||||
@@ -2760,7 +2866,7 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork
|
||||
// Delete task
|
||||
router.delete("/tasks/:id", async (req, res) => {
|
||||
try {
|
||||
const { store: scopedStore } = await getProjectContext(req);
|
||||
const { store: scopedStore, engine } = await getProjectContext(req);
|
||||
const removeDependencyReferences = req.query.removeDependencyReferences === "1"
|
||||
|| req.query.removeDependencyReferences === "true";
|
||||
const removeLineageReferences = req.query.removeLineageReferences === "1"
|
||||
@@ -2783,6 +2889,7 @@ export function registerTaskWorkflowRoutes(ctx: ApiRoutesContext, deps: TaskWork
|
||||
runId: `synthetic-dashboard-delete-${req.params.id}-${Date.now()}`,
|
||||
},
|
||||
});
|
||||
await releaseExecutionAgentBindings(engine, req.params.id);
|
||||
res.json(task);
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof ApiError) {
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { execSync } from "node:child_process";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { logger } = vi.hoisted(() => ({ logger: { log: vi.fn(), warn: vi.fn(), error: vi.fn() } }));
|
||||
vi.mock("../../logger.js", () => ({ createLogger: vi.fn(() => logger) }));
|
||||
|
||||
import { TaskStore } from "@fusion/core";
|
||||
import { SelfHealingManager } from "../../self-healing.js";
|
||||
|
||||
function git(cwd: string, command: string): string {
|
||||
return execSync(`git ${command}`, { cwd, encoding: "utf8" }).trim();
|
||||
}
|
||||
|
||||
describe("FN-5219 reliability interactions: in-progress limbo recovery", () => {
|
||||
let rootDir = "";
|
||||
let store: TaskStore;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-05-20T12:00:00.000Z"));
|
||||
rootDir = mkdtempSync(join(tmpdir(), "fn-5219-reliability-"));
|
||||
git(rootDir, "init -b main");
|
||||
git(rootDir, "config user.name 'Fusion'");
|
||||
git(rootDir, "config user.email 'hi@runfusion.ai'");
|
||||
writeFileSync(join(rootDir, "README.md"), "root\n");
|
||||
git(rootDir, "add README.md");
|
||||
git(rootDir, "commit -m 'init'");
|
||||
mkdirSync(join(rootDir, ".worktrees"), { recursive: true });
|
||||
store = new TaskStore(rootDir, undefined, { inMemoryDb: false });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
try { store?.close(); } catch {}
|
||||
if (rootDir) rmSync(rootDir, { recursive: true, force: true });
|
||||
vi.useRealTimers();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
async function createInProgressTask(title: string) {
|
||||
const task = await store.createTask({ title, description: title });
|
||||
await store.moveTask(task.id, "todo");
|
||||
await store.moveTask(task.id, "in-progress");
|
||||
return task.id;
|
||||
}
|
||||
|
||||
it("FN-5149: reset twice → stranded in-progress with missing worktree → recovered to todo", async () => {
|
||||
const mockStore = Object.assign(new EventEmitter(), {
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: false, enginePaused: false }),
|
||||
listTasks: vi.fn()
|
||||
.mockResolvedValueOnce([
|
||||
{
|
||||
id: "FN-5149",
|
||||
column: "in-progress",
|
||||
paused: false,
|
||||
branch: null,
|
||||
worktree: join(rootDir, ".worktrees", "fn-5149-missing"),
|
||||
updatedAt: "2026-05-20T12:00:00.000Z",
|
||||
steps: [{ status: "pending" }],
|
||||
log: [],
|
||||
},
|
||||
])
|
||||
.mockResolvedValueOnce([]),
|
||||
updateTask: vi.fn().mockResolvedValue({}),
|
||||
logEntry: vi.fn().mockResolvedValue(undefined),
|
||||
moveTask: vi.fn().mockResolvedValue(undefined),
|
||||
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
|
||||
getRootDir: vi.fn().mockReturnValue(rootDir),
|
||||
}) as unknown as TaskStore;
|
||||
vi.setSystemTime(new Date("2026-05-20T12:02:00.000Z"));
|
||||
|
||||
const manager = new SelfHealingManager(mockStore, {
|
||||
rootDir,
|
||||
getExecutingTaskIds: () => new Set<string>(),
|
||||
});
|
||||
|
||||
const first = await manager.recoverInProgressLimbo();
|
||||
const second = await manager.recoverOrphanedExecutions();
|
||||
|
||||
expect(first).toBe(1);
|
||||
expect(second).toBe(0);
|
||||
expect(mockStore.moveTask).toHaveBeenCalledWith("FN-5149", "todo", { preserveProgress: true });
|
||||
expect(mockStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
mutationType: "task:auto-recover-in-progress-limbo",
|
||||
target: "FN-5149",
|
||||
}));
|
||||
});
|
||||
|
||||
it("reconcile-task-worktree-metadata runs first so a live rebindable worktree wins", async () => {
|
||||
const id = await createInProgressTask("metadata rebind wins");
|
||||
const liveWorktree = join(rootDir, ".worktrees", `${id.toLowerCase()}-live`);
|
||||
const branch = `fusion/${id.toLowerCase()}`;
|
||||
git(rootDir, `worktree add -b ${branch} ${liveWorktree}`);
|
||||
writeFileSync(join(liveWorktree, `${id}.txt`), `${id}\n`);
|
||||
git(liveWorktree, `add ${id}.txt`);
|
||||
git(liveWorktree, `commit -m 'task work'`);
|
||||
git(rootDir, "checkout main");
|
||||
|
||||
await store.updateTask(id, {
|
||||
branch: null,
|
||||
worktree: join(rootDir, ".worktrees", `${id.toLowerCase()}-missing`),
|
||||
steps: [{ name: "step", status: "pending" }],
|
||||
});
|
||||
vi.setSystemTime(new Date("2026-05-20T12:02:00.000Z"));
|
||||
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir,
|
||||
getExecutingTaskIds: () => new Set<string>(),
|
||||
});
|
||||
|
||||
const repaired = await manager.reconcileTaskWorktreeMetadata({ includeTaskIds: new Set([id]) });
|
||||
const recovered = await manager.recoverInProgressLimbo();
|
||||
const updated = await store.getTask(id);
|
||||
|
||||
expect(repaired).toBe(1);
|
||||
expect(recovered).toBe(0);
|
||||
expect(updated?.column).toBe("in-progress");
|
||||
expect(updated?.branch).toBe(branch);
|
||||
expect(updated?.worktree?.endsWith(`${id.toLowerCase()}-live`)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps in-review missing-worktree failures on the review-specific recovery path", async () => {
|
||||
const id = await createInProgressTask("review failure disjoint");
|
||||
await store.moveTask(id, "in-review");
|
||||
await store.updateTask(id, {
|
||||
status: "failed",
|
||||
error: `Refusing to start coding agent in missing worktree: ${join(rootDir, ".worktrees", "missing-review")}`,
|
||||
branch: `fusion/${id.toLowerCase()}`,
|
||||
worktree: join(rootDir, ".worktrees", "missing-review-stale"),
|
||||
steps: [{ name: "step", status: "done" }, { name: "next", status: "pending" }],
|
||||
});
|
||||
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir,
|
||||
getExecutingTaskIds: () => new Set<string>(),
|
||||
});
|
||||
|
||||
const limboRecovered = await manager.recoverInProgressLimbo();
|
||||
const reviewRecovered = await manager.recoverMissingWorktreeReviewFailures();
|
||||
const updated = await store.getTask(id);
|
||||
|
||||
expect(limboRecovered).toBe(0);
|
||||
expect(reviewRecovered).toBe(1);
|
||||
expect(updated?.column).toBe("todo");
|
||||
});
|
||||
|
||||
it("skips limbo recovery while the executor still claims the task id", async () => {
|
||||
const id = await createInProgressTask("executor claim wins");
|
||||
await store.updateTask(id, {
|
||||
branch: null,
|
||||
worktree: join(rootDir, ".worktrees", `${id.toLowerCase()}-missing`),
|
||||
steps: [{ name: "step", status: "pending" }],
|
||||
});
|
||||
vi.setSystemTime(new Date("2026-05-20T12:02:00.000Z"));
|
||||
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir,
|
||||
getExecutingTaskIds: () => new Set<string>([id]),
|
||||
});
|
||||
|
||||
const recovered = await manager.recoverInProgressLimbo();
|
||||
const updated = await store.getTask(id);
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(updated?.column).toBe("in-progress");
|
||||
expect(updated?.worktree).toContain("missing");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,212 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { EventEmitter } from "node:events";
|
||||
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:fs")>();
|
||||
return {
|
||||
...actual,
|
||||
existsSync: vi.fn(actual.existsSync),
|
||||
};
|
||||
});
|
||||
|
||||
const { logger } = vi.hoisted(() => ({ logger: { log: vi.fn(), warn: vi.fn(), error: vi.fn() } }));
|
||||
vi.mock("../logger.js", () => ({
|
||||
createLogger: vi.fn(() => logger),
|
||||
schedulerLog: { log: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
}));
|
||||
|
||||
vi.mock("../worktree-pool.js", () => ({
|
||||
WorktreePool: vi.fn(),
|
||||
RemovalReason: {
|
||||
HardCancel: "hard-cancel",
|
||||
ExecutorTransientRetry: "executor-transient-retry",
|
||||
ExecutorStuckKilled: "executor-stuck-killed",
|
||||
ExecutorDispose: "executor-dispose",
|
||||
StepSessionCleanup: "step-session-cleanup",
|
||||
MergerPostMerge: "merger-post-merge",
|
||||
MergerCleanup: "merger-cleanup",
|
||||
SelfHealingReclaim: "self-healing-reclaim",
|
||||
SelfHealingStaleActiveBranch: "self-healing-stale-active-branch",
|
||||
SelfHealingBranchConflict: "self-healing-branch-conflict",
|
||||
SelfHealingOrphanRescue: "self-healing-orphan-rescue",
|
||||
SelfHealingIdleSweep: "self-healing-idle-sweep",
|
||||
PoolPrune: "pool-prune",
|
||||
},
|
||||
scanIdleWorktrees: vi.fn().mockResolvedValue([]),
|
||||
cleanupOrphanedWorktrees: vi.fn().mockResolvedValue(0),
|
||||
scanOrphanedBranches: vi.fn().mockResolvedValue([]),
|
||||
isUsableTaskWorktree: vi.fn().mockResolvedValue(true),
|
||||
removeWorktree: vi.fn().mockResolvedValue(undefined),
|
||||
resolveWorktreeBackend: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../merger.js", () => ({ classifyOwnedLandedEvidence: vi.fn() }));
|
||||
|
||||
import { existsSync } from "node:fs";
|
||||
import { SelfHealingManager } from "../self-healing.js";
|
||||
import type { Settings, Task, TaskStore } from "@fusion/core";
|
||||
|
||||
function createMockStore(overrides: Record<string, unknown> = {}): TaskStore & EventEmitter {
|
||||
const emitter = new EventEmitter();
|
||||
return Object.assign(emitter, {
|
||||
getSettings: vi.fn().mockResolvedValue({
|
||||
autoMerge: true,
|
||||
globalPause: false,
|
||||
enginePaused: false,
|
||||
maintenanceIntervalMs: 0,
|
||||
} as unknown as Settings),
|
||||
listTasks: vi.fn().mockResolvedValue([]),
|
||||
updateTask: vi.fn().mockResolvedValue({} as Task),
|
||||
logEntry: vi.fn().mockResolvedValue(undefined),
|
||||
moveTask: vi.fn().mockResolvedValue(undefined),
|
||||
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
|
||||
getRootDir: vi.fn().mockReturnValue("/tmp/test-project"),
|
||||
...overrides,
|
||||
}) as unknown as TaskStore & EventEmitter;
|
||||
}
|
||||
|
||||
describe("recoverInProgressLimbo", () => {
|
||||
let store: TaskStore & EventEmitter;
|
||||
|
||||
const baseTask = {
|
||||
id: "FN-5149",
|
||||
column: "in-progress",
|
||||
paused: false,
|
||||
branch: null,
|
||||
worktree: "/tmp/test-project/.worktrees/missing-fn-5149",
|
||||
checkedOutBy: "agent-1",
|
||||
executionStartedAt: "2026-05-20T12:00:00.000Z",
|
||||
updatedAt: "2026-05-20T12:00:00.000Z",
|
||||
steps: [{ status: "pending" }, { status: "pending" }],
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-05-20T12:05:00.000Z"));
|
||||
vi.mocked(existsSync).mockImplementation(() => false);
|
||||
store = createMockStore();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("requeues FN-5149-signature limbo tasks to todo with audit telemetry", async () => {
|
||||
const reconcileLeaseRow = vi.fn().mockResolvedValue(undefined);
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([baseTask]);
|
||||
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
getExecutingTaskIds: () => new Set<string>(),
|
||||
leaseManager: {
|
||||
recoverAbandonedLease: vi.fn().mockResolvedValue(false),
|
||||
reconcileLeaseRow,
|
||||
} as any,
|
||||
});
|
||||
|
||||
const recovered = await manager.recoverInProgressLimbo();
|
||||
|
||||
expect(recovered).toBe(1);
|
||||
expect(reconcileLeaseRow).toHaveBeenCalledWith("FN-5149");
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-5149", expect.objectContaining({
|
||||
worktree: null,
|
||||
branch: null,
|
||||
status: null,
|
||||
error: null,
|
||||
checkedOutBy: null,
|
||||
executionStartedAt: null,
|
||||
worktreeSessionRetryCount: null,
|
||||
taskDoneRetryCount: null,
|
||||
sessionFile: null,
|
||||
}));
|
||||
expect(store.moveTask).toHaveBeenCalledWith("FN-5149", "todo", { preserveProgress: true });
|
||||
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||
domain: "database",
|
||||
mutationType: "task:auto-recover-in-progress-limbo",
|
||||
target: "FN-5149",
|
||||
}));
|
||||
});
|
||||
|
||||
it("skips tasks whose worktree still exists on disk", async () => {
|
||||
vi.mocked(existsSync).mockImplementation((path) => path === baseTask.worktree);
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([baseTask]);
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
getExecutingTaskIds: () => new Set<string>(),
|
||||
});
|
||||
|
||||
const recovered = await manager.recoverInProgressLimbo();
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.updateTask).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips tasks whose branch is still set", async () => {
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([{ ...baseTask, branch: "fusion/fn-5149" }]);
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
getExecutingTaskIds: () => new Set<string>(),
|
||||
});
|
||||
|
||||
const recovered = await manager.recoverInProgressLimbo();
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips tasks currently claimed by the executor", async () => {
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([baseTask]);
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
getExecutingTaskIds: () => new Set<string>(["FN-5149"]),
|
||||
});
|
||||
|
||||
const recovered = await manager.recoverInProgressLimbo();
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips paused tasks", async () => {
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([{ ...baseTask, paused: true }]);
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
getExecutingTaskIds: () => new Set<string>(),
|
||||
});
|
||||
|
||||
const recovered = await manager.recoverInProgressLimbo();
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips tasks still within the grace window", async () => {
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([{ ...baseTask, updatedAt: "2026-05-20T12:04:30.000Z" }]);
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
getExecutingTaskIds: () => new Set<string>(),
|
||||
});
|
||||
|
||||
const recovered = await manager.recoverInProgressLimbo();
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.moveTask).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips entirely when the engine is globally paused", async () => {
|
||||
store = createMockStore({
|
||||
getSettings: vi.fn().mockResolvedValue({ autoMerge: true, globalPause: true, enginePaused: false } as unknown as Settings),
|
||||
});
|
||||
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([baseTask]);
|
||||
const manager = new SelfHealingManager(store, {
|
||||
rootDir: "/tmp/test-project",
|
||||
getExecutingTaskIds: () => new Set<string>(),
|
||||
});
|
||||
|
||||
const recovered = await manager.recoverInProgressLimbo();
|
||||
|
||||
expect(recovered).toBe(0);
|
||||
expect(store.listTasks).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -235,6 +235,7 @@ export type DatabaseMutationType =
|
||||
| "task:auto-recover-completion-handoff-limbo"
|
||||
| "task:auto-recover-completion-handoff-limbo-exhausted"
|
||||
| "task:auto-recover-worktree-session-exhausted"
|
||||
| "task:auto-recover-in-progress-limbo"
|
||||
/** Metadata: { taskId: string; ignoredStepUpdateCount: number; stuckKillStreak: number; lastReason: "no-progress-churn" } */
|
||||
| "task:stuck-no-progress-churn-terminalized"
|
||||
| "task:auto-recover-starved-refinement"
|
||||
|
||||
@@ -678,6 +678,7 @@ export class SelfHealingManager {
|
||||
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts().then(() => undefined) },
|
||||
// FN-4962 ordering invariant: metadata reconcile must run before stale-active reclaim.
|
||||
{ name: "reconcile-task-worktree-metadata", fn: () => this.reconcileTaskWorktreeMetadata().then(() => undefined) },
|
||||
{ name: "recover-in-progress-limbo", fn: () => this.recoverInProgressLimbo().then(() => undefined) },
|
||||
{ name: "reconcile-in-review-branch-rebind", fn: () => this.reconcileInReviewBranchRebind().then(() => undefined) },
|
||||
{ name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches().then(() => undefined) },
|
||||
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls().then(() => undefined) },
|
||||
@@ -1314,6 +1315,7 @@ export class SelfHealingManager {
|
||||
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts() },
|
||||
// FN-4962 ordering invariant: metadata reconcile must run before stale-active reclaim.
|
||||
{ name: "reconcile-task-worktree-metadata", fn: () => this.reconcileTaskWorktreeMetadata() },
|
||||
{ name: "recover-in-progress-limbo", fn: () => this.recoverInProgressLimbo() },
|
||||
{ name: "reconcile-in-review-branch-rebind", fn: () => this.reconcileInReviewBranchRebind().then(() => undefined) },
|
||||
{ name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches() },
|
||||
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls() },
|
||||
@@ -5683,6 +5685,112 @@ export class SelfHealingManager {
|
||||
* established, typically when the scheduler reserved a worktree path but the
|
||||
* executor never materialized it or crashed before tracking the run.
|
||||
*/
|
||||
async recoverInProgressLimbo(): Promise<number> {
|
||||
const settings = await this.store.getSettings();
|
||||
if (settings.globalPause || settings.enginePaused) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
const tasks = await this.store.listTasks({ column: "in-progress", slim: true });
|
||||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||||
const now = Date.now();
|
||||
|
||||
const stranded = tasks.filter((task) => {
|
||||
if (task.column !== "in-progress" || task.paused || executingIds.has(task.id)) {
|
||||
return false;
|
||||
}
|
||||
const hasMissingWorktreePath = typeof task.worktree === "string" && task.worktree.length > 0 && !existsSync(task.worktree);
|
||||
const hasNoWorktreePath = !task.worktree;
|
||||
if (!hasMissingWorktreePath && !hasNoWorktreePath) {
|
||||
return false;
|
||||
}
|
||||
if (typeof task.branch === "string" && task.branch.trim().length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (task.steps.some((step) => step.status !== "pending")) {
|
||||
return false;
|
||||
}
|
||||
const staleness = now - new Date(task.updatedAt).getTime();
|
||||
return staleness >= ORPHANED_EXECUTION_RECOVERY_GRACE_MS;
|
||||
});
|
||||
|
||||
const describeWorktreeState = (task: Task): string => task.worktree ? "missing worktree path" : "cleared worktree metadata";
|
||||
|
||||
if (stranded.length === 0) return 0;
|
||||
|
||||
log.warn(`Found ${stranded.length} in-progress limbo task(s) with missing/cleared worktree + null branch`);
|
||||
|
||||
let recovered = 0;
|
||||
for (const task of stranded) {
|
||||
try {
|
||||
if (this.options.leaseManager && task.checkedOutBy) {
|
||||
await this.options.leaseManager.recoverAbandonedLease(
|
||||
task.id,
|
||||
`in-progress limbo: ${describeWorktreeState(task)} + null branch`,
|
||||
{ preserveProgress: true },
|
||||
);
|
||||
await this.options.leaseManager.reconcileLeaseRow(task.id);
|
||||
}
|
||||
|
||||
const stepStatuses = task.steps.map((step) => step.status);
|
||||
const ageMs = Math.max(0, now - new Date(task.updatedAt).getTime());
|
||||
await this.store.updateTask(task.id, {
|
||||
status: null,
|
||||
error: null,
|
||||
worktree: null,
|
||||
branch: null,
|
||||
checkedOutBy: null,
|
||||
executionStartedAt: null,
|
||||
worktreeSessionRetryCount: null,
|
||||
taskDoneRetryCount: null,
|
||||
sessionFile: null,
|
||||
});
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Auto-recovered in-progress limbo — ${describeWorktreeState(task)}/null branch with no step progress, moved back to todo`,
|
||||
JSON.stringify({
|
||||
priorWorktree: task.worktree ?? null,
|
||||
priorBranch: task.branch ?? null,
|
||||
ageMs,
|
||||
stepStatuses,
|
||||
}),
|
||||
);
|
||||
await createRunAuditor(this.store, {
|
||||
runId: generateSyntheticRunId("self-healing-in-progress-limbo", task.id),
|
||||
agentId: "self-healing",
|
||||
taskId: task.id,
|
||||
taskLineageId: task.lineageId,
|
||||
phase: "recover-in-progress-limbo",
|
||||
}).database({
|
||||
type: "task:auto-recover-in-progress-limbo",
|
||||
target: task.id,
|
||||
metadata: {
|
||||
priorWorktree: task.worktree ?? null,
|
||||
priorBranch: task.branch ?? null,
|
||||
ageMs,
|
||||
stepStatuses,
|
||||
},
|
||||
});
|
||||
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||||
recovered++;
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
log.error(`Failed to recover in-progress limbo task ${task.id}: ${errorMessage}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (recovered > 0) {
|
||||
log.log(`Recovered ${recovered} in-progress limbo task(s) → todo`);
|
||||
}
|
||||
return recovered;
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
log.error(`In-progress limbo recovery failed: ${errorMessage}`);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async recoverOrphanedExecutions(): Promise<number> {
|
||||
try {
|
||||
const tasks = await this.store.listTasks({ column: "in-progress", slim: true });
|
||||
|
||||
Reference in New Issue
Block a user