diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index d1ac81520..f2a4ef96c 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -4017,7 +4017,7 @@ export class TaskStore extends EventEmitter { async updateTask( id: string, - updates: { title?: string; description?: string; priority?: TaskPriority | null; prompt?: string; worktree?: string | null; status?: string | null; dependencies?: string[]; steps?: import("./types.js").TaskStep[]; currentStep?: number; blockedBy?: string | null; overlapBlockedBy?: string | null; assignedAgentId?: string | null; pausedByAgentId?: string | null; pausedReason?: string | null; tokenBudgetSoftAlertedAt?: string | null; tokenBudgetHardAlertedAt?: string | null; tokenBudgetOverride?: import("./types.js").TaskTokenBudgetOverride | null; dispatchStormCount?: number | null; lastDispatchAt?: string | null; assigneeUserId?: string | null; scopeOverride?: boolean | null; scopeOverrideReason?: string | null; nodeId?: string | null; effectiveNodeId?: string | null; effectiveNodeSource?: string | null; checkedOutBy?: string | null; checkedOutAt?: string | null; checkoutNodeId?: string | null; checkoutRunId?: string | null; checkoutLeaseRenewedAt?: string | null; checkoutLeaseEpoch?: number | null; paused?: boolean; baseBranch?: string | null; branch?: string | null; executionStartBranch?: string | null; baseCommitSha?: string | null; size?: "S" | "M" | "L"; reviewLevel?: number; executionMode?: import("./types.js").ExecutionMode | null; mergeRetries?: number; workflowStepRetries?: number; stuckKillCount?: number | null; postReviewFixCount?: number | null; recoveryRetryCount?: number | null; taskDoneRetryCount?: number | null; worktreeSessionRetryCount?: number | null; verificationFailureCount?: number | null; mergeConflictBounceCount?: number | null; mergeAuditBounceCount?: number | null; branchConflictRecoveryCount?: number | null; reviewerContextRetryCount?: number | null; reviewerFallbackRetryCount?: number | null; nextRecoveryAt?: string | null; enabledWorkflowSteps?: string[]; noCommitsExpected?: boolean | null; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; planningModelProvider?: string | null; planningModelId?: string | null; thinkingLevel?: string | null; error?: string | null; summary?: string | null; sessionFile?: string | null; firstExecutionAt?: string | null; cumulativeActiveMs?: number | null; executionStartedAt?: string | null; executionCompletedAt?: string | null; review?: import("./types.js").TaskReview | null; reviewState?: import("./types.js").TaskReviewState | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; mergeDetails?: import("./types.js").MergeDetails | null; sourceIssue?: import("./types.js").TaskSourceIssue | null; githubTracking?: import("./types.js").TaskGithubTracking | null; tokenUsage?: import("./types.js").TaskTokenUsage | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null }, + updates: { title?: string; description?: string; priority?: TaskPriority | null; prompt?: string; worktree?: string | null; status?: string | null; dependencies?: string[]; steps?: import("./types.js").TaskStep[]; currentStep?: number; blockedBy?: string | null; overlapBlockedBy?: string | null; assignedAgentId?: string | null; pausedByAgentId?: string | null; pausedReason?: string | null; tokenBudgetSoftAlertedAt?: string | null; worktrunkFallbackAlertedAt?: string | null; worktrunkFailure?: import("./types.js").Task["worktrunkFailure"] | null; tokenBudgetHardAlertedAt?: string | null; tokenBudgetOverride?: import("./types.js").TaskTokenBudgetOverride | null; dispatchStormCount?: number | null; lastDispatchAt?: string | null; assigneeUserId?: string | null; scopeOverride?: boolean | null; scopeOverrideReason?: string | null; nodeId?: string | null; effectiveNodeId?: string | null; effectiveNodeSource?: string | null; checkedOutBy?: string | null; checkedOutAt?: string | null; checkoutNodeId?: string | null; checkoutRunId?: string | null; checkoutLeaseRenewedAt?: string | null; checkoutLeaseEpoch?: number | null; paused?: boolean; baseBranch?: string | null; branch?: string | null; executionStartBranch?: string | null; baseCommitSha?: string | null; size?: "S" | "M" | "L"; reviewLevel?: number; executionMode?: import("./types.js").ExecutionMode | null; mergeRetries?: number; workflowStepRetries?: number; stuckKillCount?: number | null; postReviewFixCount?: number | null; recoveryRetryCount?: number | null; taskDoneRetryCount?: number | null; worktreeSessionRetryCount?: number | null; verificationFailureCount?: number | null; mergeConflictBounceCount?: number | null; mergeAuditBounceCount?: number | null; branchConflictRecoveryCount?: number | null; reviewerContextRetryCount?: number | null; reviewerFallbackRetryCount?: number | null; nextRecoveryAt?: string | null; enabledWorkflowSteps?: string[]; noCommitsExpected?: boolean | null; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; planningModelProvider?: string | null; planningModelId?: string | null; thinkingLevel?: string | null; error?: string | null; summary?: string | null; sessionFile?: string | null; firstExecutionAt?: string | null; cumulativeActiveMs?: number | null; executionStartedAt?: string | null; executionCompletedAt?: string | null; review?: import("./types.js").TaskReview | null; reviewState?: import("./types.js").TaskReviewState | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; mergeDetails?: import("./types.js").MergeDetails | null; sourceIssue?: import("./types.js").TaskSourceIssue | null; githubTracking?: import("./types.js").TaskGithubTracking | null; tokenUsage?: import("./types.js").TaskTokenUsage | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null }, runContext?: RunMutationContext, ): Promise { return this.withTaskLock(id, async () => { @@ -4160,6 +4160,16 @@ export class TaskStore extends EventEmitter { } else if (updates.tokenBudgetSoftAlertedAt !== undefined) { task.tokenBudgetSoftAlertedAt = updates.tokenBudgetSoftAlertedAt; } + if (updates.worktrunkFallbackAlertedAt === null) { + task.worktrunkFallbackAlertedAt = undefined; + } else if (updates.worktrunkFallbackAlertedAt !== undefined) { + task.worktrunkFallbackAlertedAt = updates.worktrunkFallbackAlertedAt; + } + if (updates.worktrunkFailure === null) { + task.worktrunkFailure = undefined; + } else if (updates.worktrunkFailure !== undefined) { + task.worktrunkFailure = updates.worktrunkFailure; + } if (updates.tokenBudgetHardAlertedAt === null) { task.tokenBudgetHardAlertedAt = undefined; } else if (updates.tokenBudgetHardAlertedAt !== undefined) { diff --git a/packages/engine/src/__tests__/worktrunk-failure-handler.test.ts b/packages/engine/src/__tests__/worktrunk-failure-handler.test.ts new file mode 100644 index 000000000..3a3e382b9 --- /dev/null +++ b/packages/engine/src/__tests__/worktrunk-failure-handler.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it, vi } from "vitest"; +import type { Task } from "@fusion/core"; +import { + handleWorktrunkOperationFailure, + type WorktrunkOperationFailure, + type WorktreeOperationResult, +} from "../worktrunk-failure-handler.js"; +import { + WorktrunkBinaryUnavailableError, + WorktrunkInstallDeniedError, + WorktrunkInstallFailedError, +} from "../worktrunk-installer.js"; + +function makeTask(overrides: Partial = {}): Task { + return { + id: "FN-4625", + description: "task", + column: "in-progress", + dependencies: [], + steps: [], + currentStep: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + ...overrides, + } as Task; +} + +function makeFailure(cause: Error, stderr = "boom", exitCode: number | null = 1): WorktrunkOperationFailure { + return { op: "create", cause, stderr, exitCode }; +} + +describe("handleWorktrunkOperationFailure", () => { + it("fail mode pauses, persists details, audits, and rethrows", async () => { + const pauseTask = vi.fn().mockResolvedValue(undefined); + const updateTask = vi.fn().mockResolvedValue(undefined); + const git = vi.fn().mockResolvedValue(undefined); + const notify = vi.fn(); + const cause = new Error("create failed"); + + await expect(handleWorktrunkOperationFailure({ + failure: makeFailure(cause), + task: makeTask(), + settings: { enabled: true, onFailure: "fail" }, + store: { pauseTask, updateTask } as any, + runContext: { runId: "run-1", agentId: "agent-1" }, + runAudit: { git }, + notify, + })).rejects.toThrow("create failed"); + + expect(pauseTask).toHaveBeenCalledWith("FN-4625", true, expect.any(Object)); + expect(updateTask).toHaveBeenCalledWith( + "FN-4625", + expect.objectContaining({ + pausedReason: "worktrunk_operation_failed", + worktrunkFailure: expect.objectContaining({ op: "create", stderr: "boom", exitCode: 1 }), + }), + expect.any(Object), + ); + expect(git).toHaveBeenCalledWith(expect.objectContaining({ type: "worktree:worktrunk-failure" })); + expect(notify).not.toHaveBeenCalled(); + }); + + it("fallback-native calls nativeFallback, notifies once, audits, and returns result", async () => { + const pauseTask = vi.fn(); + const updateTask = vi.fn().mockResolvedValue(undefined); + const git = vi.fn().mockResolvedValue(undefined); + const notify = vi.fn().mockResolvedValue(undefined); + const result: WorktreeOperationResult = { path: "/tmp/wt", branch: "fusion/fn-4625" }; + const nativeFallback = vi.fn().mockResolvedValue(result); + + const disposition = await handleWorktrunkOperationFailure({ + failure: makeFailure(new Error("create failed")), + task: makeTask(), + settings: { enabled: true, onFailure: "fallback-native" }, + store: { pauseTask, updateTask } as any, + runAudit: { git }, + notify, + nativeFallback, + }); + + expect(disposition).toEqual({ kind: "fallback-native", result, alerted: true }); + expect(updateTask).toHaveBeenCalledWith("FN-4625", expect.objectContaining({ worktrunkFallbackAlertedAt: expect.any(String) }), undefined); + expect(notify).toHaveBeenCalledTimes(1); + expect(git).toHaveBeenCalledWith(expect.objectContaining({ type: "worktree:worktrunk-fallback-native" })); + expect(pauseTask).not.toHaveBeenCalled(); + }); + + it("fallback-native does not re-alert when already alerted", async () => { + const updateTask = vi.fn().mockResolvedValue(undefined); + const notify = vi.fn(); + const nativeFallback = vi.fn().mockResolvedValue(undefined); + + const disposition = await handleWorktrunkOperationFailure({ + failure: makeFailure(new Error("create failed")), + task: makeTask({ worktrunkFallbackAlertedAt: "2026-05-15T00:00:00.000Z" }), + settings: { enabled: true, onFailure: "fallback-native" }, + store: { pauseTask: vi.fn(), updateTask } as any, + notify, + nativeFallback, + }); + + expect(disposition).toEqual({ kind: "fallback-native", result: undefined, alerted: false }); + expect(notify).not.toHaveBeenCalled(); + expect(updateTask).not.toHaveBeenCalledWith("FN-4625", expect.objectContaining({ worktrunkFallbackAlertedAt: expect.any(String) }), undefined); + }); + + it("fallback-native without nativeFallback degrades to fail-hard", async () => { + const pauseTask = vi.fn().mockResolvedValue(undefined); + const updateTask = vi.fn().mockResolvedValue(undefined); + const cause = new Error("create failed"); + + await expect(handleWorktrunkOperationFailure({ + failure: makeFailure(cause), + task: makeTask(), + settings: { enabled: true, onFailure: "fallback-native" }, + store: { pauseTask, updateTask } as any, + notify: vi.fn(), + })).rejects.toThrow("create failed"); + + expect(pauseTask).toHaveBeenCalledTimes(1); + expect(updateTask).toHaveBeenCalledWith("FN-4625", expect.objectContaining({ pausedReason: "worktrunk_operation_failed" }), undefined); + }); + + it("native fallback errors are rethrown", async () => { + const fallbackError = new Error("native failed"); + await expect(handleWorktrunkOperationFailure({ + failure: makeFailure(new Error("create failed")), + task: makeTask(), + settings: { enabled: true, onFailure: "fallback-native" }, + store: { pauseTask: vi.fn(), updateTask: vi.fn().mockResolvedValue(undefined) } as any, + notify: vi.fn(), + nativeFallback: vi.fn().mockRejectedValue(fallbackError), + })).rejects.toThrow("native failed"); + }); + + it("truncates stderr preview to 4KB in run-audit metadata", async () => { + const git = vi.fn().mockResolvedValue(undefined); + const longStderr = "x".repeat(5000); + await expect(handleWorktrunkOperationFailure({ + failure: makeFailure(new Error("create failed"), longStderr), + task: makeTask(), + settings: { enabled: true, onFailure: "fail" }, + store: { pauseTask: vi.fn().mockResolvedValue(undefined), updateTask: vi.fn().mockResolvedValue(undefined) } as any, + runAudit: { git }, + notify: vi.fn(), + })).rejects.toThrow(); + + const call = git.mock.calls.find((entry) => entry[0]?.type === "worktree:worktrunk-failure"); + expect(call?.[0]?.metadata?.stderrPreview.length).toBe(4097); + expect(call?.[0]?.metadata?.stderrPreview.endsWith("…")).toBe(true); + }); + + it.each([ + new WorktrunkBinaryUnavailableError("missing"), + new WorktrunkInstallFailedError("install failed", { stage: "release" }), + new WorktrunkInstallDeniedError("denied"), + ])("handles installer error class %s in fail-hard mode", async (cause) => { + await expect(handleWorktrunkOperationFailure({ + failure: makeFailure(cause, cause.message), + task: makeTask(), + settings: { enabled: true, onFailure: "fail" }, + store: { pauseTask: vi.fn().mockResolvedValue(undefined), updateTask: vi.fn().mockResolvedValue(undefined) } as any, + notify: vi.fn(), + })).rejects.toBe(cause); + }); +}); diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index ff2b90c0f..314a04858 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -128,6 +128,15 @@ export { WORKTRUNK_DOWNLOAD_MAX_BYTES, WORKTRUNK_CARGO_TIMEOUT_MS, } from "./worktrunk-installer.js"; +export { + handleWorktrunkOperationFailure, + truncateWorktrunkStderr, + type WorktreeOperationResult, + type WorktrunkDisposition, + type WorktrunkFailureNotification, + type WorktrunkOpName, + type WorktrunkOperationFailure, +} from "./worktrunk-failure-handler.js"; export { isUsageLimitError, UsageLimitPauser } from "./usage-limit-detector.js"; export { withRateLimitRetry } from "./rate-limit-retry.js"; export { ResearchOrchestrator, type ResearchOrchestratorOptions, type ResearchOrchestratorStatus, type ResearchOrchestratorStartOptions } from "./research-orchestrator.js"; diff --git a/packages/engine/src/run-audit.ts b/packages/engine/src/run-audit.ts index d662716cf..943bbf6cc 100644 --- a/packages/engine/src/run-audit.ts +++ b/packages/engine/src/run-audit.ts @@ -71,6 +71,8 @@ export type GitMutationType = | "worktree:worktrunk-sync" | "worktree:worktrunk-prune" | "worktree:worktrunk-fallback" + | "worktree:worktrunk-failure" + | "worktree:worktrunk-fallback-native" | "branch:create" | "branch:delete" | "branch:checkout"