FN-5697: retry transient auto-merge failures and fix migration versioning
Treat transient auto-merge/provider abort errors as bounded retries while preserving correct schema migration ordering. - Add transient merge retry handling with capped exponential backoff, queue re-enqueue, and exhaustion logging before failing tasks. - Extend task/core types and evaluator evidence plumbing for merge transient retry tracking and MergeTransientRetryExhausted visibility. - Add regression coverage for transient auto-merge retries and exhaustion behavior in merge error recovery tests. - Resolve migration collision by promoting workflow_steps.gateMode migration to version 77, shifting subsequent migrations, and bumping schema version to 95. - Add a changeset and architecture note documenting transient retry behavior. Files changed: .changeset/fn-5697-auto-merge-transient-retry.md | 5 ++ docs/architecture.md | 1 + packages/core/src/db.ts | 85 +++++++++++---------- packages/core/src/eval-types.ts | 1 + packages/core/src/store.ts | 19 +++-- packages/core/src/types.ts | 7 ++ packages/engine/src/__tests__/evaluator-evidence.test.ts | 1 + packages/engine/src/__tests__/merge-error-recovery.test.ts | 87 ++++++++++++++++++++++ packages/engine/src/evaluator-evidence.ts | 1 + packages/engine/src/project-engine.ts | 68 +++++++++++++++++ 10 files changed, 231 insertions(+), 44 deletions(-) Fusion-Task-Id: FN-5697 Fusion-Task-Lineage: c8be7374-7cb6-444e-9920-9227e05a43dc
This commit is contained in:
gsxdsm
5
.changeset/fn-5697-auto-merge-transient-retry.md
Normal file
5
.changeset/fn-5697-auto-merge-transient-retry.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
Auto-merge now treats transient provider/network failures during merge (for example "This operation was aborted", "socket hang up", and provider `server_error` payloads) as bounded retryable errors instead of immediate terminal failures. The engine re-enqueues affected in-review merges with exponential backoff for both direct and pull-request merge strategies, then parks the task as failed with explicit transient-retry exhaustion logs once the retry cap is reached.
|
||||
@@ -1725,6 +1725,7 @@ This section preserves the detailed lifecycle/self-healing contracts that were f
|
||||
- **PR-conflict reclaim wiring (FN-4763)**: GitHub PR refresh now persists normalized `prInfo.mergeable` conflict state and, when conflicting, funnels tasks into self-healing’s existing reclaim machinery (`reclaimPrConflictForTask` / `reclaim-pr-conflicts` stage) so branch-conflict handling stays centralized with existing `inspectBranchConflict` outcomes and unrecoverable pause semantics. PR refresh also captures `prInfo.conflictDiagnostics` (conflicting files + suggested local recovery commands) for dashboard surfacing.
|
||||
- **Worktrunk-managed lifecycles**: when `worktrunk.enabled`, self-healing defers prune/idle/worktree-cap sweeps to the worktrunk backend; branch-level stale/ conflict reclaim stays native. Orphan `fusion/*` branches are operator-managed via standard git tooling (no auto-rescue task filing).
|
||||
- **Post-finalize verification no-op (FN-4944)**: when auto-merge receives a delayed `VerificationError` after a task is already `done` with `mergeDetails.mergeConfirmed === true` (already-on-main fast-path), it must log one `[verification] ... no action` diagnostic and must not bounce the task back to `in-progress` / `merging-fix`. Defense-in-depth now re-checks the done+mergeConfirmed condition immediately before each verification-failure status write site, and emits `task:post-finalize-verification-no-op` database audit events with failure metadata for forensics.
|
||||
- **Transient auto-merge retry classification (FN-5697)**: non-conflict auto-merge errors now run through `isTransientError(...)` before terminal parking. Transient provider/network failures (for example `This operation was aborted`, `socket hang up`, and `server_error` payloads) are retried with bounded exponential backoff (`5s/10s/20s`) and `status=null` for both direct and pull-request merge strategies; once `MAX_AUTO_MERGE_TRANSIENT_RETRIES` is exhausted, tasks are parked `in-review/failed` with explicit transient-exhaustion logs.
|
||||
- **Worktree pool exclusivity (FN-4954)**: `WorktreePool.acquire(taskId)` / `release(path, taskId?)` track a `leased` map so every pooled path is either idle or leased, never both. Cross-task double-lease detection throws `PoolDoubleLeaseError` and emits `worktree:pool-double-lease-detected`; merger Step 8 now detaches HEAD and clears `task.worktree` / `task.branch` before releasing paths back to the pool.
|
||||
- **Stale registration recovery (FN-5056)**: `NativeWorktreeBackend.create` and `executor.tryCreateWorktree` detect `missing but already registered worktree` failures, run `git worktree prune` (plus `remove --force` / `add -f` fallbacks) before retrying, and emit `worktree:stale-registration-{detected,recovered,recovery-failed}` audit events.
|
||||
- **Raw worktree deletion must be paired with prune (FN-5058)**: any direct filesystem deletion of a worktree directory (`rm -rf` / `rmSync`) must be followed by best-effort `git worktree prune` via `pruneWorktreeAdminEntries` so `.git/worktrees/*` admin entries are not stranded in a missing-but-registered state (FN-5056 class).
|
||||
|
||||
@@ -149,7 +149,7 @@ export function probeFts5(db: DatabaseSync): boolean {
|
||||
|
||||
// ── Schema Definition ────────────────────────────────────────────────
|
||||
|
||||
const SCHEMA_VERSION = 94;
|
||||
const SCHEMA_VERSION = 95;
|
||||
|
||||
function normalizeTaskComments(
|
||||
steeringComments: SteeringComment[] | undefined,
|
||||
@@ -247,6 +247,7 @@ CREATE TABLE IF NOT EXISTS tasks (
|
||||
completionHandoffLimboRecoveryCount INTEGER DEFAULT 0,
|
||||
mergeConflictBounceCount INTEGER DEFAULT 0,
|
||||
mergeAuditBounceCount INTEGER DEFAULT 0,
|
||||
mergeTransientRetryCount INTEGER DEFAULT 0,
|
||||
nextRecoveryAt TEXT,
|
||||
error TEXT,
|
||||
summary TEXT,
|
||||
@@ -3357,6 +3358,12 @@ export class Database {
|
||||
|
||||
if (version < 75) {
|
||||
this.applyMigration(75, () => {
|
||||
this.addColumnIfMissing("tasks", "mergeTransientRetryCount", "INTEGER DEFAULT 0");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 76) {
|
||||
this.applyMigration(76, () => {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS experiment_sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
@@ -3399,44 +3406,44 @@ export class Database {
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 76) {
|
||||
this.applyMigration(76, () => {
|
||||
if (version < 77) {
|
||||
this.applyMigration(77, () => {
|
||||
this.addColumnIfMissing("workflow_steps", "gateMode", "TEXT NOT NULL DEFAULT 'advisory'");
|
||||
// FN-4368: advisory-by-default for all legacy workflow_steps rows; users opt in to 'gate' via UI.
|
||||
this.db.exec("UPDATE workflow_steps SET gateMode = 'advisory'");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 77) {
|
||||
this.applyMigration(77, () => {
|
||||
if (version < 78) {
|
||||
this.applyMigration(78, () => {
|
||||
this.addColumnIfMissing("tasks", "tokenBudgetSoftAlertedAt", "TEXT");
|
||||
this.addColumnIfMissing("tasks", "tokenBudgetHardAlertedAt", "TEXT");
|
||||
this.addColumnIfMissing("tasks", "tokenBudgetOverride", "TEXT");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 78) {
|
||||
this.applyMigration(78, () => {
|
||||
if (version < 79) {
|
||||
this.applyMigration(79, () => {
|
||||
this.addColumnIfMissing("tasks", "branchConflictRecoveryCount", "INTEGER DEFAULT 0");
|
||||
this.addColumnIfMissing("tasks", "reviewerContextRetryCount", "INTEGER DEFAULT 0");
|
||||
this.addColumnIfMissing("tasks", "reviewerFallbackRetryCount", "INTEGER DEFAULT 0");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 79) {
|
||||
this.applyMigration(79, () => {
|
||||
this.addColumnIfMissing("tasks", "overlapBlockedBy", "TEXT");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 80) {
|
||||
this.applyMigration(80, () => {
|
||||
this.addColumnIfMissing("milestones", "acceptanceCriteria", "TEXT");
|
||||
this.addColumnIfMissing("tasks", "overlapBlockedBy", "TEXT");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 81) {
|
||||
this.applyMigration(81, () => {
|
||||
this.addColumnIfMissing("milestones", "acceptanceCriteria", "TEXT");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 82) {
|
||||
this.applyMigration(82, () => {
|
||||
this.addColumnIfMissing("tasks", "firstExecutionAt", "TEXT");
|
||||
this.addColumnIfMissing("tasks", "cumulativeActiveMs", "INTEGER");
|
||||
if (this.hasColumn("tasks", "executionStartedAt")) {
|
||||
@@ -3452,14 +3459,14 @@ export class Database {
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 82) {
|
||||
this.applyMigration(82, () => {
|
||||
if (version < 83) {
|
||||
this.applyMigration(83, () => {
|
||||
this.addColumnIfMissing("tasks", "worktreeSessionRetryCount", "INTEGER DEFAULT 0");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 83) {
|
||||
this.applyMigration(83, () => {
|
||||
if (version < 84) {
|
||||
this.applyMigration(84, () => {
|
||||
if (!this.hasTable("secrets")) {
|
||||
this.db.exec(`
|
||||
CREATE TABLE secrets (
|
||||
@@ -3484,8 +3491,8 @@ export class Database {
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 84) {
|
||||
this.applyMigration(84, () => {
|
||||
if (version < 85) {
|
||||
this.applyMigration(85, () => {
|
||||
if (!this.hasColumn("tasks", "title")) {
|
||||
console.log("[title-id-drift] db.ts migration normalized 0 active titles");
|
||||
return;
|
||||
@@ -3514,27 +3521,27 @@ export class Database {
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 85) {
|
||||
this.applyMigration(85, () => {
|
||||
this.addColumnIfMissing("tasks", "completionHandoffLimboRecoveryCount", "INTEGER DEFAULT 0");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 86) {
|
||||
this.applyMigration(86, () => {
|
||||
this.addColumnIfMissing("tasks", "prInfos", "TEXT");
|
||||
this.addColumnIfMissing("tasks", "completionHandoffLimboRecoveryCount", "INTEGER DEFAULT 0");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 87) {
|
||||
this.applyMigration(87, () => {
|
||||
this.addColumnIfMissing("tasks", "deletedAt", "TEXT");
|
||||
this.db.exec("CREATE INDEX IF NOT EXISTS idx_tasks_deletedAt ON tasks(deletedAt)");
|
||||
this.addColumnIfMissing("tasks", "prInfos", "TEXT");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 88) {
|
||||
this.applyMigration(88, () => {
|
||||
this.addColumnIfMissing("tasks", "deletedAt", "TEXT");
|
||||
this.db.exec("CREATE INDEX IF NOT EXISTS idx_tasks_deletedAt ON tasks(deletedAt)");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 89) {
|
||||
this.applyMigration(89, () => {
|
||||
this.addColumnIfMissing("tasks", "allowResurrection", "INTEGER DEFAULT 0");
|
||||
try {
|
||||
const taskColumns = this.getTableColumns("tasks");
|
||||
@@ -3563,8 +3570,8 @@ export class Database {
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 89) {
|
||||
this.applyMigration(89, () => {
|
||||
if (version < 90) {
|
||||
this.applyMigration(90, () => {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS mergeQueue (
|
||||
taskId TEXT PRIMARY KEY REFERENCES tasks(id) ON DELETE CASCADE,
|
||||
@@ -3588,20 +3595,20 @@ export class Database {
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 90) {
|
||||
this.applyMigration(90, () => {
|
||||
this.addColumnIfMissing("tasks", "scopeAutoWiden", "TEXT DEFAULT '[]'");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 91) {
|
||||
this.applyMigration(91, () => {
|
||||
this.addColumnIfMissing("missions", "baseBranch", "TEXT");
|
||||
this.addColumnIfMissing("tasks", "scopeAutoWiden", "TEXT DEFAULT '[]'");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 92) {
|
||||
this.applyMigration(92, () => {
|
||||
this.addColumnIfMissing("missions", "baseBranch", "TEXT");
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 93) {
|
||||
this.applyMigration(93, () => {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS goals (
|
||||
id TEXT PRIMARY KEY,
|
||||
@@ -3619,8 +3626,8 @@ export class Database {
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 93) {
|
||||
this.applyMigration(93, () => {
|
||||
if (version < 94) {
|
||||
this.applyMigration(94, () => {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS goal_citations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
@@ -3652,8 +3659,8 @@ export class Database {
|
||||
});
|
||||
}
|
||||
|
||||
if (version < 94) {
|
||||
this.applyMigration(94, () => {
|
||||
if (version < 95) {
|
||||
this.applyMigration(95, () => {
|
||||
this.addColumnIfMissing("tasks", "autoMerge", "INTEGER");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -151,6 +151,7 @@ export interface TaskMetadataEvidence extends TaskEvidenceEntryBase {
|
||||
verificationFailureCount: number;
|
||||
mergeConflictBounceCount: number;
|
||||
mergeAuditBounceCount: number;
|
||||
mergeTransientRetryCount: number;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -102,6 +102,7 @@ interface TaskRow {
|
||||
verificationFailureCount: number | null;
|
||||
mergeConflictBounceCount: number | null;
|
||||
mergeAuditBounceCount: number | null;
|
||||
mergeTransientRetryCount: number | null;
|
||||
branchConflictRecoveryCount: number | null;
|
||||
reviewerContextRetryCount: number | null;
|
||||
reviewerFallbackRetryCount: number | null;
|
||||
@@ -1405,6 +1406,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
verificationFailureCount: row.verificationFailureCount ?? undefined,
|
||||
mergeConflictBounceCount: row.mergeConflictBounceCount ?? undefined,
|
||||
mergeAuditBounceCount: row.mergeAuditBounceCount ?? undefined,
|
||||
mergeTransientRetryCount: row.mergeTransientRetryCount ?? undefined,
|
||||
branchConflictRecoveryCount: row.branchConflictRecoveryCount ?? undefined,
|
||||
reviewerContextRetryCount: row.reviewerContextRetryCount ?? undefined,
|
||||
reviewerFallbackRetryCount: row.reviewerFallbackRetryCount ?? undefined,
|
||||
@@ -1878,7 +1880,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
"modelPresetId", "modelProvider", "modelId",
|
||||
"validatorModelProvider", "validatorModelId",
|
||||
"planningModelProvider", "planningModelId",
|
||||
"mergeRetries", "workflowStepRetries", "stuckKillCount", "postReviewFixCount", "recoveryRetryCount", "taskDoneRetryCount", "worktreeSessionRetryCount", "completionHandoffLimboRecoveryCount", "verificationFailureCount", "mergeConflictBounceCount", "mergeAuditBounceCount", "branchConflictRecoveryCount", "reviewerContextRetryCount", "reviewerFallbackRetryCount", "nextRecoveryAt",
|
||||
"mergeRetries", "workflowStepRetries", "stuckKillCount", "postReviewFixCount", "recoveryRetryCount", "taskDoneRetryCount", "worktreeSessionRetryCount", "completionHandoffLimboRecoveryCount", "verificationFailureCount", "mergeConflictBounceCount", "mergeAuditBounceCount", "mergeTransientRetryCount", "branchConflictRecoveryCount", "reviewerContextRetryCount", "reviewerFallbackRetryCount", "nextRecoveryAt",
|
||||
"error", "summary", "thinkingLevel", "executionMode",
|
||||
"tokenUsageInputTokens", "tokenUsageOutputTokens", "tokenUsageCachedTokens", "tokenUsageCacheWriteTokens", "tokenUsageTotalTokens", "tokenUsageFirstUsedAt", "tokenUsageLastUsedAt", "tokenBudgetSoftAlertedAt", "tokenBudgetHardAlertedAt", "tokenBudgetOverride",
|
||||
"createdAt", "updatedAt", "columnMovedAt", "firstExecutionAt", "cumulativeActiveMs", "executionStartedAt", "executionCompletedAt",
|
||||
@@ -1927,7 +1929,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
"modelPresetId", "modelProvider", "modelId",
|
||||
"validatorModelProvider", "validatorModelId",
|
||||
"planningModelProvider", "planningModelId",
|
||||
"mergeRetries", "workflowStepRetries", "stuckKillCount", "postReviewFixCount", "recoveryRetryCount", "taskDoneRetryCount", "worktreeSessionRetryCount", "completionHandoffLimboRecoveryCount", "verificationFailureCount", "mergeConflictBounceCount", "mergeAuditBounceCount", "branchConflictRecoveryCount", "reviewerContextRetryCount", "reviewerFallbackRetryCount", "nextRecoveryAt",
|
||||
"mergeRetries", "workflowStepRetries", "stuckKillCount", "postReviewFixCount", "recoveryRetryCount", "taskDoneRetryCount", "worktreeSessionRetryCount", "completionHandoffLimboRecoveryCount", "verificationFailureCount", "mergeConflictBounceCount", "mergeAuditBounceCount", "mergeTransientRetryCount", "branchConflictRecoveryCount", "reviewerContextRetryCount", "reviewerFallbackRetryCount", "nextRecoveryAt",
|
||||
"error", "summary", "thinkingLevel", "executionMode",
|
||||
"tokenUsageInputTokens", "tokenUsageOutputTokens", "tokenUsageCachedTokens", "tokenUsageCacheWriteTokens", "tokenUsageTotalTokens", "tokenUsageFirstUsedAt", "tokenUsageLastUsedAt", "tokenBudgetSoftAlertedAt", "tokenBudgetHardAlertedAt", "tokenBudgetOverride",
|
||||
"createdAt", "updatedAt", "columnMovedAt", "firstExecutionAt", "cumulativeActiveMs", "executionStartedAt", "executionCompletedAt",
|
||||
@@ -2000,6 +2002,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
task.verificationFailureCount ?? 0,
|
||||
task.mergeConflictBounceCount ?? 0,
|
||||
task.mergeAuditBounceCount ?? 0,
|
||||
task.mergeTransientRetryCount ?? 0,
|
||||
task.branchConflictRecoveryCount ?? 0,
|
||||
task.reviewerContextRetryCount ?? 0,
|
||||
task.reviewerFallbackRetryCount ?? 0,
|
||||
@@ -2090,7 +2093,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
id, lineageId, title, description, priority, "column", status, size, reviewLevel, currentStep,
|
||||
worktree, blockedBy, overlapBlockedBy, paused, userPaused, baseBranch, branch, executionStartBranch, baseCommitSha, modelPresetId, modelProvider,
|
||||
modelId, validatorModelProvider, validatorModelId, planningModelProvider, planningModelId, mergeRetries,
|
||||
workflowStepRetries, stuckKillCount, postReviewFixCount, recoveryRetryCount, taskDoneRetryCount, worktreeSessionRetryCount, completionHandoffLimboRecoveryCount, verificationFailureCount, mergeConflictBounceCount, mergeAuditBounceCount, branchConflictRecoveryCount, reviewerContextRetryCount, reviewerFallbackRetryCount, nextRecoveryAt, error,
|
||||
workflowStepRetries, stuckKillCount, postReviewFixCount, recoveryRetryCount, taskDoneRetryCount, worktreeSessionRetryCount, completionHandoffLimboRecoveryCount, verificationFailureCount, mergeConflictBounceCount, mergeAuditBounceCount, mergeTransientRetryCount, branchConflictRecoveryCount, reviewerContextRetryCount, reviewerFallbackRetryCount, nextRecoveryAt, error,
|
||||
summary, thinkingLevel, executionMode, tokenUsageInputTokens, tokenUsageOutputTokens, tokenUsageCachedTokens,
|
||||
tokenUsageCacheWriteTokens, tokenUsageTotalTokens, tokenUsageFirstUsedAt, tokenUsageLastUsedAt, tokenBudgetSoftAlertedAt, tokenBudgetHardAlertedAt, tokenBudgetOverride, createdAt, updatedAt, columnMovedAt,
|
||||
firstExecutionAt, cumulativeActiveMs, executionStartedAt, executionCompletedAt,
|
||||
@@ -2117,7 +2120,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
id, lineageId, title, description, priority, "column", status, size, reviewLevel, currentStep,
|
||||
worktree, blockedBy, overlapBlockedBy, paused, userPaused, baseBranch, branch, executionStartBranch, baseCommitSha, modelPresetId, modelProvider,
|
||||
modelId, validatorModelProvider, validatorModelId, planningModelProvider, planningModelId, mergeRetries,
|
||||
workflowStepRetries, stuckKillCount, postReviewFixCount, recoveryRetryCount, taskDoneRetryCount, worktreeSessionRetryCount, completionHandoffLimboRecoveryCount, verificationFailureCount, mergeConflictBounceCount, mergeAuditBounceCount, branchConflictRecoveryCount, reviewerContextRetryCount, reviewerFallbackRetryCount, nextRecoveryAt, error,
|
||||
workflowStepRetries, stuckKillCount, postReviewFixCount, recoveryRetryCount, taskDoneRetryCount, worktreeSessionRetryCount, completionHandoffLimboRecoveryCount, verificationFailureCount, mergeConflictBounceCount, mergeAuditBounceCount, mergeTransientRetryCount, branchConflictRecoveryCount, reviewerContextRetryCount, reviewerFallbackRetryCount, nextRecoveryAt, error,
|
||||
summary, thinkingLevel, executionMode, tokenUsageInputTokens, tokenUsageOutputTokens, tokenUsageCachedTokens,
|
||||
tokenUsageCacheWriteTokens, tokenUsageTotalTokens, tokenUsageFirstUsedAt, tokenUsageLastUsedAt, tokenBudgetSoftAlertedAt, tokenBudgetHardAlertedAt, tokenBudgetOverride, createdAt, updatedAt, columnMovedAt,
|
||||
firstExecutionAt, cumulativeActiveMs, executionStartedAt, executionCompletedAt,
|
||||
@@ -2163,6 +2166,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
verificationFailureCount = excluded.verificationFailureCount,
|
||||
mergeConflictBounceCount = excluded.mergeConflictBounceCount,
|
||||
mergeAuditBounceCount = excluded.mergeAuditBounceCount,
|
||||
mergeTransientRetryCount = excluded.mergeTransientRetryCount,
|
||||
branchConflictRecoveryCount = excluded.branchConflictRecoveryCount,
|
||||
reviewerContextRetryCount = excluded.reviewerContextRetryCount,
|
||||
reviewerFallbackRetryCount = excluded.reviewerFallbackRetryCount,
|
||||
@@ -5390,7 +5394,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
|
||||
async updateTask(
|
||||
id: string,
|
||||
updates: { title?: string; description?: string; priority?: TaskPriority | null; prompt?: string; worktree?: string | null; status?: string | null; dependencies?: string[]; steps?: import("./types.js").TaskStep[]; currentStep?: number; blockedBy?: string | null; overlapBlockedBy?: string | null; assignedAgentId?: string | null; pausedByAgentId?: string | null; pausedReason?: string | null; tokenBudgetSoftAlertedAt?: string | null; worktrunkFallbackAlertedAt?: string | null; worktrunkFailure?: import("./types.js").Task["worktrunkFailure"] | null; tokenBudgetHardAlertedAt?: string | null; tokenBudgetOverride?: import("./types.js").TaskTokenBudgetOverride | null; dispatchStormCount?: number | null; lastDispatchAt?: string | null; assigneeUserId?: string | null; scopeOverride?: boolean | null; scopeOverrideReason?: string | null; scopeAutoWiden?: string[] | null; nodeId?: string | null; effectiveNodeId?: string | null; effectiveNodeSource?: string | null; checkedOutBy?: string | null; checkedOutAt?: string | null; checkoutNodeId?: string | null; checkoutRunId?: string | null; checkoutLeaseRenewedAt?: string | null; checkoutLeaseEpoch?: number | null; paused?: boolean; baseBranch?: string | null; autoMerge?: boolean | null; branch?: string | null; executionStartBranch?: string | null; baseCommitSha?: string | null; size?: "S" | "M" | "L"; reviewLevel?: number; executionMode?: import("./types.js").ExecutionMode | null; mergeRetries?: number; workflowStepRetries?: number; stuckKillCount?: number | null; postReviewFixCount?: number | null; recoveryRetryCount?: number | null; taskDoneRetryCount?: number | null; worktreeSessionRetryCount?: number | null; completionHandoffLimboRecoveryCount?: number | null; verificationFailureCount?: number | null; mergeConflictBounceCount?: number | null; mergeAuditBounceCount?: number | null; branchConflictRecoveryCount?: number | null; reviewerContextRetryCount?: number | null; reviewerFallbackRetryCount?: number | null; nextRecoveryAt?: string | null; enabledWorkflowSteps?: string[]; noCommitsExpected?: boolean | null; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; planningModelProvider?: string | null; planningModelId?: string | null; thinkingLevel?: string | null; error?: string | null; summary?: string | null; sessionFile?: string | null; firstExecutionAt?: string | null; cumulativeActiveMs?: number | null; executionStartedAt?: string | null; executionCompletedAt?: string | null; review?: import("./types.js").TaskReview | null; reviewState?: import("./types.js").TaskReviewState | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; mergeDetails?: import("./types.js").MergeDetails | null; sourceIssue?: import("./types.js").TaskSourceIssue | null; sourceMetadataPatch?: Record<string, unknown> | null; githubTracking?: import("./types.js").TaskGithubTracking | null; tokenUsage?: import("./types.js").TaskTokenUsage | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
|
||||
updates: { title?: string; description?: string; priority?: TaskPriority | null; prompt?: string; worktree?: string | null; status?: string | null; dependencies?: string[]; steps?: import("./types.js").TaskStep[]; currentStep?: number; blockedBy?: string | null; overlapBlockedBy?: string | null; assignedAgentId?: string | null; pausedByAgentId?: string | null; pausedReason?: string | null; tokenBudgetSoftAlertedAt?: string | null; worktrunkFallbackAlertedAt?: string | null; worktrunkFailure?: import("./types.js").Task["worktrunkFailure"] | null; tokenBudgetHardAlertedAt?: string | null; tokenBudgetOverride?: import("./types.js").TaskTokenBudgetOverride | null; dispatchStormCount?: number | null; lastDispatchAt?: string | null; assigneeUserId?: string | null; scopeOverride?: boolean | null; scopeOverrideReason?: string | null; scopeAutoWiden?: string[] | null; nodeId?: string | null; effectiveNodeId?: string | null; effectiveNodeSource?: string | null; checkedOutBy?: string | null; checkedOutAt?: string | null; checkoutNodeId?: string | null; checkoutRunId?: string | null; checkoutLeaseRenewedAt?: string | null; checkoutLeaseEpoch?: number | null; paused?: boolean; baseBranch?: string | null; autoMerge?: boolean | null; branch?: string | null; executionStartBranch?: string | null; baseCommitSha?: string | null; size?: "S" | "M" | "L"; reviewLevel?: number; executionMode?: import("./types.js").ExecutionMode | null; mergeRetries?: number; workflowStepRetries?: number; stuckKillCount?: number | null; postReviewFixCount?: number | null; recoveryRetryCount?: number | null; taskDoneRetryCount?: number | null; worktreeSessionRetryCount?: number | null; completionHandoffLimboRecoveryCount?: number | null; verificationFailureCount?: number | null; mergeConflictBounceCount?: number | null; mergeAuditBounceCount?: number | null; mergeTransientRetryCount?: number | null; branchConflictRecoveryCount?: number | null; reviewerContextRetryCount?: number | null; reviewerFallbackRetryCount?: number | null; nextRecoveryAt?: string | null; enabledWorkflowSteps?: string[]; noCommitsExpected?: boolean | null; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; planningModelProvider?: string | null; planningModelId?: string | null; thinkingLevel?: string | null; error?: string | null; summary?: string | null; sessionFile?: string | null; firstExecutionAt?: string | null; cumulativeActiveMs?: number | null; executionStartedAt?: string | null; executionCompletedAt?: string | null; review?: import("./types.js").TaskReview | null; reviewState?: import("./types.js").TaskReviewState | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; mergeDetails?: import("./types.js").MergeDetails | null; sourceIssue?: import("./types.js").TaskSourceIssue | null; sourceMetadataPatch?: Record<string, unknown> | null; githubTracking?: import("./types.js").TaskGithubTracking | null; tokenUsage?: import("./types.js").TaskTokenUsage | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
|
||||
runContext?: RunMutationContext,
|
||||
): Promise<Task> {
|
||||
return this.withTaskLock(id, async () => {
|
||||
@@ -5734,6 +5738,11 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
} else if (updates.mergeAuditBounceCount !== undefined) {
|
||||
task.mergeAuditBounceCount = updates.mergeAuditBounceCount;
|
||||
}
|
||||
if (updates.mergeTransientRetryCount === null) {
|
||||
task.mergeTransientRetryCount = undefined;
|
||||
} else if (updates.mergeTransientRetryCount !== undefined) {
|
||||
task.mergeTransientRetryCount = updates.mergeTransientRetryCount;
|
||||
}
|
||||
if (updates.branchConflictRecoveryCount === null) {
|
||||
task.branchConflictRecoveryCount = undefined;
|
||||
} else if (updates.branchConflictRecoveryCount !== undefined) {
|
||||
|
||||
@@ -1893,6 +1893,13 @@ export interface Task {
|
||||
* `MAX_MERGE_AUDIT_BOUNCES`, the task is parked with `status="failed"` and a
|
||||
* recovery follow-up task is created. */
|
||||
mergeAuditBounceCount?: number;
|
||||
/** Number of transient auto-merge retries consumed after provider/network abort
|
||||
* errors (for example AbortError, socket hang up, server_error payloads).
|
||||
* Distinct from `mergeRetries` (in-cycle conflict retries) and
|
||||
* `mergeConflictBounceCount` (in-review→in-progress conflict bounces).
|
||||
* Bounded by `MAX_AUTO_MERGE_TRANSIENT_RETRIES`; once exhausted, the task is
|
||||
* parked with `status="failed"` instead of re-enqueued. */
|
||||
mergeTransientRetryCount?: number;
|
||||
/** Number of branch-conflict recovery attempts consumed by executor branch
|
||||
* conflict auto-recovery loops. Incremented once per recovery retry attempt. */
|
||||
branchConflictRecoveryCount?: number;
|
||||
|
||||
@@ -151,6 +151,7 @@ describe("collectTaskEvaluationEvidence", () => {
|
||||
verificationFailureCount: 7,
|
||||
mergeConflictBounceCount: 8,
|
||||
mergeAuditBounceCount: 0,
|
||||
mergeTransientRetryCount: 0,
|
||||
});
|
||||
|
||||
const summary = evidence.taskMetadata[0]?.summary ?? "";
|
||||
|
||||
@@ -54,6 +54,7 @@ type MockTask = {
|
||||
mergeDetails?: { mergeConfirmed?: boolean; commitSha?: string; mergedAt?: string } | null;
|
||||
verificationFailureCount?: number;
|
||||
mergeConflictBounceCount?: number;
|
||||
mergeTransientRetryCount?: number;
|
||||
branch?: string;
|
||||
worktree?: string;
|
||||
sourceType?: string;
|
||||
@@ -591,7 +592,56 @@ describe("ProjectEngine merge error recovery", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("re-enqueues direct merge on transient non-conflict errors", async () => {
|
||||
vi.useFakeTimers();
|
||||
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
const store = makeStore();
|
||||
vi.mocked(aiMergeTask).mockRejectedValueOnce(new Error("This operation was aborted"));
|
||||
|
||||
const engine = createEngine(store);
|
||||
const privateEngine = engine as unknown as { internalEnqueueMerge: (taskId: string) => void };
|
||||
const enqueueSpy = vi.spyOn(privateEngine, "internalEnqueueMerge");
|
||||
await runMergeCycle(engine);
|
||||
|
||||
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, {
|
||||
mergeTransientRetryCount: 1,
|
||||
status: null,
|
||||
});
|
||||
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||
TASK_ID,
|
||||
expect.objectContaining({ status: "failed" }),
|
||||
);
|
||||
expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 5000);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(enqueueSpy).toHaveBeenCalledWith(TASK_ID);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("parks direct merge when transient retry cap is exhausted", async () => {
|
||||
const store = makeStore({
|
||||
tasks: [makeTask({ mergeTransientRetryCount: 3 }), makeTask({ mergeTransientRetryCount: 3 })],
|
||||
});
|
||||
vi.mocked(aiMergeTask).mockRejectedValueOnce(new Error("socket hang up"));
|
||||
|
||||
const engine = createEngine(store);
|
||||
await runMergeCycle(engine);
|
||||
|
||||
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, {
|
||||
status: "failed",
|
||||
mergeRetries: 3,
|
||||
error: "socket hang up",
|
||||
});
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
TASK_ID,
|
||||
expect.stringContaining("transient retries exhausted"),
|
||||
"MergeTransientRetryExhausted",
|
||||
);
|
||||
});
|
||||
|
||||
it("stores terminal merge metadata for non-conflict direct merge errors", async () => {
|
||||
vi.useFakeTimers();
|
||||
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
const store = makeStore();
|
||||
vi.mocked(aiMergeTask).mockRejectedValueOnce(new Error("remote branch missing"));
|
||||
|
||||
@@ -603,7 +653,13 @@ describe("ProjectEngine merge error recovery", () => {
|
||||
mergeRetries: 3,
|
||||
error: "remote branch missing",
|
||||
});
|
||||
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||
TASK_ID,
|
||||
expect.objectContaining({ mergeTransientRetryCount: expect.any(Number) }),
|
||||
);
|
||||
expect(setTimeoutSpy).not.toHaveBeenCalledWith(expect.any(Function), 5000);
|
||||
expect(hasErrorLog(errorSpy, "after non-conflict error")).toBe(false);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("parks merge-confirmed tasks in stable failed state when finalization is blocked by incomplete steps", async () => {
|
||||
@@ -698,6 +754,37 @@ describe("ProjectEngine merge error recovery", () => {
|
||||
expect(hasErrorLog(errorSpy, "sqlite locked")).toBe(true);
|
||||
});
|
||||
|
||||
it("re-enqueues pull-request merge on transient strategy errors", async () => {
|
||||
vi.useFakeTimers();
|
||||
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
const processPullRequestMerge = vi.fn(async () => {
|
||||
throw new Error("socket hang up");
|
||||
});
|
||||
const store = makeStore();
|
||||
|
||||
const engine = createEngine(store, {
|
||||
getMergeStrategy: () => "pull-request",
|
||||
processPullRequestMerge,
|
||||
});
|
||||
const privateEngine = engine as unknown as { internalEnqueueMerge: (taskId: string) => void };
|
||||
const enqueueSpy = vi.spyOn(privateEngine, "internalEnqueueMerge");
|
||||
|
||||
await runMergeCycle(engine);
|
||||
|
||||
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, {
|
||||
mergeTransientRetryCount: 1,
|
||||
status: null,
|
||||
});
|
||||
expect(store.updateTask).not.toHaveBeenCalledWith(
|
||||
TASK_ID,
|
||||
expect.objectContaining({ status: "failed" }),
|
||||
);
|
||||
expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 5000);
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(enqueueSpy).toHaveBeenCalledWith(TASK_ID);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("logs when non-direct merge strategy recovery update fails", async () => {
|
||||
const store = makeStore({
|
||||
updateTask: vi.fn(async () => {
|
||||
|
||||
@@ -238,6 +238,7 @@ export async function collectTaskEvaluationEvidence(params: {
|
||||
verificationFailureCount: task.verificationFailureCount ?? 0,
|
||||
mergeConflictBounceCount: task.mergeConflictBounceCount ?? 0,
|
||||
mergeAuditBounceCount: task.mergeAuditBounceCount ?? 0,
|
||||
mergeTransientRetryCount: task.mergeTransientRetryCount ?? 0,
|
||||
},
|
||||
}],
|
||||
commits: commitEvidence,
|
||||
|
||||
@@ -39,6 +39,7 @@ import {
|
||||
createAutomatedFollowup,
|
||||
extractFailingTestFiles,
|
||||
} from "./verification-followup-dedup.js";
|
||||
import { isTransientError } from "./transient-error-detector.js";
|
||||
import { TunnelProcessManager } from "./remote-access/tunnel-process-manager.js";
|
||||
import type {
|
||||
ExternalTunnelInfo,
|
||||
@@ -284,6 +285,10 @@ export class ProjectEngine {
|
||||
private shuttingDown = false;
|
||||
|
||||
private static readonly MAX_AUTO_MERGE_RETRIES = 3;
|
||||
/** FN-5697/FN-5674: cap transient provider/network abort retries in auto-merge.
|
||||
* Examples: "This operation was aborted", "socket hang up", `server_error`.
|
||||
* After this cap, the task is parked failed for human visibility. */
|
||||
private static readonly MAX_AUTO_MERGE_TRANSIENT_RETRIES = 3;
|
||||
/** Cap on outer in-review→in-progress bounces caused by deterministic
|
||||
* verification failures during auto-merge. After this many failed merges
|
||||
* for the same task, we stop bouncing it back, mark it failed, and create
|
||||
@@ -2254,6 +2259,16 @@ export class ProjectEngine {
|
||||
// re-attempt; the catch-block-top logEntry already recorded the
|
||||
// failure on the task log.
|
||||
try {
|
||||
if (await this.maybeRetryTransientMerge(store, taskId, taskOnErr, errorMsg)) {
|
||||
continue;
|
||||
}
|
||||
if (this.isTransientMergeRetryExhausted(taskOnErr, errorMsg)) {
|
||||
await store.logEntry(
|
||||
taskId,
|
||||
`Auto-merge transient retries exhausted (${ProjectEngine.MAX_AUTO_MERGE_TRANSIENT_RETRIES}/${ProjectEngine.MAX_AUTO_MERGE_TRANSIENT_RETRIES}); parking task as failed: ${errorMsg}`,
|
||||
"MergeTransientRetryExhausted",
|
||||
);
|
||||
}
|
||||
await store.updateTask(taskId, {
|
||||
status: "failed",
|
||||
mergeRetries: ProjectEngine.MAX_AUTO_MERGE_RETRIES,
|
||||
@@ -2274,6 +2289,16 @@ export class ProjectEngine {
|
||||
// Non-direct merge strategy (e.g. pull-request) errored — park as
|
||||
// failed so the cooldown sweep stops re-attempting silently.
|
||||
try {
|
||||
if (await this.maybeRetryTransientMerge(store, taskId, taskOnErr, errorMsg)) {
|
||||
continue;
|
||||
}
|
||||
if (this.isTransientMergeRetryExhausted(taskOnErr, errorMsg)) {
|
||||
await store.logEntry(
|
||||
taskId,
|
||||
`Auto-merge transient retries exhausted (${ProjectEngine.MAX_AUTO_MERGE_TRANSIENT_RETRIES}/${ProjectEngine.MAX_AUTO_MERGE_TRANSIENT_RETRIES}); parking task as failed: ${errorMsg}`,
|
||||
"MergeTransientRetryExhausted",
|
||||
);
|
||||
}
|
||||
await store.updateTask(taskId, {
|
||||
status: "failed",
|
||||
mergeRetries: ProjectEngine.MAX_AUTO_MERGE_RETRIES,
|
||||
@@ -2312,6 +2337,49 @@ export class ProjectEngine {
|
||||
}
|
||||
}
|
||||
|
||||
private isTransientMergeRetryExhausted(task: Task | null, errorMsg: string): boolean {
|
||||
if (!task || !isTransientError(errorMsg)) {
|
||||
return false;
|
||||
}
|
||||
const current = task.mergeTransientRetryCount ?? 0;
|
||||
return current >= ProjectEngine.MAX_AUTO_MERGE_TRANSIENT_RETRIES;
|
||||
}
|
||||
|
||||
private async maybeRetryTransientMerge(
|
||||
store: TaskStore,
|
||||
taskId: string,
|
||||
taskOnErr: Task | null,
|
||||
errorMsg: string,
|
||||
): Promise<boolean> {
|
||||
if (!taskOnErr || !isTransientError(errorMsg)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const currentRetries = taskOnErr.mergeTransientRetryCount ?? 0;
|
||||
if (currentRetries >= ProjectEngine.MAX_AUTO_MERGE_TRANSIENT_RETRIES) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const nextRetryCount = currentRetries + 1;
|
||||
const delayMs = 5000 * Math.pow(2, currentRetries);
|
||||
await store.updateTask(taskId, {
|
||||
mergeTransientRetryCount: nextRetryCount,
|
||||
status: null,
|
||||
});
|
||||
await store.logEntry(
|
||||
taskId,
|
||||
`Auto-merge transient retry ${nextRetryCount}/${ProjectEngine.MAX_AUTO_MERGE_TRANSIENT_RETRIES} scheduled in ${delayMs / 1000}s: ${errorMsg}`,
|
||||
"MergeTransientRetry",
|
||||
);
|
||||
runtimeLog.log(
|
||||
`Auto-merge transient retry ${nextRetryCount}/${ProjectEngine.MAX_AUTO_MERGE_TRANSIENT_RETRIES} for ${taskId} in ${delayMs / 1000}s`,
|
||||
);
|
||||
setTimeout(() => {
|
||||
if (!this.shuttingDown) this.internalEnqueueMerge(taskId);
|
||||
}, delayMs);
|
||||
return true;
|
||||
}
|
||||
|
||||
private wireAutoMerge(store: TaskStore, _cwd: string): void {
|
||||
this.taskMovedHandler = async ({ task, to }: { task: Task; to: string }) => {
|
||||
if (to !== "in-review") return;
|
||||
|
||||
Reference in New Issue
Block a user