feat(FN-4398): complete Step 2 — add retry summary and retry storm types

Fusion-Task-Id: FN-4398
Fusion-Task-Lineage: 8b898b2e-3468-4fa7-8546-b8f6ba52cf46
This commit is contained in:
Fusion
2026-05-14 14:49:07 -07:00
committed by gsxdsm
parent d20d543e6c
commit 831e4969e9
7 changed files with 200 additions and 1 deletions

View File

@@ -0,0 +1,80 @@
import { describe, expect, it } from "vitest";
import { computeRetrySummary } from "../retry-summary.js";
import type { TaskDetail } from "../types.js";
function makeTask(overrides: Partial<TaskDetail> = {}): TaskDetail {
return {
id: "FN-1",
lineageId: "lineage-1",
description: "desc",
column: "todo",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
prompt: "prompt",
createdAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-01T00:00:00.000Z",
...overrides,
};
}
describe("computeRetrySummary", () => {
it("returns zeros when counters are missing", () => {
expect(computeRetrySummary(makeTask())).toEqual({
stuckKill: 0,
recovery: 0,
taskDone: 0,
workflowStep: 0,
verification: 0,
postReviewFix: 0,
mergeConflict: 0,
branchConflict: 0,
reviewerContext: 0,
reviewerFallback: 0,
total: 0,
});
});
it("aggregates every retry counter", () => {
const summary = computeRetrySummary(makeTask({
stuckKillCount: 1,
recoveryRetryCount: 2,
taskDoneRetryCount: 3,
workflowStepRetries: 4,
verificationFailureCount: 5,
postReviewFixCount: 6,
mergeConflictBounceCount: 7,
branchConflictRecoveryCount: 8,
reviewerContextRetryCount: 9,
reviewerFallbackRetryCount: 10,
}));
expect(summary).toEqual({
stuckKill: 1,
recovery: 2,
taskDone: 3,
workflowStep: 4,
verification: 5,
postReviewFix: 6,
mergeConflict: 7,
branchConflict: 8,
reviewerContext: 9,
reviewerFallback: 10,
total: 55,
});
});
it("treats null/undefined fields as zero", () => {
const summary = computeRetrySummary(makeTask({
stuckKillCount: undefined,
recoveryRetryCount: undefined,
branchConflictRecoveryCount: undefined,
reviewerContextRetryCount: undefined,
reviewerFallbackRetryCount: undefined,
}));
expect(summary.total).toBe(0);
});
});

View File

@@ -1,5 +1,5 @@
export { COLUMNS, DEFAULT_COLUMN, isColumn, normalizeColumn, COLUMN_LABELS, COLUMN_DESCRIPTIONS, VALID_TRANSITIONS, DEFAULT_SETTINGS, DEFAULT_GLOBAL_SETTINGS, DEFAULT_PROJECT_SETTINGS, GLOBAL_SETTINGS_KEYS, PROJECT_SETTINGS_KEYS, isGlobalSettingsKey, isProjectSettingsKey, resolvePersistAgentThinkingLog, THINKING_LEVELS, THEME_MODES, COLOR_THEMES, WORKFLOW_STEP_TEMPLATES, AGENT_PERMISSIONS, PERMANENT_AGENT_ACTION_CATEGORIES, AGENT_PERMISSION_POLICY_ACTION_CATEGORIES, AGENT_PROVISIONING_APPROVAL_MODES, AGENT_PERMISSION_POLICY_PRESET_IDS, LEGACY_AGENT_PERMISSION_POLICY_ACTION_CATEGORY_ALIASES, APPROVAL_REQUEST_STATUSES, APPROVAL_REQUEST_AUDIT_EVENT_TYPES, normalizeApprovalRequestActionCategory, isValidApprovalRequestTransition, agentToConfigSnapshot, diffConfigSnapshots, isEphemeralAgent, hasAgentIdentity, CheckoutConflictError, DEFAULT_HEARTBEAT_PROCEDURE_PATH, getDefaultHeartbeatProcedurePath, EXECUTION_MODES, DEFAULT_EXECUTION_MODE, TASK_PRIORITIES, DEFAULT_TASK_PRIORITY, HIGH_FANOUT_BLOCKER_TODO_THRESHOLD, STALE_HIGH_FANOUT_BLOCKER_AGE_THRESHOLD_MS, DASHBOARD_USER_ID, normalizeMessageParticipant, validateMessageMetadata, validateDockerNodeConfig, sanitizeDockerNodeConfigForResponse, normalizeMergeConflictStrategy, normalizeMergeStrategyOverlapBehavior, normalizePostMergeAuditMode, POST_MERGE_AUDIT_MODES, normalizeMergeAuditAutoRecovery, MERGE_AUDIT_AUTO_RECOVERY_MODES, buildResearchDocumentKey, REPO_OVERRIDE_RE, SHARED_STATE_SNAPSHOT_VERSION } from "./types.js";
export type { Column, IssueInfo, IssueState, TaskSourceIssue, PrInfo, PrStatus, Task, TaskTokenUsage, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, TaskCreateInput, MeshReplicatedTaskCreatePayload, MeshReplicatedTaskApplyResult, TaskSource, SourceType, TaskDetail, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, DistributedTaskIdReserveInput, DistributedTaskIdReserveResult, DistributedTaskIdCommitInput, DistributedTaskIdCommitResult, DistributedTaskIdAbortInput, DistributedTaskIdAbortResult, DistributedTaskIdStateInput, DistributedTaskIdStateResult, AutostashOrphanRecord, AutostashOutcome, MergeDetails, MergeResult, MergeConflictStrategy, CanonicalMergeConflictStrategy, MergeStrategyOverlapBehavior, PostMergeAuditMode, MergeAuditAutoRecoveryMode, DirectMergeCommitStrategy, Settings, GlobalSettings, ProjectSettings, WebSearchBackend, ResearchEnabledSources, ResearchGlobalDefaults, ResearchProjectLimits, ResearchProjectSettings, EvalFollowUpPolicy, EvalProjectSettings, ResolvedEvalSettings, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, ExecutionMode, TaskPriority, UnavailableNodePolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, GithubIssueAction, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepGateMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, PermanentAgentActionCategory, PermanentAgentSensitiveActionCategory, PermanentAgentGatingContext, AgentPermissionPolicy, AgentPermissionPolicyRules, AgentPermissionPolicyActionCategory, AgentProvisioningApprovalMode, LegacyAgentPermissionPolicyActionCategory, ApprovalRequestActionCategoryInput, ApprovalRequestActionCategory, AgentPermissionPolicyDisposition, AgentPermissionPolicyPresetId, ApprovalRequestStatus, ApprovalRequestAuditEventType, ApprovalRequestActorSnapshot, ApprovalRequestTargetAction, ApprovalRequestAuditEvent, ApprovalRequest, ApprovalRequestCreateInput, ApprovalRequestDecisionInput, ApprovalRequestCompletionInput, ApprovalRequestListInput, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, MessageReplyReference, Mailbox, CheckoutLease, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter, AgentMemoryInclusionMode, HeartbeatPromptTemplate, HeartbeatScopeDisciplineMode } from "./types.js";
export type { Column, IssueInfo, IssueState, TaskSourceIssue, PrInfo, PrStatus, Task, TaskTokenUsage, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, TaskCreateInput, MeshReplicatedTaskCreatePayload, MeshReplicatedTaskApplyResult, TaskSource, SourceType, TaskDetail, RetrySummary, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, DistributedTaskIdReserveInput, DistributedTaskIdReserveResult, DistributedTaskIdCommitInput, DistributedTaskIdCommitResult, DistributedTaskIdAbortInput, DistributedTaskIdAbortResult, DistributedTaskIdStateInput, DistributedTaskIdStateResult, AutostashOrphanRecord, AutostashOutcome, MergeDetails, MergeResult, MergeConflictStrategy, CanonicalMergeConflictStrategy, MergeStrategyOverlapBehavior, PostMergeAuditMode, MergeAuditAutoRecoveryMode, DirectMergeCommitStrategy, Settings, GlobalSettings, ProjectSettings, WebSearchBackend, ResearchEnabledSources, ResearchGlobalDefaults, ResearchProjectLimits, ResearchProjectSettings, EvalFollowUpPolicy, EvalProjectSettings, ResolvedEvalSettings, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, ExecutionMode, TaskPriority, UnavailableNodePolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, GithubIssueAction, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepGateMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, PermanentAgentActionCategory, PermanentAgentSensitiveActionCategory, PermanentAgentGatingContext, AgentPermissionPolicy, AgentPermissionPolicyRules, AgentPermissionPolicyActionCategory, AgentProvisioningApprovalMode, LegacyAgentPermissionPolicyActionCategory, ApprovalRequestActionCategoryInput, ApprovalRequestActionCategory, AgentPermissionPolicyDisposition, AgentPermissionPolicyPresetId, ApprovalRequestStatus, ApprovalRequestAuditEventType, ApprovalRequestActorSnapshot, ApprovalRequestTargetAction, ApprovalRequestAuditEvent, ApprovalRequest, ApprovalRequestCreateInput, ApprovalRequestDecisionInput, ApprovalRequestCompletionInput, ApprovalRequestListInput, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, MessageReplyReference, Mailbox, CheckoutLease, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter, AgentMemoryInclusionMode, HeartbeatPromptTemplate, HeartbeatScopeDisciplineMode } from "./types.js";
export { AGENT_VALID_TRANSITIONS } from "./types.js";
export {
resolveAgentMemoryInclusionMode,
@@ -94,6 +94,8 @@ export type {
AgentProvisioningPolicyDecision,
} from "./agent-provisioning-policy.js";
export { TaskStore } from "./store.js";
export { computeRetrySummary, RETRY_STORM_WARNING_RATIO } from "./retry-summary.js";
export { RetryStormError, serializeRetryStormError } from "./retry-storm-error.js";
export { aggregateAgentTokenUsage } from "./agent-token-usage.js";
export type { AgentTokenUsageSummary, AgentTokenUsageWindowSummary } from "./agent-token-usage.js";
export {

View File

@@ -0,0 +1,36 @@
import type { RetrySummary } from "./types.js";
export class RetryStormError extends Error {
readonly category: string;
readonly total: number;
readonly cap: number;
readonly breakdown: RetrySummary;
constructor({ category, total, cap, breakdown }: { category: string; total: number; cap: number; breakdown: RetrySummary }) {
super(`Retry storm: ${total} retries exceeds cap ${cap} (top category: ${category})`);
this.name = "RetryStormError";
this.category = category;
this.total = total;
this.cap = cap;
this.breakdown = breakdown;
}
}
export function serializeRetryStormError(err: RetryStormError): {
type: "RetryStormError";
category: string;
total: number;
cap: number;
breakdown: RetrySummary;
} {
return {
type: "RetryStormError",
category: err.category,
total: err.total,
cap: err.cap,
breakdown: err.breakdown,
};
}

View File

@@ -0,0 +1,42 @@
import type { RetrySummary, TaskDetail } from "./types.js";
export const RETRY_STORM_WARNING_RATIO = 0.8;
const toCount = (value: number | null | undefined): number => (typeof value === "number" ? value : 0);
export function computeRetrySummary(task: TaskDetail): RetrySummary {
const stuckKill = toCount(task.stuckKillCount);
const recovery = toCount(task.recoveryRetryCount);
const taskDone = toCount(task.taskDoneRetryCount);
const workflowStep = toCount(task.workflowStepRetries);
const verification = toCount(task.verificationFailureCount);
const postReviewFix = toCount(task.postReviewFixCount);
const mergeConflict = toCount(task.mergeConflictBounceCount);
const branchConflict = toCount(task.branchConflictRecoveryCount);
const reviewerContext = toCount(task.reviewerContextRetryCount);
const reviewerFallback = toCount(task.reviewerFallbackRetryCount);
const total = stuckKill
+ recovery
+ taskDone
+ workflowStep
+ verification
+ postReviewFix
+ mergeConflict
+ branchConflict
+ reviewerContext
+ reviewerFallback;
return {
stuckKill,
recovery,
taskDone,
workflowStep,
verification,
postReviewFix,
mergeConflict,
branchConflict,
reviewerContext,
reviewerFallback,
total,
};
}

View File

@@ -256,6 +256,10 @@ export const DEFAULT_PROJECT_SETTINGS = {
autoUnpauseBaseDelayMs: 300_000,
autoUnpauseMaxDelayMs: 3_600_000,
maxStuckKills: 6,
maxBranchConflictRecoveries: 5,
maxReviewerContextRetries: 2,
maxReviewerFallbackRetries: 2,
maxTotalRetriesBeforeFail: 25,
preserveProgressOnStuckRequeue: true,
maxPostReviewFixes: 1,
maxSpawnedAgentsPerParent: 5,

View File

@@ -34,6 +34,7 @@ import { assertProjectRootDir } from "./project-root-guard.js";
import { generateTaskLineageId, normalizeTaskCommitAssociation } from "./task-lineage.js";
import { createDistributedTaskIdAllocator, reconcileTaskIdState, resolveLocalNodeId, type DistributedTaskIdAllocator } from "./distributed-task-id.js";
import { detectStalledReview } from "./stalled-review-detector.js";
import { computeRetrySummary } from "./retry-summary.js";
import {
detectTaskIdIntegrityAnomalies,
type TaskIdIntegrityReport,
@@ -3188,6 +3189,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
}
task.stalledReview = detectStalledReview(task, { now: Date.now() });
// Derived at read time only; retrySummary is never persisted to SQLite.
task.retrySummary = computeRetrySummary(task);
// Sync steps from PROMPT.md if task.steps is empty
if (task.steps.length === 0) {
@@ -3295,6 +3298,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
}
}
task.stalledReview = detectStalledReview(task, { now });
// Derived at read time only; retrySummary is never persisted to SQLite.
task.retrySummary = computeRetrySummary(task);
// Slim path: aggregate the timed-execution total server-side, then
// strip the heavy log payload from the wire response. Without this
@@ -3403,6 +3408,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
}
task.timedExecutionMs = this.computeTimedExecutionMs(task.log);
task.stalledReview = detectStalledReview(task, { now });
// Derived at read time only; retrySummary is never persisted to SQLite.
task.retrySummary = computeRetrySummary(task);
task.log = [];
return task;
});

View File

@@ -1495,8 +1495,24 @@ export interface Task {
updatedAt: string;
}
export type RetrySummary = {
stuckKill: number;
recovery: number;
taskDone: number;
workflowStep: number;
verification: number;
postReviewFix: number;
mergeConflict: number;
branchConflict: number;
reviewerContext: number;
reviewerFallback: number;
total: number;
};
export interface TaskDetail extends Task {
prompt: string;
/** Derived aggregate of retry counters (computed on read; never persisted). */
retrySummary?: RetrySummary;
}
/** A task candidate from the inbox-lite work selection, with metadata about why it was selected. */
@@ -2548,6 +2564,18 @@ export interface ProjectSettings {
/** Maximum number of times the stuck-task detector can kill and re-queue a task
* before it is marked as permanently failed. Default: 6. */
maxStuckKills?: number;
/** Maximum branch-conflict auto-recovery retries before failing the task.
* Default: 5. */
maxBranchConflictRecoveries?: number;
/** Maximum reviewer context-limit compact-and-retry attempts before failing.
* Default: 2. */
maxReviewerContextRetries?: number;
/** Maximum reviewer fallback-model retry attempts before failing.
* Default: 2. */
maxReviewerFallbackRetries?: number;
/** Master cap across all retry categories before throwing RetryStormError.
* Default: 25. */
maxTotalRetriesBeforeFail?: number;
/** When the stuck-task detector kills and re-queues a task, preserve the
* task's step progress (step statuses + currentStep) instead of resetting
* every step to `pending`. The worktree and branch are still cleared so