feat(FN-4398): complete Step 2 — add retry summary and retry storm types
Fusion-Task-Id: FN-4398 Fusion-Task-Lineage: 8b898b2e-3468-4fa7-8546-b8f6ba52cf46
This commit is contained in:
80
packages/core/src/__tests__/retry-summary.test.ts
Normal file
80
packages/core/src/__tests__/retry-summary.test.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { computeRetrySummary } from "../retry-summary.js";
|
||||
import type { TaskDetail } from "../types.js";
|
||||
|
||||
function makeTask(overrides: Partial<TaskDetail> = {}): TaskDetail {
|
||||
return {
|
||||
id: "FN-1",
|
||||
lineageId: "lineage-1",
|
||||
description: "desc",
|
||||
column: "todo",
|
||||
dependencies: [],
|
||||
steps: [],
|
||||
currentStep: 0,
|
||||
log: [],
|
||||
prompt: "prompt",
|
||||
createdAt: "2026-01-01T00:00:00.000Z",
|
||||
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("computeRetrySummary", () => {
|
||||
it("returns zeros when counters are missing", () => {
|
||||
expect(computeRetrySummary(makeTask())).toEqual({
|
||||
stuckKill: 0,
|
||||
recovery: 0,
|
||||
taskDone: 0,
|
||||
workflowStep: 0,
|
||||
verification: 0,
|
||||
postReviewFix: 0,
|
||||
mergeConflict: 0,
|
||||
branchConflict: 0,
|
||||
reviewerContext: 0,
|
||||
reviewerFallback: 0,
|
||||
total: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("aggregates every retry counter", () => {
|
||||
const summary = computeRetrySummary(makeTask({
|
||||
stuckKillCount: 1,
|
||||
recoveryRetryCount: 2,
|
||||
taskDoneRetryCount: 3,
|
||||
workflowStepRetries: 4,
|
||||
verificationFailureCount: 5,
|
||||
postReviewFixCount: 6,
|
||||
mergeConflictBounceCount: 7,
|
||||
branchConflictRecoveryCount: 8,
|
||||
reviewerContextRetryCount: 9,
|
||||
reviewerFallbackRetryCount: 10,
|
||||
}));
|
||||
|
||||
expect(summary).toEqual({
|
||||
stuckKill: 1,
|
||||
recovery: 2,
|
||||
taskDone: 3,
|
||||
workflowStep: 4,
|
||||
verification: 5,
|
||||
postReviewFix: 6,
|
||||
mergeConflict: 7,
|
||||
branchConflict: 8,
|
||||
reviewerContext: 9,
|
||||
reviewerFallback: 10,
|
||||
total: 55,
|
||||
});
|
||||
});
|
||||
|
||||
it("treats null/undefined fields as zero", () => {
|
||||
const summary = computeRetrySummary(makeTask({
|
||||
stuckKillCount: undefined,
|
||||
recoveryRetryCount: undefined,
|
||||
branchConflictRecoveryCount: undefined,
|
||||
reviewerContextRetryCount: undefined,
|
||||
reviewerFallbackRetryCount: undefined,
|
||||
}));
|
||||
|
||||
expect(summary.total).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
export { COLUMNS, DEFAULT_COLUMN, isColumn, normalizeColumn, COLUMN_LABELS, COLUMN_DESCRIPTIONS, VALID_TRANSITIONS, DEFAULT_SETTINGS, DEFAULT_GLOBAL_SETTINGS, DEFAULT_PROJECT_SETTINGS, GLOBAL_SETTINGS_KEYS, PROJECT_SETTINGS_KEYS, isGlobalSettingsKey, isProjectSettingsKey, resolvePersistAgentThinkingLog, THINKING_LEVELS, THEME_MODES, COLOR_THEMES, WORKFLOW_STEP_TEMPLATES, AGENT_PERMISSIONS, PERMANENT_AGENT_ACTION_CATEGORIES, AGENT_PERMISSION_POLICY_ACTION_CATEGORIES, AGENT_PROVISIONING_APPROVAL_MODES, AGENT_PERMISSION_POLICY_PRESET_IDS, LEGACY_AGENT_PERMISSION_POLICY_ACTION_CATEGORY_ALIASES, APPROVAL_REQUEST_STATUSES, APPROVAL_REQUEST_AUDIT_EVENT_TYPES, normalizeApprovalRequestActionCategory, isValidApprovalRequestTransition, agentToConfigSnapshot, diffConfigSnapshots, isEphemeralAgent, hasAgentIdentity, CheckoutConflictError, DEFAULT_HEARTBEAT_PROCEDURE_PATH, getDefaultHeartbeatProcedurePath, EXECUTION_MODES, DEFAULT_EXECUTION_MODE, TASK_PRIORITIES, DEFAULT_TASK_PRIORITY, HIGH_FANOUT_BLOCKER_TODO_THRESHOLD, STALE_HIGH_FANOUT_BLOCKER_AGE_THRESHOLD_MS, DASHBOARD_USER_ID, normalizeMessageParticipant, validateMessageMetadata, validateDockerNodeConfig, sanitizeDockerNodeConfigForResponse, normalizeMergeConflictStrategy, normalizeMergeStrategyOverlapBehavior, normalizePostMergeAuditMode, POST_MERGE_AUDIT_MODES, normalizeMergeAuditAutoRecovery, MERGE_AUDIT_AUTO_RECOVERY_MODES, buildResearchDocumentKey, REPO_OVERRIDE_RE, SHARED_STATE_SNAPSHOT_VERSION } from "./types.js";
|
||||
export type { Column, IssueInfo, IssueState, TaskSourceIssue, PrInfo, PrStatus, Task, TaskTokenUsage, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, TaskCreateInput, MeshReplicatedTaskCreatePayload, MeshReplicatedTaskApplyResult, TaskSource, SourceType, TaskDetail, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, DistributedTaskIdReserveInput, DistributedTaskIdReserveResult, DistributedTaskIdCommitInput, DistributedTaskIdCommitResult, DistributedTaskIdAbortInput, DistributedTaskIdAbortResult, DistributedTaskIdStateInput, DistributedTaskIdStateResult, AutostashOrphanRecord, AutostashOutcome, MergeDetails, MergeResult, MergeConflictStrategy, CanonicalMergeConflictStrategy, MergeStrategyOverlapBehavior, PostMergeAuditMode, MergeAuditAutoRecoveryMode, DirectMergeCommitStrategy, Settings, GlobalSettings, ProjectSettings, WebSearchBackend, ResearchEnabledSources, ResearchGlobalDefaults, ResearchProjectLimits, ResearchProjectSettings, EvalFollowUpPolicy, EvalProjectSettings, ResolvedEvalSettings, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, ExecutionMode, TaskPriority, UnavailableNodePolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, GithubIssueAction, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepGateMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, PermanentAgentActionCategory, PermanentAgentSensitiveActionCategory, PermanentAgentGatingContext, AgentPermissionPolicy, AgentPermissionPolicyRules, AgentPermissionPolicyActionCategory, AgentProvisioningApprovalMode, LegacyAgentPermissionPolicyActionCategory, ApprovalRequestActionCategoryInput, ApprovalRequestActionCategory, AgentPermissionPolicyDisposition, AgentPermissionPolicyPresetId, ApprovalRequestStatus, ApprovalRequestAuditEventType, ApprovalRequestActorSnapshot, ApprovalRequestTargetAction, ApprovalRequestAuditEvent, ApprovalRequest, ApprovalRequestCreateInput, ApprovalRequestDecisionInput, ApprovalRequestCompletionInput, ApprovalRequestListInput, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, MessageReplyReference, Mailbox, CheckoutLease, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter, AgentMemoryInclusionMode, HeartbeatPromptTemplate, HeartbeatScopeDisciplineMode } from "./types.js";
|
||||
export type { Column, IssueInfo, IssueState, TaskSourceIssue, PrInfo, PrStatus, Task, TaskTokenUsage, TaskAttachment, TaskComment, TaskCommentInput, TaskDocument, TaskDocumentRevision, TaskDocumentCreateInput, TaskDocumentWithTask, TaskCreateInput, MeshReplicatedTaskCreatePayload, MeshReplicatedTaskApplyResult, TaskSource, SourceType, TaskDetail, RetrySummary, InboxTask, TodoList, TodoItem, TodoListCreateInput, TodoListUpdateInput, TodoItemCreateInput, TodoItemUpdateInput, TodoListWithItems, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, DistributedTaskIdReserveInput, DistributedTaskIdReserveResult, DistributedTaskIdCommitInput, DistributedTaskIdCommitResult, DistributedTaskIdAbortInput, DistributedTaskIdAbortResult, DistributedTaskIdStateInput, DistributedTaskIdStateResult, AutostashOrphanRecord, AutostashOutcome, MergeDetails, MergeResult, MergeConflictStrategy, CanonicalMergeConflictStrategy, MergeStrategyOverlapBehavior, PostMergeAuditMode, MergeAuditAutoRecoveryMode, DirectMergeCommitStrategy, Settings, GlobalSettings, ProjectSettings, WebSearchBackend, ResearchEnabledSources, ResearchGlobalDefaults, ResearchProjectLimits, ResearchProjectSettings, EvalFollowUpPolicy, EvalProjectSettings, ResolvedEvalSettings, SettingsScope, DaemonTokenSettings, TaskStep, StepStatus, TaskLogEntry, RunMutationContext, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, ExecutionMode, TaskPriority, UnavailableNodePolicy, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, GithubIssueAction, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepGateMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, PermanentAgentActionCategory, PermanentAgentSensitiveActionCategory, PermanentAgentGatingContext, AgentPermissionPolicy, AgentPermissionPolicyRules, AgentPermissionPolicyActionCategory, AgentProvisioningApprovalMode, LegacyAgentPermissionPolicyActionCategory, ApprovalRequestActionCategoryInput, ApprovalRequestActionCategory, AgentPermissionPolicyDisposition, AgentPermissionPolicyPresetId, ApprovalRequestStatus, ApprovalRequestAuditEventType, ApprovalRequestActorSnapshot, ApprovalRequestTargetAction, ApprovalRequestAuditEvent, ApprovalRequest, ApprovalRequestCreateInput, ApprovalRequestDecisionInput, ApprovalRequestCompletionInput, ApprovalRequestListInput, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, AgentBudgetConfig, AgentBudgetStatus, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, BlockedStateSnapshot, HeartbeatInvocationSource, AgentTaskSession, AgentRating, AgentRatingSummary, AgentRatingInput, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, NotificationEvent, NotificationPayload, NotificationProviderConfig, CustomProvider, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, MessageMetadata, MessageReplyReference, Mailbox, CheckoutLease, RunAuditDomain, RunAuditEvent, RunAuditEventInput, RunAuditEventFilter, AgentMemoryInclusionMode, HeartbeatPromptTemplate, HeartbeatScopeDisciplineMode } from "./types.js";
|
||||
export { AGENT_VALID_TRANSITIONS } from "./types.js";
|
||||
export {
|
||||
resolveAgentMemoryInclusionMode,
|
||||
@@ -94,6 +94,8 @@ export type {
|
||||
AgentProvisioningPolicyDecision,
|
||||
} from "./agent-provisioning-policy.js";
|
||||
export { TaskStore } from "./store.js";
|
||||
export { computeRetrySummary, RETRY_STORM_WARNING_RATIO } from "./retry-summary.js";
|
||||
export { RetryStormError, serializeRetryStormError } from "./retry-storm-error.js";
|
||||
export { aggregateAgentTokenUsage } from "./agent-token-usage.js";
|
||||
export type { AgentTokenUsageSummary, AgentTokenUsageWindowSummary } from "./agent-token-usage.js";
|
||||
export {
|
||||
|
||||
36
packages/core/src/retry-storm-error.ts
Normal file
36
packages/core/src/retry-storm-error.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import type { RetrySummary } from "./types.js";
|
||||
|
||||
export class RetryStormError extends Error {
|
||||
readonly category: string;
|
||||
|
||||
readonly total: number;
|
||||
|
||||
readonly cap: number;
|
||||
|
||||
readonly breakdown: RetrySummary;
|
||||
|
||||
constructor({ category, total, cap, breakdown }: { category: string; total: number; cap: number; breakdown: RetrySummary }) {
|
||||
super(`Retry storm: ${total} retries exceeds cap ${cap} (top category: ${category})`);
|
||||
this.name = "RetryStormError";
|
||||
this.category = category;
|
||||
this.total = total;
|
||||
this.cap = cap;
|
||||
this.breakdown = breakdown;
|
||||
}
|
||||
}
|
||||
|
||||
export function serializeRetryStormError(err: RetryStormError): {
|
||||
type: "RetryStormError";
|
||||
category: string;
|
||||
total: number;
|
||||
cap: number;
|
||||
breakdown: RetrySummary;
|
||||
} {
|
||||
return {
|
||||
type: "RetryStormError",
|
||||
category: err.category,
|
||||
total: err.total,
|
||||
cap: err.cap,
|
||||
breakdown: err.breakdown,
|
||||
};
|
||||
}
|
||||
42
packages/core/src/retry-summary.ts
Normal file
42
packages/core/src/retry-summary.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import type { RetrySummary, TaskDetail } from "./types.js";
|
||||
|
||||
export const RETRY_STORM_WARNING_RATIO = 0.8;
|
||||
|
||||
const toCount = (value: number | null | undefined): number => (typeof value === "number" ? value : 0);
|
||||
|
||||
export function computeRetrySummary(task: TaskDetail): RetrySummary {
|
||||
const stuckKill = toCount(task.stuckKillCount);
|
||||
const recovery = toCount(task.recoveryRetryCount);
|
||||
const taskDone = toCount(task.taskDoneRetryCount);
|
||||
const workflowStep = toCount(task.workflowStepRetries);
|
||||
const verification = toCount(task.verificationFailureCount);
|
||||
const postReviewFix = toCount(task.postReviewFixCount);
|
||||
const mergeConflict = toCount(task.mergeConflictBounceCount);
|
||||
const branchConflict = toCount(task.branchConflictRecoveryCount);
|
||||
const reviewerContext = toCount(task.reviewerContextRetryCount);
|
||||
const reviewerFallback = toCount(task.reviewerFallbackRetryCount);
|
||||
const total = stuckKill
|
||||
+ recovery
|
||||
+ taskDone
|
||||
+ workflowStep
|
||||
+ verification
|
||||
+ postReviewFix
|
||||
+ mergeConflict
|
||||
+ branchConflict
|
||||
+ reviewerContext
|
||||
+ reviewerFallback;
|
||||
|
||||
return {
|
||||
stuckKill,
|
||||
recovery,
|
||||
taskDone,
|
||||
workflowStep,
|
||||
verification,
|
||||
postReviewFix,
|
||||
mergeConflict,
|
||||
branchConflict,
|
||||
reviewerContext,
|
||||
reviewerFallback,
|
||||
total,
|
||||
};
|
||||
}
|
||||
@@ -256,6 +256,10 @@ export const DEFAULT_PROJECT_SETTINGS = {
|
||||
autoUnpauseBaseDelayMs: 300_000,
|
||||
autoUnpauseMaxDelayMs: 3_600_000,
|
||||
maxStuckKills: 6,
|
||||
maxBranchConflictRecoveries: 5,
|
||||
maxReviewerContextRetries: 2,
|
||||
maxReviewerFallbackRetries: 2,
|
||||
maxTotalRetriesBeforeFail: 25,
|
||||
preserveProgressOnStuckRequeue: true,
|
||||
maxPostReviewFixes: 1,
|
||||
maxSpawnedAgentsPerParent: 5,
|
||||
|
||||
@@ -34,6 +34,7 @@ import { assertProjectRootDir } from "./project-root-guard.js";
|
||||
import { generateTaskLineageId, normalizeTaskCommitAssociation } from "./task-lineage.js";
|
||||
import { createDistributedTaskIdAllocator, reconcileTaskIdState, resolveLocalNodeId, type DistributedTaskIdAllocator } from "./distributed-task-id.js";
|
||||
import { detectStalledReview } from "./stalled-review-detector.js";
|
||||
import { computeRetrySummary } from "./retry-summary.js";
|
||||
import {
|
||||
detectTaskIdIntegrityAnomalies,
|
||||
type TaskIdIntegrityReport,
|
||||
@@ -3188,6 +3189,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
}
|
||||
|
||||
task.stalledReview = detectStalledReview(task, { now: Date.now() });
|
||||
// Derived at read time only; retrySummary is never persisted to SQLite.
|
||||
task.retrySummary = computeRetrySummary(task);
|
||||
|
||||
// Sync steps from PROMPT.md if task.steps is empty
|
||||
if (task.steps.length === 0) {
|
||||
@@ -3295,6 +3298,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
}
|
||||
}
|
||||
task.stalledReview = detectStalledReview(task, { now });
|
||||
// Derived at read time only; retrySummary is never persisted to SQLite.
|
||||
task.retrySummary = computeRetrySummary(task);
|
||||
|
||||
// Slim path: aggregate the timed-execution total server-side, then
|
||||
// strip the heavy log payload from the wire response. Without this
|
||||
@@ -3403,6 +3408,8 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
}
|
||||
task.timedExecutionMs = this.computeTimedExecutionMs(task.log);
|
||||
task.stalledReview = detectStalledReview(task, { now });
|
||||
// Derived at read time only; retrySummary is never persisted to SQLite.
|
||||
task.retrySummary = computeRetrySummary(task);
|
||||
task.log = [];
|
||||
return task;
|
||||
});
|
||||
|
||||
@@ -1495,8 +1495,24 @@ export interface Task {
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export type RetrySummary = {
|
||||
stuckKill: number;
|
||||
recovery: number;
|
||||
taskDone: number;
|
||||
workflowStep: number;
|
||||
verification: number;
|
||||
postReviewFix: number;
|
||||
mergeConflict: number;
|
||||
branchConflict: number;
|
||||
reviewerContext: number;
|
||||
reviewerFallback: number;
|
||||
total: number;
|
||||
};
|
||||
|
||||
export interface TaskDetail extends Task {
|
||||
prompt: string;
|
||||
/** Derived aggregate of retry counters (computed on read; never persisted). */
|
||||
retrySummary?: RetrySummary;
|
||||
}
|
||||
|
||||
/** A task candidate from the inbox-lite work selection, with metadata about why it was selected. */
|
||||
@@ -2548,6 +2564,18 @@ export interface ProjectSettings {
|
||||
/** Maximum number of times the stuck-task detector can kill and re-queue a task
|
||||
* before it is marked as permanently failed. Default: 6. */
|
||||
maxStuckKills?: number;
|
||||
/** Maximum branch-conflict auto-recovery retries before failing the task.
|
||||
* Default: 5. */
|
||||
maxBranchConflictRecoveries?: number;
|
||||
/** Maximum reviewer context-limit compact-and-retry attempts before failing.
|
||||
* Default: 2. */
|
||||
maxReviewerContextRetries?: number;
|
||||
/** Maximum reviewer fallback-model retry attempts before failing.
|
||||
* Default: 2. */
|
||||
maxReviewerFallbackRetries?: number;
|
||||
/** Master cap across all retry categories before throwing RetryStormError.
|
||||
* Default: 25. */
|
||||
maxTotalRetriesBeforeFail?: number;
|
||||
/** When the stuck-task detector kills and re-queues a task, preserve the
|
||||
* task's step progress (step statuses + currentStep) instead of resetting
|
||||
* every step to `pending`. The worktree and branch are still cleared so
|
||||
|
||||
Reference in New Issue
Block a user