6991 lines
300 KiB
TypeScript
6991 lines
300 KiB
TypeScript
/**
|
||
* SelfHealingManager — enables unattended multi-day/week operation by
|
||
* providing automatic recovery from common failure modes.
|
||
*
|
||
* Four subsystems:
|
||
* 1. **Auto-unpause**: Clears rate-limit-triggered `globalPause` with
|
||
* escalating backoff (5 min → 60 min cap). Resets on sustained unpause.
|
||
* 2. **Stuck kill budget**: Caps how many times a task can be killed by the
|
||
* stuck-task detector before marking it as permanently failed.
|
||
* 3. **Periodic maintenance**: Worktree pruning, orphan cleanup, SQLite
|
||
* WAL checkpoint — all on a configurable interval (default 15 min).
|
||
* 4. **Worktree cap enforcement**: Prevents unbounded worktree accumulation
|
||
* by cleaning oldest idle worktrees when count exceeds 2× maxWorktrees.
|
||
* 5. **Completion handoff limbo recovery**: Re-enqueues merge-eligible in-review
|
||
* tasks stuck after `Task marked done by agent` with missing fan-out state.
|
||
*
|
||
* Worktrunk ownership/deference table (`worktrunk.enabled`):
|
||
* - `pruneWorktrees`: defer to backend prune
|
||
* - `cleanupOrphans`: defer to backend prune/remove semantics
|
||
* - `reapUnregisteredOrphans`: defer to backend prune/remove semantics
|
||
* - `enforceWorktreeCap`: defer to backend prune/remove semantics
|
||
* - `reclaimSelfOwnedBranchConflicts`: remains native (branch-level)
|
||
* - `reclaimStaleActiveBranches`: remains native (branch-level)
|
||
* - `scanOrphanedBranches` rescue: remains native (branch-level)
|
||
*/
|
||
|
||
import { exec, execSync } from "node:child_process";
|
||
import { promisify } from "node:util";
|
||
import { existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||
import { isAbsolute, join, relative, resolve } from "node:path";
|
||
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, countRecentIdenticalStallEntries, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
|
||
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
|
||
import { createLogger } from "./logger.js";
|
||
import { RemovalReason, getRegisteredWorktreeBranchMap, getRegisteredWorktreePaths, isUsableTaskWorktree, removeWorktree, resolveWorktreeBackend, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||
import {
|
||
classifyMissingWorktreeSessionStartFailure,
|
||
extractMissingWorktreePathFromSessionStartFailure,
|
||
isMissingWorktreeSessionStartFailure,
|
||
isRecoverableMissingWorktreeReviewFailureNoProgress,
|
||
isRecoverableMissingWorktreeReviewFailureWithProgress,
|
||
} from "./restart-recovery-coordinator.js";
|
||
import { classifyError, extractMissingModulePath, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js";
|
||
import { classifyForeignOnlyContamination, deriveTaskIdFromFusionBranch, inspectBranchConflict, listUniqueBranchCommits } from "./branch-conflicts.js";
|
||
import { createRunAuditor, generateSyntheticRunId, type RunAuditor } from "./run-audit.js";
|
||
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
|
||
import { activeSessionRegistry } from "./active-session-registry.js";
|
||
import { findAlreadyMergedTaskCommit } from "./already-merged-detector.js";
|
||
import { resolveWorktreesDir } from "./worktree-paths.js";
|
||
import { canonicalFusionBranchName } from "./worktree-names.js";
|
||
import type { OwnedLandedClassification } from "./merger.js";
|
||
import { recoverForeignOnlyContamination } from "./recovery/foreign-only-contamination.js";
|
||
import {
|
||
buildNtfyClickUrl,
|
||
isNtfyEventEnabled,
|
||
resolveNtfyEvents,
|
||
sendNtfyNotification,
|
||
type NtfyNotifier,
|
||
} from "./notifier.js";
|
||
import type { GhostBugDecision } from "./triage-preflight.js";
|
||
import { DependencyBlockedTodoReporter } from "./dependency-blocked-todo-reporter.js";
|
||
|
||
const log = createLogger("self-healing");
|
||
const worktreeMetadataReconcileLog = createLogger("worktree-metadata-reconcile");
|
||
const execAsync = promisify(exec);
|
||
const DONE_TASK_INTEGRITY_SWEEP_LIMIT = 50;
|
||
const BOARD_STALL_NOTIFICATION_COOLDOWN_MS = 60 * 60_000;
|
||
export const STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS = 10 * 60_000;
|
||
export const COMPLETION_HANDOFF_LIMBO_GRACE_MS = 5 * 60_000;
|
||
export const MAX_COMPLETION_HANDOFF_LIMBO_RECOVERIES = 3;
|
||
|
||
export async function archiveAsGhostBug(
|
||
store: TaskStore,
|
||
taskId: string,
|
||
taskTitle: string,
|
||
decision: GhostBugDecision,
|
||
): Promise<void> {
|
||
await store.logEntry(
|
||
taskId,
|
||
"Auto-archived as ghost bug — cited code construct not present on main",
|
||
JSON.stringify({ reason: decision.reason, findings: decision.findings }, null, 2),
|
||
);
|
||
await store.recordActivity({
|
||
type: "task:auto-archived-ghost-bug",
|
||
taskId,
|
||
taskTitle,
|
||
details: "Cited construct not found on main",
|
||
metadata: {
|
||
reason: decision.reason,
|
||
findings: decision.findings.slice(0, 10),
|
||
},
|
||
});
|
||
await store.moveTask(taskId, "archived");
|
||
}
|
||
|
||
async function classifyOwnedLandedEvidenceForSelfHealing(rootDir: string, task: Task, mergeTargetBranch: string): Promise<OwnedLandedClassification> {
|
||
const { classifyOwnedLandedEvidence } = await import("./merger.js");
|
||
return classifyOwnedLandedEvidence(rootDir, task, { mergeTargetBranch });
|
||
}
|
||
|
||
function formatRecoveryTimestamp(date = new Date()): string {
|
||
const pad = (value: number) => String(value).padStart(2, "0");
|
||
return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}-${pad(date.getUTCHours())}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}`;
|
||
}
|
||
|
||
async function preserveWorktreeChanges(repoDir: string, worktreePath: string, taskId: string): Promise<string | null> {
|
||
try {
|
||
const status = (await execAsync("git status --porcelain", { cwd: worktreePath, encoding: "utf-8" })).stdout.trim();
|
||
if (!status) {
|
||
return null;
|
||
}
|
||
|
||
const diff = (await execAsync("git diff HEAD --binary", { cwd: worktreePath, encoding: "utf-8", maxBuffer: 10 * 1024 * 1024 })).stdout;
|
||
const recoveryDir = join(repoDir, ".fusion", "recovery");
|
||
mkdirSync(recoveryDir, { recursive: true });
|
||
const patchPath = join(recoveryDir, `${taskId.toLowerCase()}-${formatRecoveryTimestamp()}.patch`);
|
||
writeFileSync(patchPath, diff, "utf-8");
|
||
return patchPath;
|
||
} catch (error) {
|
||
log.warn(`Failed to preserve worktree changes for ${taskId}: ${error instanceof Error ? error.message : String(error)}`);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function matchGlob(path: string, pattern: string): boolean {
|
||
if (pattern.includes("**")) {
|
||
const regexPattern = pattern
|
||
.replace(/\./g, "\\.")
|
||
.replace(/\*\*/g, "<<<DOUBLESTAR>>>")
|
||
.replace(/\*/g, "[^/]*")
|
||
.replace(/<<<DOUBLESTAR>>>/g, ".*");
|
||
const regex = new RegExp(`^${regexPattern}$`);
|
||
return regex.test(path);
|
||
}
|
||
|
||
const lastSlash = pattern.lastIndexOf("/");
|
||
if (lastSlash !== -1) {
|
||
const patternDir = pattern.slice(0, lastSlash);
|
||
const patternFile = pattern.slice(lastSlash + 1);
|
||
const pathDir = path.lastIndexOf("/") !== -1 ? path.slice(0, path.lastIndexOf("/")) : "";
|
||
const pathFile = path.lastIndexOf("/") !== -1 ? path.slice(path.lastIndexOf("/")) : path;
|
||
|
||
if (patternDir.includes("*")) {
|
||
const dirRegex = new RegExp(`^${patternDir.replace(/\./g, "\\.").replace(/\*/g, "[^/]*")}$`);
|
||
if (!dirRegex.test(pathDir)) return false;
|
||
} else if (!pathDir.endsWith(patternDir) && patternDir !== pathDir) {
|
||
return false;
|
||
}
|
||
|
||
return matchGlob(pathFile, patternFile);
|
||
}
|
||
|
||
const fileName = path.lastIndexOf("/") !== -1 ? path.slice(path.lastIndexOf("/") + 1) : path;
|
||
const regexPattern = pattern.replace(/\./g, "\\.").replace(/\*/g, "[^/]*");
|
||
const regex = new RegExp(`^${regexPattern}$`);
|
||
return regex.test(fileName) || regex.test(path);
|
||
}
|
||
|
||
function matchesScope(filePath: string, scopePatterns: string[]): boolean {
|
||
for (const pattern of scopePatterns) {
|
||
if (matchGlob(filePath, pattern)) return true;
|
||
const dirPattern = pattern.replace(/\/\*+$/, "");
|
||
if (dirPattern !== pattern && filePath.startsWith(dirPattern + "/")) return true;
|
||
if (pattern.endsWith("/") && filePath.startsWith(pattern)) return true;
|
||
const patternDir = pattern.lastIndexOf("/") >= 0 ? pattern.slice(0, pattern.lastIndexOf("/")) : "";
|
||
const fileDir = filePath.lastIndexOf("/") >= 0 ? filePath.slice(0, filePath.lastIndexOf("/")) : "";
|
||
if (patternDir && fileDir === patternDir) return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
export interface SelfHealingOptions {
|
||
/** Project root directory (parent of .worktrees/) */
|
||
rootDir: string;
|
||
/** Optional callback to release TaskExecutor in-memory worktree ownership for a task. */
|
||
releaseExecutorWorktreeOwnership?: (taskId: string) => void;
|
||
/** Optional AgentStore for agent-level self-healing checks. */
|
||
agentStore?: AgentStore;
|
||
/** Canonical stale-lease recovery manager. */
|
||
leaseManager?: MeshLeaseManager;
|
||
/**
|
||
* Callback to recover a completed task that is stranded in todo/in-progress.
|
||
* Called by periodic self-healing passes when task work is complete but the
|
||
* final transition never happened (for example killed after task_done).
|
||
*
|
||
* Should return true if the task was successfully transitioned, false if
|
||
* recovery failed.
|
||
*/
|
||
recoverCompletedTask?: (task: Task) => Promise<boolean>;
|
||
/**
|
||
* Returns the set of task IDs currently being executed by the executor.
|
||
* Used to avoid recovering tasks that are actively being worked on.
|
||
*/
|
||
getExecutingTaskIds?: () => Set<string>;
|
||
/**
|
||
* Recover a triage task whose spec was approved but whose final transition
|
||
* out of `status: "planning"` never completed.
|
||
*/
|
||
recoverApprovedTriageTask?: (task: Task) => Promise<boolean>;
|
||
/**
|
||
* Returns the set of task IDs currently being specified by triage.
|
||
* Used to avoid recovering active triage sessions.
|
||
*/
|
||
getPlanningTaskIds?: () => Set<string>;
|
||
/**
|
||
* Evict tasks from the triage processor's `processing` set that have been
|
||
* there longer than the staleness threshold (hung promises from stuck kills).
|
||
* Called before recovery checks so stale entries don't block recovery.
|
||
*/
|
||
evictStaleTriageProcessing?: () => Set<string>;
|
||
/**
|
||
* Auto-revive an `in-review` task whose pre-merge workflow step failed.
|
||
* Delegates to the executor, which injects the failure feedback into
|
||
* `PROMPT.md`, resets steps, and schedules todo → in-progress.
|
||
*
|
||
* Should return true if the task was successfully sent back, false otherwise.
|
||
*/
|
||
recoverFailedPreMergeStep?: (task: Task) => Promise<boolean>;
|
||
/**
|
||
* Re-enqueue a task into the auto-merge queue. Used by
|
||
* `recoverInterruptedMergingTasks` so that a stale `merging` status that was
|
||
* just cleared is retried immediately instead of waiting on the next
|
||
* 15s polling sweep — and so the engine's in-memory `mergeActive` set is
|
||
* refreshed (otherwise a leftover entry from a SIGKILL'd merge would cause
|
||
* the polling sweep's enqueue to silently no-op).
|
||
*/
|
||
enqueueMerge?: (taskId: string) => boolean;
|
||
requeueForAutoMerge?: (taskId: string) => void | Promise<void>;
|
||
isTaskActive?: (taskId: string) => boolean;
|
||
clearMergeActive?: (taskId: string) => void;
|
||
/**
|
||
* Minimum age before a transient merge status is considered stale when no
|
||
* active merge session is associated with that task.
|
||
*/
|
||
staleMergingStatusMinAgeMs?: number;
|
||
/**
|
||
* Returns the task ID actively merging in this engine process, if any.
|
||
* Used to avoid clearing a transient merge status mid-merge.
|
||
*/
|
||
getActiveMergeTaskId?: () => string | null;
|
||
/**
|
||
* Minimum blocker age before stale merge fan-out is cleared from downstream
|
||
* blockedBy pointers. Must be >= staleMergingStatusMinAgeMs.
|
||
*/
|
||
staleMergingFanoutMinAgeMs?: number;
|
||
hasActiveAgentExecution?: (agentId: string) => boolean;
|
||
restartDurableAgentHeartbeat?: (agentId: string, context: { reason: string; attempt: number }) => Promise<boolean>;
|
||
autoRecoveryDispatcher?: AutoRecoveryDispatcher;
|
||
/** Optional ChatStore for maintenance chat-retention cleanup. */
|
||
chatStore?: ChatStore;
|
||
/** Optional MessageStore for maintenance mail-retention cleanup. */
|
||
messageStore?: MessageStore;
|
||
/** Optional notifier for board-stall unrecovered alerts. */
|
||
ntfyNotifier?: Pick<NtfyNotifier, "notifyBoardStallUnrecovered">;
|
||
getProjectId?: () => string;
|
||
}
|
||
|
||
const APPROVED_TRIAGE_RECOVERY_GRACE_MS = 60_000;
|
||
const STARVED_REFINEMENT_RECOVERY_GRACE_MS = 10 * 60_000;
|
||
const STARVED_PEER_PROGRESS_THRESHOLD = 3;
|
||
const STARVED_REFINEMENT_ESCALATION_COOLDOWN_MS = STARVED_REFINEMENT_RECOVERY_GRACE_MS * 4;
|
||
const ORPHANED_EXECUTION_RECOVERY_GRACE_MS = 60_000;
|
||
const ACTIVE_MERGE_STATUSES = new Set(["merging", "merging-pr", "merging-fix"]);
|
||
const NON_TERMINAL_STEP_STATUSES = new Set(["pending", "in-progress"]);
|
||
const STRANDED_COMPLETED_TODO_ACTIVE_STATUSES = new Set([
|
||
"in-progress",
|
||
"planning",
|
||
"specifying",
|
||
|
||
"merging",
|
||
"merging-pr",
|
||
"merging-fix",
|
||
"mission-validation",
|
||
]);
|
||
/** Statuses that represent an explicit human-handoff or active merge —
|
||
* the ghost-review fallback must not disturb tasks parked in these states. */
|
||
const GHOST_REVIEW_PRESERVED_STATUSES = new Set([
|
||
"failed",
|
||
"awaiting-user-review",
|
||
"awaiting-approval",
|
||
"merging",
|
||
"merging-pr",
|
||
"merging-fix",
|
||
]);
|
||
/**
|
||
* Longer grace period for tasks that still have a worktree on disk.
|
||
* This avoids racing with `executor.resumeOrphaned()` which runs on
|
||
* engine startup and may legitimately re-execute these tasks.
|
||
* 5 minutes is well past any startup window.
|
||
*/
|
||
const ORPHANED_WITH_WORKTREE_GRACE_MS = 300_000;
|
||
|
||
/**
|
||
* Maximum times a task can be auto-requeued after the agent exits without
|
||
* calling `fn_task_done`. Bounded so a persistently-broken task cannot loop
|
||
* forever; when exhausted the task stays in `in-review` for human inspection.
|
||
*/
|
||
const MAX_TASK_DONE_RETRIES = 3;
|
||
export const MAX_WORKTREE_SESSION_RETRIES = 3;
|
||
const MAX_AUTO_MERGE_RETRIES = 3;
|
||
const MAX_STARVATION_DROPS = 3;
|
||
const DEADLOCK_RECOVERY_COOLDOWN_MS = 15 * 60_000;
|
||
const DEFAULT_STALE_MERGING_STATUS_MIN_AGE_MS = 5 * 60_000;
|
||
const DEFAULT_STALE_MERGING_FANOUT_MIN_AGE_MS = 15 * 60_000;
|
||
const DURABLE_ERROR_RECOVERY_MAX_RETRIES = 5;
|
||
const DURABLE_ERROR_RECOVERY_BASE_COOLDOWN_MS = 30_000;
|
||
const DURABLE_ERROR_RECOVERY_MAX_COOLDOWN_MS = 15 * 60_000;
|
||
const RUNNING_ON_INACTIVE_TASK_STALE_RUN_MS = 5 * 60_000;
|
||
const ORPHAN_RESCUE_SUBJECT_CAP = 10;
|
||
|
||
function bumpTaskPriority(priority: TaskPriority | undefined): TaskPriority {
|
||
switch (priority ?? "normal") {
|
||
case "low":
|
||
return "normal";
|
||
case "normal":
|
||
return "high";
|
||
case "high":
|
||
return "urgent";
|
||
case "urgent":
|
||
return "urgent";
|
||
}
|
||
}
|
||
|
||
export async function autoRecoverWorktreeSessionStartFailure(
|
||
store: TaskStore,
|
||
task: Task,
|
||
opts: {
|
||
failure: unknown;
|
||
source: "executor-session-start" | "in-review-sweep" | "resume-guard";
|
||
auditor: RunAuditor | null;
|
||
},
|
||
): Promise<{ outcome: "requeue-todo" | "escalate-exhausted"; retries: number; classification: "missing" | "incomplete" | "unregistered" | "unknown" }> {
|
||
const classification = classifyMissingWorktreeSessionStartFailure(opts.failure);
|
||
const nextCount = (task.worktreeSessionRetryCount ?? 0) + 1;
|
||
if (nextCount > MAX_WORKTREE_SESSION_RETRIES) {
|
||
await store.logEntry(
|
||
task.id,
|
||
`Auto-recovery exhausted (${MAX_WORKTREE_SESSION_RETRIES}/${MAX_WORKTREE_SESSION_RETRIES}) for unusable-worktree session-start failure — leaving in-review for human inspection`,
|
||
);
|
||
await opts.auditor?.database({
|
||
type: "task:auto-recover-worktree-session-exhausted",
|
||
target: task.id,
|
||
metadata: {
|
||
retries: task.worktreeSessionRetryCount ?? 0,
|
||
maxRetries: MAX_WORKTREE_SESSION_RETRIES,
|
||
source: opts.source,
|
||
},
|
||
});
|
||
return { outcome: "escalate-exhausted", retries: task.worktreeSessionRetryCount ?? 0, classification };
|
||
}
|
||
|
||
const staleWorktree = task.worktree;
|
||
const missingWorktreePath = extractMissingWorktreePathFromSessionStartFailure(opts.failure);
|
||
const hasMismatchedLiveWorktree =
|
||
typeof staleWorktree === "string" && staleWorktree.length > 0
|
||
&& typeof missingWorktreePath === "string" && missingWorktreePath.length > 0
|
||
&& resolve(staleWorktree) !== resolve(missingWorktreePath);
|
||
const noProgress = !hasStepProgress(task);
|
||
|
||
await store.updateTask(task.id, {
|
||
status: null,
|
||
error: null,
|
||
worktreeSessionRetryCount: nextCount,
|
||
worktree: noProgress ? null : (hasMismatchedLiveWorktree ? staleWorktree : null),
|
||
branch: noProgress ? null : (hasMismatchedLiveWorktree ? task.branch ?? null : null),
|
||
sessionFile: null,
|
||
});
|
||
|
||
const rawFailureExcerpt = typeof task.error === "string"
|
||
? task.error.slice(0, 200)
|
||
: opts.failure instanceof Error
|
||
? opts.failure.message.slice(0, 200)
|
||
: String(opts.failure).slice(0, 200);
|
||
const failureExcerpt = isMissingWorktreeSessionStartFailure(rawFailureExcerpt)
|
||
? "session-start unusable-worktree assertion"
|
||
: rawFailureExcerpt;
|
||
await store.logEntry(
|
||
task.id,
|
||
noProgress
|
||
? `Auto-recovered (no-progress): session-start refused unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`
|
||
: hasMismatchedLiveWorktree
|
||
? `Auto-recovered: stale resume referenced unusable worktree (${missingWorktreePath}) while live task worktree is ${staleWorktree} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`
|
||
: `Auto-recovered: retry/verification session targeted unusable worktree${staleWorktree ? ` (${staleWorktree})` : ""} — cleared stale session metadata and requeued to todo (attempt ${nextCount}/${MAX_WORKTREE_SESSION_RETRIES}, failure: ${failureExcerpt})`,
|
||
);
|
||
if (noProgress) {
|
||
await store.moveTask(task.id, "todo");
|
||
} else {
|
||
await store.moveTask(task.id, "todo", { preserveProgress: true });
|
||
}
|
||
return { outcome: "requeue-todo", retries: nextCount, classification };
|
||
}
|
||
|
||
interface OrphanBranchInspection {
|
||
branch: string;
|
||
tipSha: string;
|
||
uniqueCommitCount: number;
|
||
uniqueCommitSubjects: string[];
|
||
derivedTaskId: string | null;
|
||
registeredWorktreePath: string | null;
|
||
}
|
||
|
||
type RebindOutcome =
|
||
| {
|
||
taskId: string;
|
||
result: "applied";
|
||
branch: string;
|
||
aheadCount: number;
|
||
integrationBase: string;
|
||
previousBranch: string | null;
|
||
}
|
||
| {
|
||
taskId: string;
|
||
result: "skipped";
|
||
reason: "binding-intact" | "no-live-branch" | "ambiguous-candidates" | "no-unique-work";
|
||
candidates?: Array<{ branch: string; aheadCount: number }>;
|
||
};
|
||
|
||
export type RebindResult = { repaired: number; outcomes: RebindOutcome[] };
|
||
|
||
interface LandedTaskCommit {
|
||
sha: string;
|
||
subject?: string;
|
||
filesChanged?: number;
|
||
insertions?: number;
|
||
deletions?: number;
|
||
rebaseBaseSha?: string;
|
||
}
|
||
|
||
function commitOwnedByTask(taskId: string, lineageId: string | undefined, subject: string, body: string): boolean {
|
||
if (lineageId && body.includes(`Fusion-Task-Lineage: ${lineageId}`)) {
|
||
return true;
|
||
}
|
||
return body.includes(`Fusion-Task-Id: ${taskId}`) || subject.includes(taskId);
|
||
}
|
||
|
||
function shellQuote(value: string): string {
|
||
return `'${value.replace(/'/g, "'\\''")}'`;
|
||
}
|
||
|
||
export async function isBranchAheadOfBase(
|
||
task: Task,
|
||
rootDir: string,
|
||
preferredBaseRef?: string,
|
||
): Promise<{ aheadCount: number; baseRef: string } | null> {
|
||
const branchName = task.branch || canonicalFusionBranchName(task.id);
|
||
|
||
try {
|
||
await execAsync(`git rev-parse --verify ${shellQuote(branchName)}`, {
|
||
cwd: rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
} catch {
|
||
return null;
|
||
}
|
||
|
||
const requestedBaseRef = preferredBaseRef || task.mergeDetails?.mergeTargetBranch || "main";
|
||
let resolvedBaseRef = requestedBaseRef;
|
||
|
||
try {
|
||
await execAsync(`git rev-parse --verify ${shellQuote(requestedBaseRef)}`, {
|
||
cwd: rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
} catch {
|
||
const remoteRef = `origin/${requestedBaseRef}`;
|
||
try {
|
||
await execAsync(`git rev-parse --verify ${shellQuote(remoteRef)}`, {
|
||
cwd: rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
resolvedBaseRef = remoteRef;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
try {
|
||
const { stdout } = await execAsync(
|
||
`git rev-list --count ${shellQuote(resolvedBaseRef)}..${shellQuote(branchName)}`,
|
||
{ cwd: rootDir, timeout: 30_000 },
|
||
);
|
||
const aheadCount = Number.parseInt(stdout.trim(), 10);
|
||
if (!Number.isFinite(aheadCount)) {
|
||
return null;
|
||
}
|
||
return { aheadCount, baseRef: resolvedBaseRef };
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(
|
||
`Failed to compare ${branchName} against ${resolvedBaseRef} for ${task.id}: ${errorMessage}`,
|
||
);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function parseShortstat(output: string): Pick<LandedTaskCommit, "filesChanged" | "insertions" | "deletions"> {
|
||
const normalized = output.trim().replace(/\n/g, " ");
|
||
const filesMatch = normalized.match(/(\d+) files? changed/);
|
||
const insertionsMatch = normalized.match(/(\d+) insertions?\(\+\)/);
|
||
const deletionsMatch = normalized.match(/(\d+) deletions?\(-\)/);
|
||
|
||
return {
|
||
filesChanged: filesMatch ? Number.parseInt(filesMatch[1], 10) : 0,
|
||
insertions: insertionsMatch ? Number.parseInt(insertionsMatch[1], 10) : 0,
|
||
deletions: deletionsMatch ? Number.parseInt(deletionsMatch[1], 10) : 0,
|
||
};
|
||
}
|
||
|
||
function hasTerminalInvalidDoneTransition(task: Pick<Task, "error">): boolean {
|
||
const error = task.error ?? "";
|
||
return error.includes("Invalid transition:") && error.includes("→ 'done'");
|
||
}
|
||
|
||
export class SelfHealingManager {
|
||
// ── Auto-unpause state ──────────────────────────────────────────────
|
||
private unpauseTimer: ReturnType<typeof setTimeout> | null = null;
|
||
private unpauseAttempt = 0;
|
||
private lastPauseTriggeredAt = 0;
|
||
private lastUnpauseAt = 0;
|
||
|
||
// ── Maintenance timer ───────────────────────────────────────────────
|
||
private maintenanceInterval: ReturnType<typeof setInterval> | null = null;
|
||
private maintenanceRunning = false;
|
||
|
||
// ── Event listener cleanup ──────────────────────────────────────────
|
||
private settingsListener: ((data: { settings: Settings; previous: Settings }) => void) | null = null;
|
||
private taskMovedFanoutListener: ((data: { task: Task; from: string; to: string; source: string }) => void) | null = null;
|
||
|
||
// ── Per-task deadlock recovery cooldown ─────────────────────────────
|
||
private deadlockRecoveryCooldown: Map<string, number> = new Map();
|
||
private mergeStarvationDrops: Map<string, number> = new Map();
|
||
private orphanArchivedAcknowledged = new Set<string>();
|
||
private finalizeUnprovenWarned = new Set<string>();
|
||
private maintenanceTickCounter = 0;
|
||
private dependencyBlockedTodoReporter: DependencyBlockedTodoReporter | null = null;
|
||
|
||
private boardStallWindow: {
|
||
windowStartMs: number;
|
||
windowStartBlockedDepth: number;
|
||
transitionsOutOfInProgressInWindow: number;
|
||
pendingVerification: { holderIds: string[]; followerCount: number; startedAt: number; tick: number } | null;
|
||
lastNtfyAt: number | null;
|
||
} | null = null;
|
||
|
||
private static readonly PAUSED_SCOPE_DECAY_EXCLUDED_REASONS = new Set([
|
||
"branch-conflict-unrecoverable",
|
||
"worktrunk_operation_failed",
|
||
"token_budget_exceeded",
|
||
]);
|
||
|
||
constructor(
|
||
private store: TaskStore,
|
||
private options: SelfHealingOptions,
|
||
) {}
|
||
|
||
private emitTaskMerged(task: Task | undefined | null, overrides: Partial<MergeResult> = {}): void {
|
||
if (!task) return;
|
||
this.store.emit("task:merged", {
|
||
task,
|
||
branch: task.branch ?? "",
|
||
merged: true,
|
||
worktreeRemoved: false,
|
||
branchDeleted: false,
|
||
mergeConfirmed: task.mergeDetails?.mergeConfirmed,
|
||
mergedAt: task.mergeDetails?.mergedAt,
|
||
mergeTargetBranch: task.mergeDetails?.mergeTargetBranch,
|
||
...overrides,
|
||
} as MergeResult);
|
||
}
|
||
|
||
// ── Lifecycle ───────────────────────────────────────────────────────
|
||
|
||
start(): void {
|
||
// Wire up settings:updated listener for auto-unpause
|
||
this.settingsListener = ({ settings, previous }) => {
|
||
this.onSettingsUpdated(settings, previous);
|
||
};
|
||
this.store.on("settings:updated", this.settingsListener);
|
||
|
||
this.taskMovedFanoutListener = ({ task, from, to }) => {
|
||
if (
|
||
from === "in-progress"
|
||
&& (to === "todo" || to === "in-review" || to === "done" || to === "archived")
|
||
&& this.boardStallWindow
|
||
) {
|
||
// In-memory only counter; resets on engine restart.
|
||
this.boardStallWindow.transitionsOutOfInProgressInWindow++;
|
||
}
|
||
if (to === "in-review") {
|
||
void this.reconcileInReviewBranchRebind({ includeTaskIds: new Set([task.id]) }).catch((err: unknown) => {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`[self-healing] task:moved in-review rebind failed for ${task.id}: ${errorMessage}`);
|
||
});
|
||
}
|
||
const shouldReconcile =
|
||
(from === "in-review" && to === "done") ||
|
||
(from === "done" && to === "archived");
|
||
if (!shouldReconcile) return;
|
||
void this.reconcileCompletedTask(task.id, { worktreeHint: task.worktree ?? undefined }).catch((err: unknown) => {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`[self-healing] task:moved completion fan-out failed for ${task.id}: ${errorMessage}`);
|
||
});
|
||
};
|
||
this.store.on("task:moved", this.taskMovedFanoutListener);
|
||
|
||
// Start periodic maintenance
|
||
this.startMaintenance();
|
||
|
||
log.log("Started");
|
||
}
|
||
|
||
/**
|
||
* Run only the recovery subset needed at runtime startup, after the executor
|
||
* has had a chance to resume orphaned sessions.
|
||
*
|
||
* This avoids waiting for the periodic maintenance interval before fixing
|
||
* stale in-progress/planning tasks that no longer have a live worker.
|
||
*/
|
||
async runStartupRecovery(): Promise<void> {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) {
|
||
log.log(
|
||
`Startup recovery skipped — ${
|
||
settings.globalPause ? "global pause" : "engine pause"
|
||
} is active`,
|
||
);
|
||
return;
|
||
}
|
||
|
||
// Each recovery step is isolated — one failure doesn't prevent subsequent steps.
|
||
const steps: Array<{ name: string; fn: () => Promise<unknown> }> = [
|
||
{ name: "no-progress-no-task-done", fn: () => this.recoverNoProgressNoTaskDoneFailures().then(() => undefined) },
|
||
{ name: "completed-tasks", fn: () => this.recoverCompletedTasks().then(() => undefined) },
|
||
{ name: "recover-stranded-completed-todo", fn: () => this.recoverStrandedCompletedTodoTasks().then(() => undefined) },
|
||
{ name: "stale-incomplete-review", fn: () => this.recoverStaleIncompleteReviewTasks().then(() => undefined) },
|
||
{ name: "failed-pre-merge-steps", fn: () => this.recoverReviewTasksWithFailedPreMergeSteps().then(() => undefined) },
|
||
{ name: "interrupted-merging", fn: () => this.recoverInterruptedMergingTasks().then(() => undefined) },
|
||
{ name: "done-merge-metadata", fn: () => this.recoverDoneTaskMergeMetadata().then(() => undefined) },
|
||
{ name: "reconcile-done-task-integrity", fn: () => this.reconcileDoneTaskIntegrity().then(() => undefined) },
|
||
// FN-5092: must run BEFORE any merger pickup path so the merger queue is
|
||
// not stalled by a leaked `status: "merging"` on an already-done task.
|
||
{ name: "reconcile-stale-merger-status", fn: () => this.reconcileStaleMergerStatus().then(() => undefined) },
|
||
{ name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks().then(() => undefined) },
|
||
{ name: "recover-completion-handoff-limbo", fn: () => this.recoverCompletionHandoffLimbo().then(() => undefined) },
|
||
{ name: "recover-branch-misbound-in-review", fn: () => this.recoverBranchMisboundInReviewTasks().then(() => undefined) },
|
||
{ name: "recover-foreign-only-contamination-in-review", fn: () => this.recoverForeignOnlyContaminatedInReviewTasks().then(() => undefined) },
|
||
{ name: "recover-orphan-only-scope-violations", fn: () => this.recoverOrphanOnlyScopeViolations().then(() => undefined) },
|
||
{ name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks().then(() => undefined) },
|
||
{ name: "misclassified-failures", fn: () => this.recoverMisclassifiedFailures().then(() => undefined) },
|
||
{ name: "missing-worktree-review-failures", fn: () => this.recoverMissingWorktreeReviewFailures().then(() => undefined) },
|
||
{ name: "partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures().then(() => undefined) },
|
||
{ name: "orphaned-executions", fn: () => this.recoverOrphanedExecutions().then(() => undefined) },
|
||
{ name: "approved-triage", fn: () => this.recoverApprovedTriageTasks().then(() => undefined) },
|
||
{ name: "recover-starved-refinement", fn: () => this.recoverStarvedRefinementTriageTasks().then(() => undefined) },
|
||
{ name: "orphaned-planning", fn: () => this.recoverOrphanedPlanningTasks().then(() => undefined) },
|
||
{ name: "recover-orphaned-agents", fn: () => this.recoverOrphanedAgents().then(() => undefined) },
|
||
{ name: "recover-stale-heartbeat-runs", fn: () => this.recoverStaleHeartbeatRuns().then(() => undefined) },
|
||
{ name: "recover-running-on-inactive-tasks", fn: () => this.recoverAgentsRunningOnInactiveTasks().then(() => undefined) },
|
||
{ name: "recover-drifted-agent-task-links", fn: () => this.recoverDriftedAgentTaskLinks().then(() => undefined) },
|
||
{ name: "clear-stale-blocked-by", fn: () => this.clearStaleBlockedBy().then(() => undefined) },
|
||
{ name: "reconcile-self-defeating-deps", fn: () => this.reconcileSelfDefeatingDependencies().then(() => undefined) },
|
||
{ name: "reclaim-pr-conflicts", fn: () => this.reclaimPrConflicts().then(() => undefined) },
|
||
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts().then(() => undefined) },
|
||
// FN-4962 ordering invariant: metadata reconcile must run before stale-active reclaim.
|
||
{ name: "reconcile-task-worktree-metadata", fn: () => this.reconcileTaskWorktreeMetadata().then(() => undefined) },
|
||
{ name: "reconcile-in-review-branch-rebind", fn: () => this.reconcileInReviewBranchRebind().then(() => undefined) },
|
||
{ name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches().then(() => undefined) },
|
||
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls().then(() => undefined) },
|
||
{ name: "surface-in-review-stalled", fn: () => this.surfaceInReviewStalled().then(() => undefined) },
|
||
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews().then(() => undefined) },
|
||
{ name: "surface-stale-paused-todos", fn: () => this.surfaceStalePausedTodos().then(() => undefined) },
|
||
{ name: "audit-no-commits-expected-candidates", fn: () => this.auditNoCommitsExpectedCandidates().then(() => undefined) },
|
||
];
|
||
|
||
for (const step of steps) {
|
||
try {
|
||
await step.fn();
|
||
log.log(`Startup recovery step "${step.name}" completed`);
|
||
} catch (stepErr) {
|
||
const stepErrMessage = stepErr instanceof Error ? stepErr.message : String(stepErr);
|
||
log.error(`Startup recovery step "${step.name}" failed: ${stepErrMessage} — continuing with remaining steps`);
|
||
}
|
||
}
|
||
}
|
||
|
||
stop(): void {
|
||
// Remove settings listener
|
||
if (this.settingsListener) {
|
||
try {
|
||
this.store.removeListener("settings:updated", this.settingsListener);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
// Store may not support removeListener (e.g., test mocks) — non-fatal.
|
||
log.warn(`Failed to remove settings:updated listener during stop(): ${errorMessage}`);
|
||
}
|
||
this.settingsListener = null;
|
||
}
|
||
|
||
if (this.taskMovedFanoutListener) {
|
||
try {
|
||
this.store.off("task:moved", this.taskMovedFanoutListener);
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`Failed to remove task:moved listener during stop(): ${errorMessage}`);
|
||
}
|
||
this.taskMovedFanoutListener = null;
|
||
}
|
||
|
||
// Clear timers
|
||
this.cancelUnpauseTimer();
|
||
if (this.maintenanceInterval) {
|
||
clearInterval(this.maintenanceInterval);
|
||
this.maintenanceInterval = null;
|
||
}
|
||
|
||
log.log("Stopped");
|
||
}
|
||
|
||
// ── Auto-unpause ───────────────────────────────────────────────────
|
||
|
||
private onSettingsUpdated(settings: Settings, previous: Settings): void {
|
||
// globalPause false → true: schedule auto-unpause
|
||
if (!previous.globalPause && settings.globalPause) {
|
||
if (!settings.autoUnpauseEnabled) {
|
||
log.log("Global pause activated — auto-unpause disabled, requires manual intervention");
|
||
return;
|
||
}
|
||
|
||
if (settings.globalPauseReason === "manual") {
|
||
log.log("Global pause activated manually — auto-unpause skipped, requires manual intervention");
|
||
return;
|
||
}
|
||
|
||
// If pause re-triggered within 60s of our last unpause, escalate backoff
|
||
if (this.lastUnpauseAt && (Date.now() - this.lastUnpauseAt) < 60_000) {
|
||
this.unpauseAttempt++;
|
||
log.warn(`Global pause re-triggered within 60s — escalating to attempt ${this.unpauseAttempt}`);
|
||
}
|
||
|
||
this.lastPauseTriggeredAt = Date.now();
|
||
|
||
const baseDelay = settings.autoUnpauseBaseDelayMs ?? 300_000;
|
||
const maxDelay = settings.autoUnpauseMaxDelayMs ?? 3_600_000;
|
||
const delay = Math.min(baseDelay * Math.pow(2, this.unpauseAttempt), maxDelay);
|
||
|
||
this.scheduleUnpause(delay);
|
||
}
|
||
|
||
// globalPause true → false: check if we should reset backoff
|
||
if (previous.globalPause && !settings.globalPause) {
|
||
this.cancelUnpauseTimer();
|
||
|
||
// If sustained unpause (not a quick re-trigger), reset attempt counter
|
||
if (this.lastPauseTriggeredAt && (Date.now() - this.lastPauseTriggeredAt) > 60_000) {
|
||
this.unpauseAttempt = 0;
|
||
}
|
||
}
|
||
}
|
||
|
||
private scheduleUnpause(delayMs: number): void {
|
||
this.cancelUnpauseTimer();
|
||
|
||
const delaySec = Math.round(delayMs / 1000);
|
||
const delayMin = Math.round(delaySec / 60);
|
||
const display = delayMin >= 1 ? `${delayMin}m` : `${delaySec}s`;
|
||
log.warn(`Auto-unpause scheduled in ${display} (attempt ${this.unpauseAttempt + 1})`);
|
||
|
||
this.unpauseTimer = setTimeout(() => {
|
||
this.unpauseTimer = null;
|
||
void this.attemptUnpause();
|
||
}, delayMs);
|
||
}
|
||
|
||
private async attemptUnpause(): Promise<void> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
|
||
// Already unpaused (manually or by another mechanism)
|
||
if (!settings.globalPause) {
|
||
log.log("Auto-unpause: already unpaused — no action needed");
|
||
this.unpauseAttempt = 0;
|
||
return;
|
||
}
|
||
|
||
log.warn("Auto-unpause: clearing globalPause");
|
||
this.lastUnpauseAt = Date.now();
|
||
await this.store.updateSettings({ globalPause: false, globalPauseReason: undefined });
|
||
|
||
// Note: if the rate limit is still active, the next agent session will
|
||
// hit it again → UsageLimitPauser triggers globalPause → our listener
|
||
// catches the transition and schedules the next attempt with escalated backoff.
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Auto-unpause failed: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
private cancelUnpauseTimer(): void {
|
||
if (this.unpauseTimer) {
|
||
clearTimeout(this.unpauseTimer);
|
||
this.unpauseTimer = null;
|
||
}
|
||
}
|
||
|
||
// ── Stuck kill budget ─────────────────────────────────────────────
|
||
|
||
/**
|
||
* Check whether a stuck-killed task should be re-queued or marked as failed.
|
||
* Called by StuckTaskDetector's `beforeRequeue` callback.
|
||
*
|
||
* Terminal contract for stuck-loop exhaustion and no-progress churn:
|
||
* - `STUCK_LOOP_EXHAUSTED`: increments the kill budget until exhausted, then
|
||
* marks the task failed and parks it in `in-review`.
|
||
* - `STUCK_NO_PROGRESS_CHURN`: skips the budget entirely and terminalizes on
|
||
* the first trigger with operator guidance to decompose or rescope.
|
||
*
|
||
* @returns `true` if the task should be re-queued, `false` if terminalized.
|
||
*/
|
||
async checkStuckBudget(
|
||
taskId: string,
|
||
reason: "loop" | "inactivity" | "no-progress-churn" = "inactivity",
|
||
event?: { ignoredStepUpdateCount?: number },
|
||
): Promise<boolean> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
const maxKills = settings.maxStuckKills ?? 6;
|
||
|
||
const task = await this.store.getTask(taskId);
|
||
|
||
if (reason === "no-progress-churn") {
|
||
const ignoredStepUpdateCount = event?.ignoredStepUpdateCount ?? 0;
|
||
const stuckKillStreak = task.stuckKillCount ?? 0;
|
||
log.warn(
|
||
`${taskId} no-progress churn detected ` +
|
||
`(ignoredStepUpdates=${ignoredStepUpdateCount}, stuckKillStreak=${stuckKillStreak}) — marking failed`,
|
||
);
|
||
const churnError =
|
||
`STUCK_NO_PROGRESS_CHURN: detected ${ignoredStepUpdateCount} ignored step-update rebuffs after compact-and-resume failed to recover progress. ` +
|
||
`Task is likely too large; decompose via fn_task_create child tasks or rescope. No further automatic retries will run.`;
|
||
await this.store.updateTask(taskId, {
|
||
status: "failed",
|
||
error: churnError,
|
||
});
|
||
try {
|
||
await this.store.moveTask(taskId, "in-review");
|
||
} catch (moveErr: unknown) {
|
||
const moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr);
|
||
log.warn(`${taskId} moveTask("in-review") failed (${moveErrMessage}) after STUCK_NO_PROGRESS_CHURN terminalization — task already marked failed, not re-queuing`);
|
||
}
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`STUCK_NO_PROGRESS_CHURN: detected ${ignoredStepUpdateCount} ignored step-update rebuffs after compact-and-resume failed to recover progress. ` +
|
||
`No further automatic retries will run. Pause the task, manually decompose the work via fn_task_create child tasks, or move it to triage to rescope.`,
|
||
);
|
||
const churnAudit = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("fn5168-stuck-churn", taskId),
|
||
agentId: "self-healing",
|
||
taskId,
|
||
taskLineageId: task.lineageId,
|
||
phase: "stuck-no-progress-churn-terminalized",
|
||
});
|
||
await churnAudit.database({
|
||
type: "task:stuck-no-progress-churn-terminalized",
|
||
target: taskId,
|
||
metadata: {
|
||
taskId,
|
||
ignoredStepUpdateCount,
|
||
stuckKillStreak,
|
||
lastReason: reason,
|
||
},
|
||
});
|
||
return false;
|
||
}
|
||
|
||
const newCount = (task.stuckKillCount ?? 0) + 1;
|
||
|
||
if (newCount > maxKills) {
|
||
// Budget exhausted — mark as permanently failed
|
||
log.warn(`${taskId} exceeded stuck kill budget (${newCount}/${maxKills}, reason=${reason}) — marking failed`);
|
||
const exhaustedError =
|
||
`STUCK_LOOP_EXHAUSTED: stuck kill budget exhausted (${newCount}/${maxKills}) after last reason=${reason}.`;
|
||
await this.store.updateTask(taskId, {
|
||
stuckKillCount: newCount,
|
||
status: "failed",
|
||
error: exhaustedError,
|
||
});
|
||
try {
|
||
await this.store.moveTask(taskId, "in-review");
|
||
} catch (moveErr: unknown) {
|
||
// moveTask may fail if task was concurrently moved (e.g., dep-abort).
|
||
// The task is already marked failed — don't allow requeue.
|
||
const moveErrMessage = moveErr instanceof Error ? moveErr.message : String(moveErr);
|
||
log.warn(`${taskId} moveTask("in-review") failed (${moveErrMessage}) after STUCK_LOOP_EXHAUSTED terminalization — task already marked failed, not re-queuing`);
|
||
}
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`STUCK_LOOP_EXHAUSTED: stuck kill budget exhausted (${newCount}/${maxKills}), last reason=${reason}. No further automatic retries will run. Manually retry, pause, or move the task to triage to resume work.`,
|
||
);
|
||
return false;
|
||
}
|
||
|
||
// Budget remaining — allow re-queue
|
||
log.log(`${taskId} stuck kill ${newCount}/${maxKills} — will re-queue`);
|
||
await this.store.updateTask(taskId, { stuckKillCount: newCount });
|
||
await this.store.logEntry(
|
||
taskId,
|
||
`Stuck kill ${newCount}/${maxKills} — re-queuing for retry`,
|
||
);
|
||
return true;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`checkStuckBudget failed for ${taskId}: ${errorMessage}`);
|
||
// On error, allow re-queue — safer than permanently failing
|
||
return true;
|
||
}
|
||
}
|
||
|
||
// ── Lost work detection ────────────────────────────────────────────
|
||
|
||
/**
|
||
* Check whether a task's branch has any unique commits compared to main.
|
||
* If the branch has no unique commits and the task has steps marked done,
|
||
* those steps represent lost uncommitted work — reset them to "pending"
|
||
* so the next execution doesn't skip them.
|
||
*/
|
||
private async resetStepsIfWorkLost(task: Task): Promise<void> {
|
||
const completedSteps = task.steps.filter(
|
||
(s) => s.status === "done" || s.status === "in-progress",
|
||
);
|
||
if (completedSteps.length === 0) return;
|
||
|
||
const branchName = task.branch || canonicalFusionBranchName(task.id);
|
||
|
||
try {
|
||
const { stdout: mergeBaseOut } = await execAsync(
|
||
`git merge-base "${branchName}" HEAD`,
|
||
{ cwd: this.options.rootDir, encoding: "utf-8", timeout: 30_000 },
|
||
);
|
||
const mergeBase = mergeBaseOut.trim();
|
||
const { stdout: branchHeadOut } = await execAsync(
|
||
`git rev-parse "${branchName}"`,
|
||
{ cwd: this.options.rootDir, encoding: "utf-8", timeout: 30_000 },
|
||
);
|
||
const branchHead = branchHeadOut.trim();
|
||
|
||
if (mergeBase === branchHead) {
|
||
log.warn(
|
||
`${task.id} branch has no unique commits — resetting ${completedSteps.length} step(s) to pending`,
|
||
);
|
||
|
||
for (let i = 0; i < task.steps.length; i++) {
|
||
if (task.steps[i].status === "done" || task.steps[i].status === "in-progress") {
|
||
await this.store.updateStep(task.id, i, "pending");
|
||
}
|
||
}
|
||
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Reset ${completedSteps.length} step(s) to pending — branch had no commits (uncommitted work lost with worktree)`,
|
||
);
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(
|
||
`Failed to reset steps for ${task.id} after branch/worktree loss (${branchName}): ${errorMessage} — non-fatal`,
|
||
);
|
||
}
|
||
}
|
||
|
||
// ── Periodic maintenance ──────────────────────────────────────────
|
||
|
||
private async startMaintenance(): Promise<void> {
|
||
const settings = await this.store.getSettings();
|
||
const intervalMs = settings.maintenanceIntervalMs ?? 900_000;
|
||
|
||
if (intervalMs <= 0) {
|
||
log.log("Periodic maintenance disabled (maintenanceIntervalMs <= 0)");
|
||
return;
|
||
}
|
||
|
||
log.log(`Periodic maintenance every ${Math.round(intervalMs / 60_000)}m`);
|
||
this.maintenanceInterval = setInterval(() => {
|
||
void this.runMaintenance();
|
||
}, intervalMs);
|
||
}
|
||
|
||
private isPastInterruptedMergeGrace(task: Task, timeoutMs: number): boolean {
|
||
const updatedAt = task.updatedAt ? Date.parse(task.updatedAt) : 0;
|
||
if (!Number.isFinite(updatedAt) || updatedAt <= 0) return false;
|
||
return Date.now() - updatedAt >= timeoutMs;
|
||
}
|
||
|
||
private async findLandedTaskCommit(
|
||
task: Task,
|
||
options?: { preferEarliestOwnedCommit?: boolean },
|
||
): Promise<LandedTaskCommit | null> {
|
||
// Search strategies, tried in order of reliability:
|
||
// 1. mergeDetails.commitSha — already stored by the merger; verify it's
|
||
// reachable from HEAD before trusting it.
|
||
// 2. Fusion-Task-Lineage trailer — canonical immutable lineage marker.
|
||
// 3. Fusion-Task-Id trailer — legacy human task-id marker.
|
||
// 4. Subject grep — legacy/AI commits where the task ID lives in the
|
||
// subject line (e.g. `feat(FN-123): …`).
|
||
//
|
||
// (1) gives us the right sha even if the commit subject is exotic; (2)
|
||
// covers includeTaskIdInCommit=false setups where (3) would silently
|
||
// miss; (3) catches commits authored before the trailer was introduced.
|
||
|
||
// ── (1) Stored sha ────────────────────────────────────────────────────
|
||
const rebaseBaseSha = task.mergeDetails?.rebaseBaseSha;
|
||
const storedSha = task.mergeDetails?.commitSha;
|
||
if (storedSha) {
|
||
try {
|
||
await execAsync(
|
||
`git merge-base --is-ancestor ${shellQuote(storedSha)} HEAD`,
|
||
{ cwd: this.options.rootDir },
|
||
);
|
||
const { stdout } = await execAsync(
|
||
`git log -1 --format=%H%x1f%s%x1f%b ${shellQuote(storedSha)}`,
|
||
{ cwd: this.options.rootDir, maxBuffer: 1024 * 1024 },
|
||
);
|
||
const [sha, subject = "", body = ""] = stdout.trim().split("\x1f");
|
||
if (sha && commitOwnedByTask(task.id, task.lineageId, subject, body)) {
|
||
const commit: LandedTaskCommit = { sha, subject, rebaseBaseSha };
|
||
try {
|
||
const shortstat = await this.readShortstatForSha(sha, rebaseBaseSha);
|
||
if (shortstat) {
|
||
Object.assign(commit, shortstat);
|
||
}
|
||
} catch { /* stats are optional */ }
|
||
return commit;
|
||
}
|
||
} catch {
|
||
// Not reachable (rebased away, branch reset, etc.) — fall through.
|
||
}
|
||
}
|
||
|
||
const readLog = async (range: string, grepArg: string, fixedStrings: boolean) => {
|
||
const command = [
|
||
"git log",
|
||
"--format=%H%x1f%s",
|
||
"--max-count=20",
|
||
...(options?.preferEarliestOwnedCommit ? ["--reverse"] : []),
|
||
...(fixedStrings ? ["--fixed-strings"] : ["-E"]),
|
||
`--grep=${grepArg}`,
|
||
shellQuote(range),
|
||
].join(" ");
|
||
|
||
return execAsync(command, {
|
||
cwd: this.options.rootDir,
|
||
maxBuffer: 1024 * 1024,
|
||
});
|
||
};
|
||
|
||
// Search canonical lineage trailer, then legacy task-id trailer, then
|
||
// legacy subject fallback. All share bounded/full HEAD range resolution.
|
||
const search = async (grepArg: string, fixedStrings: boolean): Promise<string> => {
|
||
let out: string;
|
||
try {
|
||
const r = await readLog(
|
||
task.baseCommitSha ? `${task.baseCommitSha}..HEAD` : "HEAD",
|
||
grepArg,
|
||
fixedStrings,
|
||
);
|
||
out = r.stdout;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(
|
||
`Failed to read git log for landed commit lookup (${task.id}): ${errorMessage} — retrying with HEAD range`,
|
||
);
|
||
if (!task.baseCommitSha) return "";
|
||
const r = await readLog("HEAD", grepArg, fixedStrings);
|
||
out = r.stdout;
|
||
}
|
||
// Bounded range may exclude the landed commit when baseCommitSha was
|
||
// advanced past it; re-scan all of HEAD if empty.
|
||
if (!out.trim() && task.baseCommitSha) {
|
||
const r = await readLog("HEAD", grepArg, fixedStrings);
|
||
out = r.stdout;
|
||
}
|
||
return out;
|
||
};
|
||
|
||
// (2) Canonical lineage trailer.
|
||
let stdout = "";
|
||
if (task.lineageId) {
|
||
const lineagePattern = `^Fusion-Task-Lineage: ${task.lineageId}$`;
|
||
stdout = await search(shellQuote(lineagePattern), false);
|
||
}
|
||
|
||
// (3) Legacy task-id trailer.
|
||
if (!stdout.trim()) {
|
||
const trailerPattern = `^Fusion-Task-Id: ${task.id}$`;
|
||
stdout = await search(shellQuote(trailerPattern), false);
|
||
}
|
||
|
||
// (4) Subject grep fallback (legacy commits).
|
||
if (!stdout.trim()) {
|
||
stdout = await search(shellQuote(task.id), true);
|
||
}
|
||
|
||
const firstLine = stdout.trim().split("\n").find(Boolean);
|
||
if (!firstLine) return null;
|
||
|
||
const [sha, subject] = firstLine.split("\x1f");
|
||
if (!sha) return null;
|
||
|
||
const commit: LandedTaskCommit = { sha, subject, rebaseBaseSha };
|
||
try {
|
||
const shortstat = await this.readShortstatForSha(sha, rebaseBaseSha);
|
||
if (shortstat) {
|
||
Object.assign(commit, shortstat);
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(
|
||
`Failed to read shortstat for landed commit ${sha} (${task.id}): ${errorMessage} — continuing without stats`,
|
||
);
|
||
// Stats are useful for the task detail view but not required for recovery.
|
||
}
|
||
|
||
return commit;
|
||
}
|
||
|
||
private async findAlreadyMergedTaskCommit(input: {
|
||
taskId: string;
|
||
lineageId?: string;
|
||
repoDir: string;
|
||
baseBranch: string;
|
||
taskBranch?: string;
|
||
baseCommitSha?: string;
|
||
}) {
|
||
return findAlreadyMergedTaskCommit(input);
|
||
}
|
||
|
||
private async cleanupWorktreeOnly(task: Task): Promise<void> {
|
||
if (task.worktree && existsSync(task.worktree)) {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
await removeWorktree({
|
||
rootDir: this.options.rootDir,
|
||
worktreePath: task.worktree,
|
||
settings,
|
||
taskId: task.id,
|
||
reason: RemovalReason.SelfHealingReclaim,
|
||
});
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(
|
||
`Failed to remove worktree ${task.worktree} for ${task.id}: ${errorMessage} — non-fatal, cleanup can retry later`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
private async cleanupInterruptedMergeArtifacts(task: Task): Promise<void> {
|
||
if (task.worktree && existsSync(task.worktree)) {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
await removeWorktree({
|
||
rootDir: this.options.rootDir,
|
||
worktreePath: task.worktree,
|
||
settings,
|
||
taskId: task.id,
|
||
reason: RemovalReason.SelfHealingReclaim,
|
||
});
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(
|
||
`Failed to remove interrupted-merge worktree ${task.worktree} for ${task.id}: ${errorMessage} — non-fatal, cleanup can retry later`,
|
||
);
|
||
}
|
||
}
|
||
|
||
const branch = task.branch || canonicalFusionBranchName(task.id);
|
||
try {
|
||
await execAsync(`git branch -D ${shellQuote(branch)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 120_000,
|
||
});
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(
|
||
`Failed to delete interrupted-merge branch ${branch} for ${task.id}: ${errorMessage} — non-fatal`,
|
||
);
|
||
// Non-fatal; branch may be gone or still checked out.
|
||
}
|
||
}
|
||
|
||
private async runMaintenance(): Promise<void> {
|
||
if (this.maintenanceRunning) {
|
||
log.log("Maintenance cycle skipped — previous cycle still running");
|
||
return;
|
||
}
|
||
|
||
this.maintenanceRunning = true;
|
||
const startMs = Date.now();
|
||
this.maintenanceTickCounter++;
|
||
log.log("Maintenance cycle starting");
|
||
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
|
||
// Batch 1 — housekeeping (safe under pause: filesystem/db cleanup only)
|
||
const batch1Fns: Array<{ name: string; fn: () => Promise<unknown> }> = [
|
||
{ name: "prune-worktrees", fn: () => this.pruneWorktrees() },
|
||
{ name: "cleanup-orphans", fn: () => this.cleanupOrphans() },
|
||
{ name: "cleanup-orphaned-branches", fn: () => this.cleanupOrphanedBranches() },
|
||
{
|
||
name: "cleanup-old-chats",
|
||
fn: async () => {
|
||
const days = Number(settings.chatAutoCleanupDays ?? 0);
|
||
if (!Number.isFinite(days) || days <= 0) {
|
||
log.log("Maintenance batch 1 step \"cleanup-old-chats\" skipped — chatAutoCleanupDays is not enabled");
|
||
return;
|
||
}
|
||
if (!this.options.chatStore) {
|
||
log.log("Maintenance batch 1 step \"cleanup-old-chats\" skipped — ChatStore unavailable");
|
||
return;
|
||
}
|
||
const { sessionsDeleted, roomsDeleted } = this.options.chatStore.cleanupOldChats(days * 86_400_000);
|
||
log.log(`Maintenance batch 1 step "cleanup-old-chats" succeeded — sessions=${sessionsDeleted} rooms=${roomsDeleted}`);
|
||
},
|
||
},
|
||
{
|
||
name: "cleanup-old-mail",
|
||
fn: async () => {
|
||
const value = Number(settings.mailAutoCleanupDays ?? 0);
|
||
if (!Number.isFinite(value) || value <= 0) {
|
||
log.log(`Skipping cleanup-old-mail: setting=${String(settings.mailAutoCleanupDays ?? 0)}`);
|
||
return;
|
||
}
|
||
if (!this.options.messageStore) {
|
||
log.log("Skipping cleanup-old-mail: messageStore unavailable");
|
||
return;
|
||
}
|
||
const { messagesDeleted } = this.options.messageStore.cleanupOldMessages(value * 86_400_000);
|
||
log.log(`Maintenance batch 1 step "cleanup-old-mail" succeeded — messagesDeleted=${messagesDeleted}`);
|
||
},
|
||
},
|
||
{ name: "checkpoint-wal", fn: () => Promise.resolve(this.checkpointWal()) },
|
||
{ name: "enforce-worktree-cap", fn: () => this.enforceWorktreeCap() },
|
||
];
|
||
for (const fn of batch1Fns) {
|
||
try {
|
||
await fn.fn();
|
||
log.log(`Maintenance batch 1 step "${fn.name}" succeeded`);
|
||
} catch (stepErr) {
|
||
log.error(`Maintenance batch 1 step "${fn.name}" failed: ${stepErr instanceof Error ? stepErr.message : String(stepErr)}`);
|
||
}
|
||
}
|
||
|
||
const recoverySettings = await this.store.getSettings();
|
||
if (recoverySettings.globalPause || recoverySettings.enginePaused) {
|
||
log.log(
|
||
`Maintenance batch 2 skipped — ${
|
||
recoverySettings.globalPause ? "global pause" : "engine pause"
|
||
} is active`,
|
||
);
|
||
} else {
|
||
// Batch 2 — Task recovery (operations are independent of each other)
|
||
const batch2Fns: Array<{ name: string; fn: () => Promise<unknown> }> = [
|
||
{ name: "recover-completed-tasks", fn: () => this.recoverCompletedTasks() },
|
||
{ name: "recover-stranded-completed-todo", fn: () => this.recoverStrandedCompletedTodoTasks() },
|
||
{ name: "recover-stale-incomplete-review", fn: () => this.recoverStaleIncompleteReviewTasks() },
|
||
{ name: "recover-failed-pre-merge-steps", fn: () => this.recoverReviewTasksWithFailedPreMergeSteps() },
|
||
{ name: "recover-interrupted-merging", fn: () => this.recoverInterruptedMergingTasks() },
|
||
{ name: "recover-done-merge-metadata", fn: () => this.recoverDoneTaskMergeMetadata() },
|
||
{ name: "recover-stale-merging-status", fn: () => this.recoverStaleMergingStatus() },
|
||
{ name: "finalize-noop-review", fn: () => this.finalizeNoOpReviewTasks() },
|
||
{ name: "reconcile-done-task-integrity", fn: () => this.reconcileDoneTaskIntegrity() },
|
||
{ name: "reconcile-stale-merger-status", fn: () => this.reconcileStaleMergerStatus() },
|
||
{ name: "recover-mergeable-review", fn: () => this.recoverMergeableReviewTasks() },
|
||
{ name: "recover-merged-review", fn: () => this.recoverMergedReviewTasks() },
|
||
{ name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks() },
|
||
{ name: "recover-completion-handoff-limbo", fn: () => this.recoverCompletionHandoffLimbo() },
|
||
{ name: "recover-branch-misbound-in-review", fn: () => this.recoverBranchMisboundInReviewTasks() },
|
||
{ name: "recover-foreign-only-contamination-in-review", fn: () => this.recoverForeignOnlyContaminatedInReviewTasks() },
|
||
{ name: "recover-orphan-only-scope-violations", fn: () => this.recoverOrphanOnlyScopeViolations() },
|
||
{ name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks() },
|
||
{ name: "recover-misclassified-failures", fn: () => this.recoverMisclassifiedFailures() },
|
||
{ name: "recover-missing-worktree-review-failures", fn: () => this.recoverMissingWorktreeReviewFailures() },
|
||
{ name: "recover-no-progress-no-task-done", fn: () => this.recoverNoProgressNoTaskDoneFailures() },
|
||
{ name: "recover-partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures() },
|
||
{ name: "recover-orphaned-executions", fn: () => this.recoverOrphanedExecutions() },
|
||
{ name: "recover-approved-triage", fn: () => this.recoverApprovedTriageTasks() },
|
||
{ name: "recover-starved-refinement", fn: () => this.recoverStarvedRefinementTriageTasks() },
|
||
{ name: "recover-orphaned-planning", fn: () => this.recoverOrphanedPlanningTasks() },
|
||
{ name: "recover-ghost-review", fn: () => this.recoverGhostReviewTasks() },
|
||
{ name: "recover-orphaned-agents", fn: () => this.recoverOrphanedAgents() },
|
||
{ name: "recover-stale-heartbeat-runs", fn: () => this.recoverStaleHeartbeatRuns() },
|
||
{ name: "recover-running-on-inactive-tasks", fn: () => this.recoverAgentsRunningOnInactiveTasks() },
|
||
{ name: "recover-drifted-agent-task-links", fn: () => this.recoverDriftedAgentTaskLinks() },
|
||
{ name: "clear-stale-blocked-by", fn: () => this.clearStaleBlockedBy() },
|
||
{ name: "auto-rebound-paused-scope-decay", fn: () => this.autoReboundPausedScopeDecay() },
|
||
{ name: "auto-archive-meta-resolved", fn: () => this.autoArchiveResolvedMetaTasks() },
|
||
{ name: "auto-archive-meta-stalled", fn: () => this.autoArchiveStalledMetaTasks() },
|
||
{ name: "board-stall-auto-recovery", fn: () => this.runBoardStallAutoRecoverySweep() },
|
||
{ name: "reconcile-self-defeating-deps", fn: () => this.reconcileSelfDefeatingDependencies() },
|
||
{ name: "reclaim-pr-conflicts", fn: () => this.reclaimPrConflicts() },
|
||
{ name: "reclaim-self-owned-branch-conflicts", fn: () => this.reclaimSelfOwnedBranchConflicts() },
|
||
// FN-4962 ordering invariant: metadata reconcile must run before stale-active reclaim.
|
||
{ name: "reconcile-task-worktree-metadata", fn: () => this.reconcileTaskWorktreeMetadata() },
|
||
{ name: "reconcile-in-review-branch-rebind", fn: () => this.reconcileInReviewBranchRebind().then(() => undefined) },
|
||
{ name: "reclaim-stale-active-branches", fn: () => this.reclaimStaleActiveBranches() },
|
||
{ name: "surface-in-review-stalls", fn: () => this.surfaceInReviewStalls() },
|
||
{ name: "surface-in-review-stalled", fn: () => this.surfaceInReviewStalled() },
|
||
{ name: "surface-stale-paused-reviews", fn: () => this.surfaceStalePausedReviews() },
|
||
{ name: "surface-stale-paused-todos", fn: () => this.surfaceStalePausedTodos() },
|
||
{ name: "audit-no-commits-expected-candidates", fn: () => this.auditNoCommitsExpectedCandidates() },
|
||
];
|
||
for (const fn of batch2Fns) {
|
||
try {
|
||
await fn.fn();
|
||
log.log(`Maintenance batch 2 step "${fn.name}" succeeded`);
|
||
} catch (stepErr) {
|
||
log.error(`Maintenance batch 2 step "${fn.name}" failed: ${stepErr instanceof Error ? stepErr.message : String(stepErr)}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Batch 3 — Archive (runs after recovery so we don't archive recoverable tasks)
|
||
const batch3Fns: Array<{ name: string; fn: () => Promise<unknown> }> = [
|
||
{ name: "archive-stale-done", fn: () => this.archiveStaleDoneTasks() },
|
||
];
|
||
for (const fn of batch3Fns) {
|
||
try {
|
||
await fn.fn();
|
||
log.log(`Maintenance batch 3 step "${fn.name}" succeeded`);
|
||
} catch (stepErr) {
|
||
log.error(`Maintenance batch 3 step "${fn.name}" failed: ${stepErr instanceof Error ? stepErr.message : String(stepErr)}`);
|
||
}
|
||
}
|
||
|
||
const elapsedMs = Date.now() - startMs;
|
||
log.log(`Maintenance cycle completed in ${elapsedMs}ms`);
|
||
} finally {
|
||
this.maintenanceRunning = false;
|
||
}
|
||
}
|
||
|
||
// ── Auto-archive of stale done tasks ──────────────────────────────
|
||
|
||
/**
|
||
* Auto-archive done tasks older than the project retention setting so the
|
||
* active task database does not accumulate completed task payloads forever.
|
||
* Archived task metadata is retained in the separate archive database and can
|
||
* be restored by unarchiving.
|
||
*/
|
||
private static readonly AUTO_ARCHIVE_AFTER_MS = 48 * 60 * 60 * 1000;
|
||
|
||
async archiveStaleDoneTasks(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
const doneAutoArchiveDaysRaw = settings.doneAutoArchiveDays;
|
||
const doneAutoArchiveDaysNumber = Number(doneAutoArchiveDaysRaw);
|
||
const doneAutoArchiveDays =
|
||
Number.isFinite(doneAutoArchiveDaysNumber) && Number.isInteger(doneAutoArchiveDaysNumber) && doneAutoArchiveDaysNumber > 0
|
||
? doneAutoArchiveDaysNumber
|
||
: 0;
|
||
if (settings.autoArchiveDoneTasksEnabled === false && doneAutoArchiveDays === 0) {
|
||
return 0;
|
||
}
|
||
const archiveAfterMs = doneAutoArchiveDays > 0
|
||
? doneAutoArchiveDays * 24 * 60 * 60 * 1000
|
||
: (settings.autoArchiveDoneAfterMs ?? SelfHealingManager.AUTO_ARCHIVE_AFTER_MS);
|
||
if (!Number.isFinite(archiveAfterMs) || archiveAfterMs <= 0) {
|
||
return 0;
|
||
}
|
||
|
||
// Slim listing — we only need id/column/columnMovedAt/updatedAt to decide
|
||
// staleness. Pulling full task payloads (logs, comments, steps) here used
|
||
// to drag in tens of MB on busy boards and stalled the maintenance loop.
|
||
const tasks = await this.store.listTasks({ slim: true, includeArchived: false });
|
||
const now = Date.now();
|
||
const cutoff = now - archiveAfterMs;
|
||
|
||
// Build a set of task IDs that have at least one *active* dependent —
|
||
// i.e., another task in triage/todo/in-progress/in-review that lists
|
||
// this ID in its `dependencies`. Archiving such a task wipes
|
||
// `.fusion/tasks/{id}/` on disk, which downstream agents are told they
|
||
// may read for sibling-spec context (executor prompt). Done/archived
|
||
// dependents have already consumed the spec and don't block.
|
||
const tasksWithActiveDependents = new Set<string>();
|
||
for (const t of tasks) {
|
||
if (t.column === "done" || t.column === "archived") continue;
|
||
for (const depId of t.dependencies ?? []) {
|
||
tasksWithActiveDependents.add(depId);
|
||
}
|
||
}
|
||
|
||
const stale = tasks.filter((t) => {
|
||
if (t.column !== "done") return false;
|
||
// Prefer columnMovedAt (when the task entered done); fall back to updatedAt
|
||
// for legacy tasks that lack the field.
|
||
const ts = t.columnMovedAt || t.updatedAt;
|
||
const movedAt = ts ? Date.parse(ts) : NaN;
|
||
if (!Number.isFinite(movedAt)) return false;
|
||
if (movedAt >= cutoff) return false;
|
||
if (tasksWithActiveDependents.has(t.id)) {
|
||
log.log(`Skipping auto-archive of ${t.id}: has active dependents`);
|
||
return false;
|
||
}
|
||
return true;
|
||
});
|
||
|
||
if (stale.length === 0) return 0;
|
||
|
||
log.log(`Auto-archiving ${stale.length} done task(s) older than ${archiveAfterMs}ms`);
|
||
|
||
let archived = 0;
|
||
const thresholdDays = Math.floor(archiveAfterMs / 86_400_000);
|
||
for (const task of stale) {
|
||
try {
|
||
await this.store.archiveTaskAndCleanup(task.id);
|
||
archived++;
|
||
const ts = task.columnMovedAt || task.updatedAt;
|
||
const movedAt = ts ? Date.parse(ts) : NaN;
|
||
const ageDays = Number.isFinite(movedAt) ? Math.floor((now - movedAt) / 86_400_000) : 0;
|
||
log.log(`auto-archive: archived ${task.id} (age ${ageDays}d, threshold ${thresholdDays}d)`);
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to auto-archive ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (archived > 0) {
|
||
log.log(`Auto-archived ${archived} stale done task(s)`);
|
||
}
|
||
return archived;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Auto-archive sweep failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
// ── Completed task recovery ──────────────────────────────────────
|
||
|
||
/**
|
||
* Recover tasks stuck in in-progress whose work is actually complete.
|
||
*
|
||
* This catches tasks where the agent called task_done() (all steps marked
|
||
* done, summary written) but the session was killed before the executor
|
||
* could call moveTask("in-review"). Without this, such tasks sit
|
||
* indefinitely in in-progress with no active session.
|
||
*
|
||
* @returns Number of tasks recovered
|
||
*/
|
||
async recoverCompletedTasks(): Promise<number> {
|
||
const recoverFn = this.options.recoverCompletedTask;
|
||
if (!recoverFn) return 0;
|
||
|
||
try {
|
||
const tasks = await this.store.listTasks({ column: "in-progress", slim: true });
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
|
||
const stuckCompleted = tasks.filter((t) =>
|
||
t.column === "in-progress" &&
|
||
!t.paused &&
|
||
!executingIds.has(t.id) &&
|
||
t.steps.length > 0 &&
|
||
t.steps.every((s) => s.status === "done" || s.status === "skipped"),
|
||
);
|
||
|
||
if (stuckCompleted.length === 0) return 0;
|
||
|
||
log.warn(`Found ${stuckCompleted.length} completed task(s) stuck in in-progress`);
|
||
|
||
let recovered = 0;
|
||
for (const task of stuckCompleted) {
|
||
// Re-check in-flight state inside the loop. The initial filter used a
|
||
// snapshot taken before any awaits; another path (executor resume,
|
||
// task:moved dispatch) may have claimed the task in between.
|
||
const latestExecutingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
if (latestExecutingIds.has(task.id)) {
|
||
log.log(`${task.id} started executing concurrently — skipping recovery this cycle`);
|
||
continue;
|
||
}
|
||
log.log(`Recovering completed task ${task.id}: ${task.title || task.description?.slice(0, 60) || "(untitled)"}`);
|
||
const success = await recoverFn(task);
|
||
if (success) recovered++;
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} completed task(s) → in-review`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Completed task recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover todo tasks whose implementation steps are fully complete.
|
||
*
|
||
* This closes the lifecycle gap where self-healing paths can requeue a
|
||
* finished task back to todo with progress preserved; these tasks should be
|
||
* promoted via normal transition flow instead of waiting for re-execution.
|
||
*/
|
||
async recoverStrandedCompletedTodoTasks(): Promise<number> {
|
||
const recoverFn = this.options.recoverCompletedTask;
|
||
if (!recoverFn) return 0;
|
||
|
||
try {
|
||
const tasks = await this.store.listTasks({ column: "todo", slim: true });
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
|
||
const stranded = tasks.filter((task) => {
|
||
if (task.column !== "todo" || task.paused) return false;
|
||
if (executingIds.has(task.id)) return false;
|
||
if (task.steps.length === 0 || !task.steps.every((s) => s.status === "done" || s.status === "skipped")) return false;
|
||
if (task.error) return false;
|
||
if (task.status && STRANDED_COMPLETED_TODO_ACTIVE_STATUSES.has(task.status)) return false;
|
||
if (task.reviewState?.refreshStatus === "refreshing") return false;
|
||
return true;
|
||
});
|
||
|
||
if (stranded.length === 0) return 0;
|
||
|
||
log.warn(`Found ${stranded.length} completed task(s) stranded in todo`);
|
||
|
||
let recovered = 0;
|
||
for (const task of stranded) {
|
||
const latestExecutingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
if (latestExecutingIds.has(task.id)) {
|
||
log.log(`${task.id} started executing concurrently — skipping stranded todo recovery this cycle`);
|
||
continue;
|
||
}
|
||
|
||
const success = await recoverFn(task);
|
||
if (success) recovered++;
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} stranded completed todo task(s)`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Stranded completed todo task recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Clear stale transient merge statuses when no active merger owns the task.
|
||
*
|
||
* @returns Number of tasks unblocked by clearing stale status
|
||
*/
|
||
async recoverStaleMergingStatus(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
|
||
const minAgeMs = this.options.staleMergingStatusMinAgeMs ?? DEFAULT_STALE_MERGING_STATUS_MIN_AGE_MS;
|
||
if (!Number.isFinite(minAgeMs) || minAgeMs <= 0) return 0;
|
||
|
||
const now = Date.now();
|
||
const activeMergeTaskId = this.options.getActiveMergeTaskId?.() ?? null;
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const stale = tasks.filter((task) => {
|
||
if (task.column !== "in-review" || task.paused) return false;
|
||
if (!task.status || (task.status !== "merging" && task.status !== "merging-pr")) return false;
|
||
if (activeMergeTaskId && activeMergeTaskId === task.id) return false;
|
||
|
||
const updatedAtMs = task.updatedAt ? Date.parse(task.updatedAt) : Number.NaN;
|
||
if (!Number.isFinite(updatedAtMs)) return false;
|
||
return now - updatedAtMs >= minAgeMs;
|
||
});
|
||
|
||
if (stale.length === 0) return 0;
|
||
|
||
let recovered = 0;
|
||
for (const task of stale) {
|
||
const previousStatus = task.status;
|
||
try {
|
||
log.warn(`Clearing stale merge status for ${task.id}: ${previousStatus}`);
|
||
await this.store.updateTask(task.id, { status: null });
|
||
this.options.clearMergeActive?.(task.id);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovered: cleared stale '${previousStatus}' status (no active merger)`,
|
||
);
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to clear stale merge status for ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Stale merging status recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
async reclaimPrConflicts(): Promise<number> {
|
||
const tasks = await this.store.listTasks({ slim: true });
|
||
const candidates = tasks.filter((task) => {
|
||
const prList = task.prInfos ?? (task.prInfo ? [task.prInfo] : []);
|
||
return prList.some((pr) => pr.mergeable === "conflicting");
|
||
});
|
||
let reclaimed = 0;
|
||
for (const task of candidates) {
|
||
const result = await this.reclaimPrConflictForTask(task.id);
|
||
if (result.outcome !== "skipped") {
|
||
reclaimed++;
|
||
}
|
||
}
|
||
return reclaimed;
|
||
}
|
||
|
||
async reclaimPrConflictForTask(taskId: string): Promise<{ outcome: "reclaimed" | "stale-resolved" | "tip-already-merged" | "paused-unrecoverable" | "skipped"; reason?: string; perPr?: Array<{ number: number; outcome: "reclaimed" | "stale-resolved" | "tip-already-merged" | "paused-unrecoverable" | "skipped"; reason?: string }> }> {
|
||
const task = await this.store.getTask(taskId);
|
||
if (!task) return { outcome: "skipped", reason: "task-not-found" };
|
||
const conflictingPrs = (task.prInfos ?? (task.prInfo ? [task.prInfo] : [])).filter((pr) => pr.mergeable === "conflicting");
|
||
if (conflictingPrs.length === 0) {
|
||
return { outcome: "skipped", reason: "no-conflicting-pr" };
|
||
}
|
||
const withPerPr = (result: { outcome: "reclaimed" | "stale-resolved" | "tip-already-merged" | "paused-unrecoverable" | "skipped"; reason?: string }) => {
|
||
if (conflictingPrs.length <= 1) {
|
||
return result;
|
||
}
|
||
return {
|
||
...result,
|
||
perPr: conflictingPrs.map((pr) => ({ number: pr.number, outcome: result.outcome, reason: result.reason })),
|
||
};
|
||
};
|
||
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return withPerPr({ outcome: "skipped", reason: "engine-paused" });
|
||
if (!task.branch || !task.worktree) return withPerPr({ outcome: "skipped", reason: "missing-branch-or-worktree" });
|
||
if (task.userPaused) return withPerPr({ outcome: "skipped", reason: "user-paused" });
|
||
if (task.checkedOutBy) return withPerPr({ outcome: "skipped", reason: "checked-out" });
|
||
if (task.pausedReason === "worktrunk_operation_failed") return withPerPr({ outcome: "skipped", reason: "worktrunk-paused" });
|
||
if (activeSessionRegistry.isPathActive(task.worktree)) return withPerPr({ outcome: "skipped", reason: "active-session" });
|
||
if (!await isUsableTaskWorktree(this.options.rootDir, task.worktree)) return withPerPr({ outcome: "skipped", reason: "unusable-worktree" });
|
||
|
||
try {
|
||
const inspection = await inspectBranchConflict({
|
||
repoDir: this.options.rootDir,
|
||
branchName: task.branch,
|
||
conflictingWorktreePath: task.worktree,
|
||
requestingTaskId: task.id,
|
||
ownerTaskId: task.id,
|
||
startPoint: task.baseCommitSha ?? task.mergeDetails?.mergeTargetBranch ?? "main",
|
||
});
|
||
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal-pr-conflict", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "reclaim-pr-conflicts",
|
||
});
|
||
|
||
if (inspection.kind === "stale") {
|
||
await auditor.database({ type: "task:pr-conflict-reclaim", target: task.id, metadata: { outcome: "skipped", reason: "stale" } });
|
||
return withPerPr({ outcome: "skipped", reason: "stale" });
|
||
}
|
||
if (inspection.kind === "stale-resolved") {
|
||
await this.store.updateTask(task.id, { worktree: null, branch: null, baseCommitSha: null });
|
||
await auditor.database({ type: "task:pr-conflict-reclaim", target: task.id, metadata: { outcome: "stale-resolved" } });
|
||
return withPerPr({ outcome: "stale-resolved" });
|
||
}
|
||
if (inspection.kind === "tip-already-merged") {
|
||
await this.reclaimSelfOwnedBranchConflicts();
|
||
await auditor.database({ type: "task:pr-conflict-reclaim", target: task.id, metadata: { outcome: "tip-already-merged" } });
|
||
return withPerPr({ outcome: "tip-already-merged" });
|
||
}
|
||
if (inspection.kind === "live-foreign") {
|
||
throw inspection.error;
|
||
}
|
||
|
||
const inProgressCandidates = await this.store.listTasks({ column: "in-progress", slim: true });
|
||
const inProgressByWorktree = new Map<string, string>();
|
||
for (const inProgressTask of inProgressCandidates) {
|
||
if (inProgressTask.worktree) inProgressByWorktree.set(inProgressTask.worktree, inProgressTask.id);
|
||
}
|
||
const wasPausedBranchConflict = task.paused === true && task.pausedReason === "branch-conflict-unrecoverable";
|
||
if (inspection.kind === "fully-subsumed") {
|
||
const taskIdUpper = task.id.toUpperCase();
|
||
const branchOwnerTaskId = deriveTaskIdFromFusionBranch(task.branch);
|
||
const activeOwner = inProgressByWorktree.get(inspection.livePath);
|
||
const ownedByOtherInProgressTask = Boolean(activeOwner && activeOwner !== task.id);
|
||
const canAutoReclaimLiveZero = branchOwnerTaskId !== null && branchOwnerTaskId === taskIdUpper && !ownedByOtherInProgressTask;
|
||
if (canAutoReclaimLiveZero) {
|
||
await removeWorktree({ rootDir: this.options.rootDir, worktreePath: inspection.livePath, settings, taskId: task.id, reason: RemovalReason.SelfHealingBranchConflict });
|
||
await execAsync("git worktree prune", { cwd: this.options.rootDir, timeout: 120_000, maxBuffer: 10 * 1024 * 1024 });
|
||
await execAsync(`git branch -D ${JSON.stringify(task.branch)}`, { cwd: this.options.rootDir, timeout: 120_000, maxBuffer: 10 * 1024 * 1024 });
|
||
await this.store.updateTask(task.id, { worktree: null, branch: null, paused: false, pausedReason: undefined, status: null, error: null });
|
||
await auditor.database({ type: "task:pr-conflict-reclaim", target: task.id, metadata: { outcome: "reclaimed", mode: "fully-subsumed", recoveredFromPaused: wasPausedBranchConflict } });
|
||
return withPerPr({ outcome: "reclaimed" });
|
||
}
|
||
}
|
||
|
||
await this.store.updateTask(task.id, {
|
||
worktree: inspection.livePath,
|
||
branch: task.branch,
|
||
paused: false,
|
||
pausedReason: undefined,
|
||
status: null,
|
||
error: null,
|
||
});
|
||
if (task.column === "in-review") {
|
||
await this.store.moveTask(task.id, "todo", {
|
||
moveSource: "engine",
|
||
preserveWorktree: true,
|
||
preserveProgress: true,
|
||
preserveResumeState: true,
|
||
});
|
||
}
|
||
await auditor.database({ type: "task:pr-conflict-reclaim", target: task.id, metadata: { outcome: "reclaimed", mode: inspection.kind } });
|
||
return withPerPr({ outcome: "reclaimed" });
|
||
} catch (err: unknown) {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
const patchPath = await preserveWorktreeChanges(this.options.rootDir, task.worktree, task.id);
|
||
if (patchPath) {
|
||
await this.store.logEntry(task.id, `Preserved uncommitted worktree changes before pause: ${patchPath}`);
|
||
}
|
||
const dispatcher = this.options.autoRecoveryDispatcher ?? new AutoRecoveryDispatcher({
|
||
taskStore: this.store,
|
||
auditEmitter: createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "reclaim-pr-conflicts",
|
||
}),
|
||
});
|
||
const decision = await dispatcher.dispatch({
|
||
class: "branch-conflict-unrecoverable",
|
||
taskId: task.id,
|
||
pausedReason: "branch-conflict-unrecoverable",
|
||
evidence: { branchName: task.branch, worktreePath: task.worktree },
|
||
}, {
|
||
task,
|
||
retryCount: task.recoveryRetryCount ?? 0,
|
||
settings: (await this.store.getSettings()).autoRecovery ?? { mode: "deterministic-only", maxRetries: 3 },
|
||
});
|
||
if (decision.action === "pause") {
|
||
await this.store.updateTask(task.id, {
|
||
status: "failed",
|
||
error: `Task branch conflict: ${task.branch} is not safely reclaimable (${message})`,
|
||
paused: true,
|
||
pausedReason: "branch-conflict-unrecoverable",
|
||
});
|
||
await this.store.moveTask(task.id, "in-review");
|
||
await this.store.logEntry(task.id, `Auto-recovery failed: branch conflict unrecoverable — ${message}`);
|
||
}
|
||
return withPerPr({ outcome: "paused-unrecoverable", reason: message });
|
||
}
|
||
}
|
||
|
||
/**
|
||
* STANDING: do not auto-discard stranded commits. Reclaim preserves commits;
|
||
* unrecoverable conflicts are escalated for human review.
|
||
*
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async reclaimSelfOwnedBranchConflicts(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const todoCandidates = await this.store.listTasks({ column: "todo", slim: true });
|
||
const inProgressCandidates = await this.store.listTasks({ column: "in-progress", slim: true });
|
||
const inProgressByWorktree = new Map<string, string>();
|
||
for (const inProgressTask of inProgressCandidates) {
|
||
if (inProgressTask.worktree) {
|
||
inProgressByWorktree.set(inProgressTask.worktree, inProgressTask.id);
|
||
}
|
||
}
|
||
const inReviewPausedCandidates = (await this.store.listTasks({ column: "in-review", slim: true }))
|
||
.filter((task) => task.paused === true && task.pausedReason === "branch-conflict-unrecoverable");
|
||
const candidates = [...todoCandidates, ...inProgressCandidates, ...inReviewPausedCandidates];
|
||
|
||
const activeTaskIds = new Set<string>();
|
||
if (this.options.agentStore) {
|
||
try {
|
||
const activeRuns = await this.options.agentStore.listActiveHeartbeatRuns();
|
||
const activeWindowMs = RUNNING_ON_INACTIVE_TASK_STALE_RUN_MS;
|
||
const now = Date.now();
|
||
for (const run of activeRuns) {
|
||
const startedAtMs = Date.parse(run.startedAt ?? "");
|
||
if (!Number.isFinite(startedAtMs) || now - startedAtMs > activeWindowMs) continue;
|
||
const taskId = run.contextSnapshot && typeof run.contextSnapshot.taskId === "string"
|
||
? run.contextSnapshot.taskId.toUpperCase()
|
||
: null;
|
||
if (taskId) activeTaskIds.add(taskId);
|
||
}
|
||
} catch (err: unknown) {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
log.warn(`Unable to enumerate active heartbeat runs for self-owned branch reclaim sweep: ${message}`);
|
||
}
|
||
}
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
if (task.checkedOutBy || activeTaskIds.has(task.id.toUpperCase()) || !task.branch || !task.worktree) continue;
|
||
if (task.userPaused) continue;
|
||
if (task.pausedReason === "worktrunk_operation_failed") {
|
||
log.log(`[self-healing] skipping worktrunk-paused task ${task.id}`);
|
||
continue;
|
||
}
|
||
// FN-4811 follow-up (FN-4819): defer reclaim when the worktree is currently bound
|
||
// to a live executor/merger/step session. Without this, the sweep tries to
|
||
// `removeWorktree` and trips the active-session gate, which throws, which the outer
|
||
// catch escalates to AutoRecoveryDispatcher with class "branch-conflict-unrecoverable".
|
||
// That escalation marks the task `failed + paused`, even though the active session
|
||
// is making real progress. The right behavior is to skip this task this sweep and
|
||
// let the session complete — the reclaim will retry on a later sweep when no one
|
||
// is using the worktree.
|
||
if (activeSessionRegistry.isPathActive(task.worktree)) {
|
||
log.log(`[self-healing] deferring reclaim for ${task.id}: worktree ${task.worktree} has active session`);
|
||
continue;
|
||
}
|
||
if (!await isUsableTaskWorktree(this.options.rootDir, task.worktree)) continue;
|
||
|
||
try {
|
||
const inspection = await inspectBranchConflict({
|
||
repoDir: this.options.rootDir,
|
||
branchName: task.branch,
|
||
conflictingWorktreePath: task.worktree,
|
||
requestingTaskId: task.id,
|
||
ownerTaskId: task.id,
|
||
startPoint: task.baseCommitSha ?? task.mergeDetails?.mergeTargetBranch ?? "main",
|
||
});
|
||
|
||
if (inspection.kind === "stale") {
|
||
continue;
|
||
}
|
||
if (inspection.kind === "stale-resolved") {
|
||
await this.store.updateTask(task.id, {
|
||
worktree: null,
|
||
branch: null,
|
||
baseCommitSha: null,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[recovery] cache-invalidate ${task.id} branch=${task.branch ?? "?"} reason=stale-resolved-no-live-ref-or-mapping`,
|
||
);
|
||
continue;
|
||
}
|
||
if (inspection.kind === "tip-already-merged") {
|
||
const branchName = task.branch;
|
||
let reclaimedCleanly = false;
|
||
try {
|
||
if (inspection.livePath && existsSync(inspection.livePath)) {
|
||
await removeWorktree({
|
||
rootDir: this.options.rootDir,
|
||
worktreePath: inspection.livePath,
|
||
settings,
|
||
taskId: task.id,
|
||
reason: RemovalReason.SelfHealingBranchConflict,
|
||
});
|
||
}
|
||
// Branch-level reclaim remains active in worktrunk mode; this is
|
||
// idempotent git metadata cleanup, not layout ownership.
|
||
// FN-4742: keep native prune; see WorktreeBackend.prune docs
|
||
await execAsync("git worktree prune", {
|
||
cwd: this.options.rootDir,
|
||
timeout: 120_000,
|
||
maxBuffer: 10 * 1024 * 1024,
|
||
});
|
||
await execAsync(`git branch -D ${JSON.stringify(branchName)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 120_000,
|
||
maxBuffer: 10 * 1024 * 1024,
|
||
});
|
||
|
||
await this.store.updateTask(task.id, {
|
||
worktree: null,
|
||
branch: null,
|
||
baseCommitSha: null,
|
||
paused: false,
|
||
pausedReason: undefined,
|
||
status: null,
|
||
error: null,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[recovery] tip-already-merged ${task.id} branch=${branchName} tip=${inspection.tipSha.slice(0, 12)} integrationRef=${inspection.integrationRef} reason=stale-cached-metadata-ghost-conflict`,
|
||
);
|
||
|
||
if (task.column === "in-review") {
|
||
await this.store.moveTask(task.id, "todo", {
|
||
moveSource: "engine",
|
||
preserveProgress: true,
|
||
preserveResumeState: true,
|
||
});
|
||
}
|
||
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "tip-already-merged",
|
||
});
|
||
await auditor.git({
|
||
type: "branch:auto-reclaim",
|
||
target: branchName,
|
||
metadata: {
|
||
taskId: task.id,
|
||
branch: branchName,
|
||
worktreePath: inspection.livePath,
|
||
existingTipSha: inspection.tipSha,
|
||
integrationRef: inspection.integrationRef,
|
||
trigger: "self-healing-sweep-ghost-conflict",
|
||
},
|
||
});
|
||
} catch (auditErr: unknown) {
|
||
log.warn(`Failed to write tip-already-merged run-audit event for ${task.id}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
|
||
}
|
||
|
||
recovered++;
|
||
reclaimedCleanly = true;
|
||
} catch (tipMergedErr: unknown) {
|
||
const message = tipMergedErr instanceof Error ? tipMergedErr.message : String(tipMergedErr);
|
||
await this.store.logEntry(task.id, `Auto-recovery warning: tip-already-merged cleanup failed — ${message}`);
|
||
log.warn(`Failed tip-already-merged cleanup for ${task.id}: ${message}`);
|
||
}
|
||
|
||
if (reclaimedCleanly) {
|
||
continue;
|
||
}
|
||
throw new Error(`tip-already-merged cleanup failed for ${task.id}`);
|
||
}
|
||
if (inspection.kind === "live-foreign") {
|
||
throw inspection.error;
|
||
}
|
||
|
||
const wasPausedBranchConflict = task.paused === true && task.pausedReason === "branch-conflict-unrecoverable";
|
||
|
||
if (inspection.kind === "fully-subsumed") {
|
||
const taskIdUpper = task.id.toUpperCase();
|
||
const branchOwnerTaskId = deriveTaskIdFromFusionBranch(task.branch);
|
||
const activeOwner = inProgressByWorktree.get(inspection.livePath);
|
||
const ownedByOtherInProgressTask = Boolean(activeOwner && activeOwner !== task.id);
|
||
const canAutoReclaimLiveZero =
|
||
branchOwnerTaskId !== null &&
|
||
branchOwnerTaskId === taskIdUpper &&
|
||
!activeTaskIds.has(taskIdUpper) &&
|
||
!ownedByOtherInProgressTask;
|
||
|
||
if (canAutoReclaimLiveZero) {
|
||
let reclaimedCleanly = false;
|
||
try {
|
||
await removeWorktree({
|
||
rootDir: this.options.rootDir,
|
||
worktreePath: inspection.livePath,
|
||
settings,
|
||
taskId: task.id,
|
||
reason: RemovalReason.SelfHealingBranchConflict,
|
||
});
|
||
// Branch-level reclaim remains active in worktrunk mode; this is
|
||
// idempotent git metadata cleanup, not layout ownership.
|
||
// FN-4742: keep native prune; see WorktreeBackend.prune docs
|
||
await execAsync("git worktree prune", {
|
||
cwd: this.options.rootDir,
|
||
timeout: 120_000,
|
||
maxBuffer: 10 * 1024 * 1024,
|
||
});
|
||
await execAsync(`git branch -D ${JSON.stringify(task.branch)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 120_000,
|
||
maxBuffer: 10 * 1024 * 1024,
|
||
});
|
||
|
||
await this.store.updateTask(task.id, {
|
||
worktree: null,
|
||
branch: null,
|
||
paused: false,
|
||
pausedReason: undefined,
|
||
status: null,
|
||
error: null,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[recovery] reclaim-live-zero-commits ${task.id} branch=${task.branch} worktree=${inspection.livePath} tip=${inspection.tipSha.slice(0, 12)} reason=zero-unique-commits-vs-main`,
|
||
);
|
||
|
||
if (task.column === "in-review") {
|
||
await this.store.moveTask(task.id, "todo", {
|
||
moveSource: "engine",
|
||
preserveProgress: true,
|
||
preserveResumeState: true,
|
||
});
|
||
}
|
||
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "reclaim-live-zero-commits",
|
||
});
|
||
await auditor.git({
|
||
type: "branch:auto-reclaim",
|
||
target: task.branch,
|
||
metadata: {
|
||
taskId: task.id,
|
||
branch: task.branch,
|
||
worktreePath: inspection.livePath,
|
||
existingTipSha: inspection.tipSha,
|
||
strandedCommitCount: 0,
|
||
subsumed: true,
|
||
recoveredFromPaused: wasPausedBranchConflict,
|
||
previousPausedReason: wasPausedBranchConflict ? task.pausedReason : null,
|
||
trigger: "self-healing-sweep-live-zero",
|
||
},
|
||
});
|
||
} catch (auditErr: unknown) {
|
||
log.warn(`Failed to write branch:auto-reclaim run-audit event for ${task.id}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
|
||
}
|
||
|
||
recovered++;
|
||
reclaimedCleanly = true;
|
||
} catch (reclaimErr: unknown) {
|
||
const reclaimMessage = reclaimErr instanceof Error ? reclaimErr.message : String(reclaimErr);
|
||
await this.store.logEntry(task.id, `Auto-recovery warning: reclaim-live-zero-commits failed — ${reclaimMessage}`);
|
||
log.warn(`Failed reclaim-live-zero-commits for ${task.id}: ${reclaimMessage}`);
|
||
}
|
||
|
||
if (reclaimedCleanly) {
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
|
||
const preservedCommitCount = inspection.kind === "fully-subsumed"
|
||
? 0
|
||
: inspection.taskAttributedCommitCount;
|
||
await this.store.updateTask(task.id, {
|
||
worktree: inspection.livePath,
|
||
branch: task.branch,
|
||
paused: false,
|
||
pausedReason: undefined,
|
||
status: null,
|
||
error: null,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[recovery] ${wasPausedBranchConflict ? "reclaim-paused-review" : "reclaim-self-owned"} ${task.id} at ${inspection.livePath} (${preservedCommitCount} commits preserved, tip ${inspection.tipSha.slice(0, 12)})`,
|
||
);
|
||
|
||
if (task.column === "in-review") {
|
||
await this.store.moveTask(task.id, "todo", {
|
||
moveSource: "engine",
|
||
preserveWorktree: true,
|
||
preserveProgress: true,
|
||
preserveResumeState: true,
|
||
});
|
||
}
|
||
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: wasPausedBranchConflict ? "reclaim-paused-review" : "reclaim-self-owned-branch-conflicts",
|
||
});
|
||
await auditor.git({
|
||
type: "branch:auto-reclaim",
|
||
target: task.branch,
|
||
metadata: {
|
||
taskId: task.id,
|
||
branch: task.branch,
|
||
worktreePath: inspection.livePath,
|
||
existingTipSha: inspection.tipSha,
|
||
strandedCommitCount: inspection.kind === "fully-subsumed" ? 0 : inspection.strandedCommits.length,
|
||
subsumed: inspection.kind === "fully-subsumed",
|
||
recoveredFromPaused: wasPausedBranchConflict,
|
||
previousPausedReason: wasPausedBranchConflict ? task.pausedReason : null,
|
||
trigger: "self-healing-sweep",
|
||
},
|
||
});
|
||
} catch (auditErr: unknown) {
|
||
log.warn(`Failed to write branch:auto-reclaim run-audit event for ${task.id}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
|
||
}
|
||
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
const patchPath = await preserveWorktreeChanges(this.options.rootDir, task.worktree, task.id);
|
||
if (patchPath) {
|
||
await this.store.logEntry(task.id, `Preserved uncommitted worktree changes before pause: ${patchPath}`);
|
||
}
|
||
const dispatcher = this.options.autoRecoveryDispatcher ?? new AutoRecoveryDispatcher({
|
||
taskStore: this.store,
|
||
auditEmitter: createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "reclaim-self-owned-branch-conflicts",
|
||
}),
|
||
});
|
||
const decision = await dispatcher.dispatch({
|
||
class: "branch-conflict-unrecoverable",
|
||
taskId: task.id,
|
||
pausedReason: "branch-conflict-unrecoverable",
|
||
evidence: {
|
||
branchName: task.branch,
|
||
worktreePath: task.worktree,
|
||
},
|
||
}, {
|
||
task,
|
||
retryCount: task.recoveryRetryCount ?? 0,
|
||
settings: (await this.store.getSettings()).autoRecovery ?? { mode: "deterministic-only", maxRetries: 3 },
|
||
});
|
||
if (decision.action === "pause") {
|
||
await this.store.updateTask(task.id, {
|
||
status: "failed",
|
||
error: `Task branch conflict: ${task.branch} is not safely reclaimable (${message})`,
|
||
paused: true,
|
||
pausedReason: "branch-conflict-unrecoverable",
|
||
});
|
||
await this.store.moveTask(task.id, "in-review");
|
||
await this.store.logEntry(task.id, `Auto-recovery failed: branch conflict unrecoverable — ${message}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Reclaimed ${recovered} self-owned branch conflict task(s)`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Self-owned branch conflict reclaim sweep failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
async reclaimStaleActiveBranches(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
|
||
const activeTaskIds = new Set<string>();
|
||
if (this.options.agentStore) {
|
||
try {
|
||
const activeRuns = await this.options.agentStore.listActiveHeartbeatRuns();
|
||
const activeWindowMs = RUNNING_ON_INACTIVE_TASK_STALE_RUN_MS;
|
||
const now = Date.now();
|
||
for (const run of activeRuns) {
|
||
const startedAtMs = Date.parse(run.startedAt ?? "");
|
||
if (!Number.isFinite(startedAtMs) || now - startedAtMs > activeWindowMs) continue;
|
||
const taskId = run.contextSnapshot && typeof run.contextSnapshot.taskId === "string"
|
||
? run.contextSnapshot.taskId.toUpperCase()
|
||
: null;
|
||
if (taskId) activeTaskIds.add(taskId);
|
||
}
|
||
} catch (err: unknown) {
|
||
const message = err instanceof Error ? err.message : String(err);
|
||
log.warn(`Unable to enumerate active heartbeat runs for stale-active branch reclaim sweep: ${message}`);
|
||
}
|
||
}
|
||
|
||
const branchesRaw = String(execSync("git branch --list 'fusion/*'", {
|
||
cwd: this.options.rootDir,
|
||
encoding: "utf-8",
|
||
stdio: ["pipe", "pipe", "pipe"],
|
||
}) || "");
|
||
const branches = branchesRaw
|
||
.split("\n")
|
||
.map((line) => line.replace(/^\*\s*/, "").trim())
|
||
.filter(Boolean);
|
||
if (branches.length === 0) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ slim: true, includeArchived: true });
|
||
const taskById = new Map(tasks.map((task) => [task.id.toUpperCase(), task]));
|
||
|
||
let reclaimed = 0;
|
||
for (const branch of branches) {
|
||
const derivedTaskId = this.deriveTaskIdFromFusionBranch(branch);
|
||
if (!derivedTaskId) continue;
|
||
|
||
const task = taskById.get(derivedTaskId.toUpperCase());
|
||
if (!task || task.column === "archived" || task.checkedOutBy || task.userPaused) continue;
|
||
if (task.pausedReason === "worktrunk_operation_failed") {
|
||
log.log(`[self-healing] skipping worktrunk-paused task ${task.id}`);
|
||
continue;
|
||
}
|
||
if (activeTaskIds.has(task.id.toUpperCase())) continue;
|
||
|
||
const emitDeferredReclaimAudit = async (reason: "active-session" | "recent-execution-started" | "worktree-has-uncommitted-changes", hasActiveSession: boolean, hasUncommittedChanges: boolean): Promise<void> => {
|
||
log.log(`[self-healing] deferring stale-active-branch reclaim for ${task.id}: reason=${reason}`);
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "reclaim-stale-active-branches",
|
||
});
|
||
await auditor.git({
|
||
type: "branch:stale-active-reclaim-deferred",
|
||
target: branch,
|
||
metadata: {
|
||
taskId: task.id,
|
||
branch,
|
||
reason,
|
||
executionStartedAt: task.executionStartedAt ?? null,
|
||
hasActiveSession,
|
||
hasUncommittedChanges,
|
||
},
|
||
});
|
||
} catch (auditErr: unknown) {
|
||
log.warn(`Failed to write branch:stale-active-reclaim-deferred run-audit event for ${task.id}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
|
||
}
|
||
};
|
||
|
||
const hasActiveSession = Boolean(task.worktree && activeSessionRegistry.isPathActive(task.worktree));
|
||
if (hasActiveSession) {
|
||
await emitDeferredReclaimAudit("active-session", true, false);
|
||
continue;
|
||
}
|
||
|
||
const executionStartedAtMs = task.executionStartedAt ? Date.parse(task.executionStartedAt) : Number.NaN;
|
||
const isRecentlyStarted = Number.isFinite(executionStartedAtMs) && Date.now() - executionStartedAtMs <= STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS;
|
||
if (isRecentlyStarted) {
|
||
await emitDeferredReclaimAudit("recent-execution-started", false, false);
|
||
continue;
|
||
}
|
||
|
||
if (task.worktree && existsSync(task.worktree)) {
|
||
try {
|
||
if (statSync(task.worktree).isDirectory()) {
|
||
const { stdout } = await execAsync(`git -C ${JSON.stringify(task.worktree)} status --porcelain`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
maxBuffer: 10 * 1024 * 1024,
|
||
});
|
||
if ((stdout ?? "").trim().length > 0) {
|
||
await emitDeferredReclaimAudit("worktree-has-uncommitted-changes", false, true);
|
||
continue;
|
||
}
|
||
}
|
||
} catch (statusErr: unknown) {
|
||
log.warn(`[self-healing] stale-active-branch reclaim could not determine worktree status for ${task.id}: ${statusErr instanceof Error ? statusErr.message : String(statusErr)}`);
|
||
}
|
||
}
|
||
|
||
if (task.worktree && await isUsableTaskWorktree(this.options.rootDir, task.worktree)) continue;
|
||
|
||
const inspection = await this.inspectOrphanedBranch(branch);
|
||
if (!inspection) continue;
|
||
|
||
if (inspection.uniqueCommitCount > 0) {
|
||
log.warn(`[recovery] stale-active-branch-rescue-needed ${task.id} branch=${branch} unique=${inspection.uniqueCommitCount} tip=${inspection.tipSha.slice(0, 12)}`);
|
||
continue;
|
||
}
|
||
|
||
await execAsync(`git branch -D ${JSON.stringify(branch)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 120_000,
|
||
maxBuffer: 10 * 1024 * 1024,
|
||
});
|
||
// Branch-level reclaim remains active in worktrunk mode; this is
|
||
// idempotent git metadata cleanup, not layout ownership.
|
||
// FN-4742: keep native prune; see WorktreeBackend.prune docs
|
||
await execAsync("git worktree prune", {
|
||
cwd: this.options.rootDir,
|
||
timeout: 120_000,
|
||
maxBuffer: 10 * 1024 * 1024,
|
||
});
|
||
|
||
await this.store.updateTask(task.id, {
|
||
worktree: null,
|
||
branch: null,
|
||
baseCommitSha: null,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`[recovery] stale-active-branch-reclaim ${task.id} branch=${branch} reason=zero-unique-commits-no-worktree`,
|
||
);
|
||
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "reclaim-stale-active-branches",
|
||
});
|
||
await auditor.git({
|
||
type: "branch:stale-active-reclaim",
|
||
target: branch,
|
||
metadata: {
|
||
taskId: task.id,
|
||
branch,
|
||
tipSha: inspection.tipSha,
|
||
uniqueCommitCount: inspection.uniqueCommitCount,
|
||
reason: "zero-unique-commits-no-worktree",
|
||
},
|
||
});
|
||
} catch (auditErr: unknown) {
|
||
log.warn(`Failed to write branch:stale-active-reclaim run-audit event for ${task.id}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
|
||
}
|
||
|
||
reclaimed++;
|
||
}
|
||
|
||
if (reclaimed > 0) {
|
||
log.log(`Reclaimed ${reclaimed} stale active fusion branch(es) with no usable worktree`);
|
||
}
|
||
return reclaimed;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Stale active branch reclaim sweep failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Clear `blockedBy` on todo tasks whose blocker has reached a terminal or
|
||
* stuck state.
|
||
*
|
||
* Stale-blocker conditions (clear if ANY apply):
|
||
* 1. Blocker task does not exist (id missing entirely)
|
||
* 2. Blocker `column === "done"` or `column === "archived"`
|
||
* 3. Blocker `column === "in-review"` and `paused === true`
|
||
* 4. Blocker `column === "in-review"` and `status === "failed"`
|
||
* and `(mergeRetries ?? 0) >= MAX_AUTO_MERGE_RETRIES`
|
||
* 5. Blocker `column === "in-review"` and `status === "merging" | "merging-pr"`
|
||
* (or a stale post-recovery `status === null` aftermath) with stale
|
||
* `updatedAt` (older than `staleMergingFanoutMinAgeMs`) and no active
|
||
* merger ownership in this process
|
||
*
|
||
* @returns Number of tasks unblocked
|
||
*/
|
||
private async findWorktreePathForBranch(branchName: string): Promise<string | undefined> {
|
||
try {
|
||
const { stdout } = await execAsync("git worktree list --porcelain", {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
const lines = stdout.split("\n");
|
||
let currentWorktree: string | undefined;
|
||
let currentBranch: string | undefined;
|
||
for (const rawLine of lines) {
|
||
const line = rawLine.trim();
|
||
if (!line) {
|
||
if (currentWorktree && currentBranch === branchName) return currentWorktree;
|
||
currentWorktree = undefined;
|
||
currentBranch = undefined;
|
||
continue;
|
||
}
|
||
if (line.startsWith("worktree ")) {
|
||
currentWorktree = line.slice("worktree ".length).trim();
|
||
continue;
|
||
}
|
||
if (line.startsWith("branch refs/heads/")) {
|
||
currentBranch = line.slice("branch refs/heads/".length).trim();
|
||
}
|
||
}
|
||
if (currentWorktree && currentBranch === branchName) return currentWorktree;
|
||
return undefined;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`[self-healing] reconcileCompletedTask: failed to read worktree list for ${branchName}: ${errorMessage}`);
|
||
return undefined;
|
||
}
|
||
}
|
||
|
||
private async clearCompletionBranchIfSubsumed(task: Task, branchName: string): Promise<boolean> {
|
||
try {
|
||
await execAsync(`git rev-parse --verify ${shellQuote(branchName)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
} catch {
|
||
return false;
|
||
}
|
||
|
||
const baseBranch = task.baseBranch || "main";
|
||
const comparison = await listUniqueBranchCommits(this.options.rootDir, baseBranch, branchName);
|
||
if (comparison.commits.length > 0) {
|
||
log.warn(
|
||
`[self-healing] reconcileCompletedTask ${task.id}: branch ${branchName} has ${comparison.commits.length} unique commit(s) vs ${comparison.mainRef}; skip deletion`,
|
||
);
|
||
return false;
|
||
}
|
||
|
||
try {
|
||
await execAsync(`git branch -D ${shellQuote(branchName)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
return true;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`[self-healing] reconcileCompletedTask ${task.id}: failed to delete branch ${branchName}: ${errorMessage}`);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async reconcileCompletedTask(
|
||
taskId: string,
|
||
options?: { worktreeHint?: string },
|
||
): Promise<{ blockedByCleared: number; worktreeRemoved: boolean; branchRemoved: boolean }> {
|
||
const result = { blockedByCleared: 0, worktreeRemoved: false, branchRemoved: false };
|
||
const prefix = `[self-healing] reconcileCompletedTask ${taskId}:`;
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return result;
|
||
|
||
const task = await this.store.getTask(taskId);
|
||
await this.reconcileTaskWorktreeMetadata({ includeTaskIds: new Set([taskId]) });
|
||
const allTasks = await this.store.listTasks({ slim: true, includeArchived: true });
|
||
const taskById = new Map(allTasks.map((t) => [t.id, t]));
|
||
const todoTasks = await this.store.listTasks({ column: "todo", slim: true });
|
||
const inProgressTasks = await this.store.listTasks({ column: "in-progress", slim: true });
|
||
const inReviewTasks = (await this.store.listTasks({ column: "in-review", slim: true })).filter((t) => !t.paused);
|
||
|
||
const dependents = [...todoTasks, ...inProgressTasks, ...inReviewTasks].filter(
|
||
(t) => t.blockedBy === taskId || t.overlapBlockedBy === taskId,
|
||
);
|
||
const todoTaskIds = new Set(todoTasks.map((t) => t.id));
|
||
for (const dependent of dependents) {
|
||
try {
|
||
const unresolvedDeps = dependent.dependencies.filter((depId) => {
|
||
const dep = taskById.get(depId);
|
||
return dep && dep.column !== "done" && dep.column !== "in-review" && dep.column !== "archived";
|
||
});
|
||
const overlapBlockedBy = dependent.overlapBlockedBy === taskId ? null : (dependent.overlapBlockedBy ?? null);
|
||
const overlapBlockerTask = overlapBlockedBy ? taskById.get(overlapBlockedBy) : undefined;
|
||
const hasActiveOverlapBlocker = Boolean(
|
||
overlapBlockerTask
|
||
&& (overlapBlockerTask.column === "in-progress" || (overlapBlockerTask.column === "in-review" && !overlapBlockerTask.paused)),
|
||
);
|
||
|
||
if (todoTaskIds.has(dependent.id)) {
|
||
if (unresolvedDeps.length > 0) {
|
||
const nextBlocker = unresolvedDeps[0]!;
|
||
await this.store.updateTask(dependent.id, { blockedBy: nextBlocker, overlapBlockedBy, status: "queued" });
|
||
await this.store.logEntry(
|
||
dependent.id,
|
||
`Auto-recovered (FN-4523): cleared stale blockedBy — blocker ${taskId} is done; now blocked by ${nextBlocker}`,
|
||
);
|
||
} else if (hasActiveOverlapBlocker) {
|
||
await this.store.updateTask(dependent.id, { blockedBy: null, overlapBlockedBy, status: "queued" });
|
||
await this.store.logEntry(
|
||
dependent.id,
|
||
`Auto-recovered (FN-4523): preserved queued status — still blocked by file scope overlap with ${overlapBlockedBy}`,
|
||
);
|
||
} else {
|
||
await this.store.updateTask(dependent.id, { blockedBy: null, overlapBlockedBy: null, status: null });
|
||
await this.store.logEntry(
|
||
dependent.id,
|
||
`Auto-recovered (FN-4523): cleared stale blockedBy — blocker ${taskId} is done`,
|
||
);
|
||
}
|
||
} else {
|
||
await this.store.updateTask(dependent.id, {
|
||
blockedBy: null,
|
||
...(dependent.overlapBlockedBy === taskId ? { overlapBlockedBy: null } : {}),
|
||
});
|
||
await this.store.logEntry(
|
||
dependent.id,
|
||
`Auto-recovered (FN-4523): cleared stale blockedBy — blocker ${taskId} is done`,
|
||
);
|
||
}
|
||
result.blockedByCleared++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`${prefix} failed blockedBy fan-out for ${dependent.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
const branchName = task?.branch || canonicalFusionBranchName(taskId);
|
||
const hintedWorktreePath = options?.worktreeHint;
|
||
let worktreePath = hintedWorktreePath;
|
||
if (!worktreePath || !existsSync(worktreePath)) {
|
||
worktreePath = task?.worktree;
|
||
}
|
||
if (!worktreePath || !existsSync(worktreePath)) {
|
||
worktreePath = await this.findWorktreePathForBranch(branchName);
|
||
}
|
||
if (worktreePath && existsSync(worktreePath)) {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
await removeWorktree({
|
||
rootDir: this.options.rootDir,
|
||
worktreePath,
|
||
settings,
|
||
taskId,
|
||
reason: RemovalReason.SelfHealingStaleActiveBranch,
|
||
});
|
||
result.worktreeRemoved = true;
|
||
if (task) {
|
||
const patch = {
|
||
worktree: null as string | null,
|
||
...(task.branch === branchName ? { branch: null as string | null } : {}),
|
||
};
|
||
await this.store.updateTask(task.id, patch as Partial<Task>);
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`${prefix} failed to remove worktree ${worktreePath}: ${errorMessage}`);
|
||
}
|
||
} else {
|
||
log.log(`${prefix} no live worktree found for branch ${branchName}`);
|
||
}
|
||
|
||
this.options.releaseExecutorWorktreeOwnership?.(taskId);
|
||
|
||
if (task) {
|
||
result.branchRemoved = await this.clearCompletionBranchIfSubsumed(task, branchName);
|
||
}
|
||
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", taskId),
|
||
agentId: "self-healing",
|
||
taskId,
|
||
taskLineageId: task?.lineageId ?? undefined,
|
||
phase: "completion-fanout",
|
||
});
|
||
await auditor.database({
|
||
type: "task:auto-recover-completion-fanout",
|
||
target: taskId,
|
||
metadata: {
|
||
blockedByCleared: result.blockedByCleared,
|
||
worktreeRemoved: result.worktreeRemoved,
|
||
branchRemoved: result.branchRemoved,
|
||
branch: branchName,
|
||
worktreePath: result.worktreeRemoved ? worktreePath : undefined,
|
||
},
|
||
});
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`${prefix} failed to record run-audit event: ${errorMessage}`);
|
||
}
|
||
|
||
return result;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`${prefix} failed: ${errorMessage}`);
|
||
return result;
|
||
}
|
||
}
|
||
|
||
private async emitWorktreeMetadataAuditEvent(input: {
|
||
taskId: string;
|
||
mutationType: "task:auto-recover-worktree-metadata-rebound" | "task:auto-recover-worktree-metadata-cleared";
|
||
previousWorktree: string | null;
|
||
newWorktree: string | null;
|
||
previousBranch: string | null;
|
||
newBranch: string | null;
|
||
}): Promise<void> {
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", input.taskId),
|
||
agentId: "self-healing",
|
||
taskId: input.taskId,
|
||
phase: "worktree-metadata-reconcile",
|
||
});
|
||
await auditor.database({
|
||
type: input.mutationType,
|
||
target: input.taskId,
|
||
metadata: {
|
||
taskId: input.taskId,
|
||
previousWorktree: input.previousWorktree,
|
||
newWorktree: input.newWorktree,
|
||
previousBranch: input.previousBranch,
|
||
newBranch: input.newBranch,
|
||
},
|
||
});
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
worktreeMetadataReconcileLog.warn(
|
||
`Failed to record ${input.mutationType} for ${input.taskId}: ${errorMessage}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
private async emitBranchRebindAuditEvent(input: {
|
||
taskId: string;
|
||
mutationType: "task:auto-rebind-applied" | "task:auto-rebind-skipped";
|
||
metadata: Record<string, unknown>;
|
||
}): Promise<void> {
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", input.taskId),
|
||
agentId: "self-healing",
|
||
taskId: input.taskId,
|
||
phase: "in-review-branch-rebind",
|
||
});
|
||
await auditor.database({
|
||
type: input.mutationType as unknown as Parameters<typeof auditor.database>[0]["type"],
|
||
target: input.taskId,
|
||
metadata: input.metadata,
|
||
});
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
worktreeMetadataReconcileLog.warn(
|
||
`Failed to record ${input.mutationType} for ${input.taskId}: ${errorMessage}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
async reconcileInReviewBranchRebind(options?: { includeTaskIds?: Set<string> }): Promise<RebindResult> {
|
||
const result: RebindResult = { repaired: 0, outcomes: [] };
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return result;
|
||
|
||
const allTasks = await this.store.listTasks({ slim: true, includeArchived: false });
|
||
const tasks = allTasks.filter((task) => task.column === "in-review");
|
||
const fusionRefOutput = await execAsync("git for-each-ref --format='%(refname:short)' refs/heads/fusion/", {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
}).catch(() => ({ stdout: "" }));
|
||
const fusionBranches = fusionRefOutput.stdout.split("\n").map((line) => line.trim()).filter(Boolean);
|
||
|
||
for (const task of tasks) {
|
||
if (options?.includeTaskIds && !options.includeTaskIds.has(task.id)) continue;
|
||
|
||
const existingBinding = task.branch;
|
||
if (existingBinding) {
|
||
try {
|
||
await execAsync(`git show-ref --verify --quiet ${shellQuote(`refs/heads/${existingBinding}`)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
result.outcomes.push({ taskId: task.id, result: "skipped", reason: "binding-intact" });
|
||
continue;
|
||
} catch {
|
||
// broken binding, evaluate candidates
|
||
}
|
||
}
|
||
|
||
const normalizedId = task.id.toLowerCase();
|
||
const candidates = new Set<string>([canonicalFusionBranchName(task.id), `fusion/`.concat(task.id)]);
|
||
for (const branch of fusionBranches) {
|
||
const stem = branch.startsWith("fusion/") ? branch.slice("fusion/".length) : "";
|
||
if (stem.toLowerCase() === normalizedId) candidates.add(branch);
|
||
}
|
||
|
||
const integrationBase = task.baseBranch || "main";
|
||
const existingCandidatesByRef = new Map<string, { branch: string; aheadCount: number }>();
|
||
for (const branch of candidates) {
|
||
try {
|
||
await execAsync(`git show-ref --verify --quiet ${shellQuote(`refs/heads/${branch}`)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
} catch {
|
||
continue;
|
||
}
|
||
|
||
let comparisonBase = integrationBase;
|
||
try {
|
||
await execAsync(`git rev-parse --verify ${shellQuote(comparisonBase)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
} catch {
|
||
const originBase = `origin/${comparisonBase}`;
|
||
await execAsync(`git rev-parse --verify ${shellQuote(originBase)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
comparisonBase = originBase;
|
||
}
|
||
const mergeBase = (await execAsync(`git merge-base ${shellQuote(comparisonBase)} ${shellQuote(branch)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
})).stdout.trim();
|
||
const aheadCountRaw = await execAsync(`git rev-list --count ${shellQuote(mergeBase)}..${shellQuote(branch)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
const aheadCount = Number.parseInt(aheadCountRaw.stdout.trim(), 10);
|
||
const normalizedBranchRef = branch.toLowerCase();
|
||
const existingCandidate = existingCandidatesByRef.get(normalizedBranchRef);
|
||
const normalizedCandidate = canonicalFusionBranchName(task.id);
|
||
if (!existingCandidate || branch === normalizedCandidate) {
|
||
existingCandidatesByRef.set(normalizedBranchRef, {
|
||
branch,
|
||
aheadCount: Number.isFinite(aheadCount) ? aheadCount : 0,
|
||
});
|
||
}
|
||
}
|
||
|
||
const existingCandidates = [...existingCandidatesByRef.values()];
|
||
|
||
if (existingCandidates.length === 0) {
|
||
await this.emitBranchRebindAuditEvent({
|
||
taskId: task.id,
|
||
mutationType: "task:auto-rebind-skipped",
|
||
metadata: { taskId: task.id, reason: "no-live-branch" },
|
||
});
|
||
result.outcomes.push({ taskId: task.id, result: "skipped", reason: "no-live-branch" });
|
||
continue;
|
||
}
|
||
|
||
const withUniqueWork = existingCandidates.filter((candidate) => candidate.aheadCount > 0);
|
||
if (withUniqueWork.length === 1) {
|
||
const selected = withUniqueWork[0];
|
||
const patch: Partial<Task> = { branch: selected.branch, worktree: null as unknown as string };
|
||
if (!task.baseCommitSha) {
|
||
const derivedBaseCommit = (await execAsync(
|
||
`git merge-base ${shellQuote(integrationBase)} ${shellQuote(selected.branch)}`,
|
||
{ cwd: this.options.rootDir, timeout: 30_000 },
|
||
)).stdout.trim();
|
||
if (derivedBaseCommit) {
|
||
patch.baseCommitSha = derivedBaseCommit;
|
||
}
|
||
}
|
||
// TODO(FN-5066): tighten composition once helper API is final.
|
||
try {
|
||
const maybeAsserting = this as unknown as { assertSafeToAutoMutate?: (opts: unknown) => Promise<void> };
|
||
await maybeAsserting.assertSafeToAutoMutate?.({ taskId: task.id, reason: "in-review-branch-rebind" });
|
||
} catch (assertErr: unknown) {
|
||
const message = assertErr instanceof Error ? assertErr.message : String(assertErr);
|
||
log.warn(`[self-healing] assertSafeToAutoMutate warning for ${task.id}: ${message}; continuing rebind`);
|
||
}
|
||
await this.store.updateTask(task.id, patch);
|
||
await this.emitBranchRebindAuditEvent({
|
||
taskId: task.id,
|
||
mutationType: "task:auto-rebind-applied",
|
||
metadata: {
|
||
taskId: task.id,
|
||
branch: selected.branch,
|
||
aheadCount: selected.aheadCount,
|
||
integrationBase,
|
||
source: "auto-rebind-in-review",
|
||
previousBranch: task.branch ?? null,
|
||
},
|
||
});
|
||
result.repaired++;
|
||
result.outcomes.push({
|
||
taskId: task.id,
|
||
result: "applied",
|
||
branch: selected.branch,
|
||
aheadCount: selected.aheadCount,
|
||
integrationBase,
|
||
previousBranch: task.branch ?? null,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
if (withUniqueWork.length > 1) {
|
||
await this.emitBranchRebindAuditEvent({
|
||
taskId: task.id,
|
||
mutationType: "task:auto-rebind-skipped",
|
||
metadata: { taskId: task.id, reason: "ambiguous-candidates", candidates: withUniqueWork },
|
||
});
|
||
log.warn(`[self-healing] ambiguous branch rebind candidates for ${task.id}: ${JSON.stringify(withUniqueWork)}`);
|
||
result.outcomes.push({ taskId: task.id, result: "skipped", reason: "ambiguous-candidates", candidates: withUniqueWork });
|
||
continue;
|
||
}
|
||
|
||
await this.emitBranchRebindAuditEvent({
|
||
taskId: task.id,
|
||
mutationType: "task:auto-rebind-skipped",
|
||
metadata: { taskId: task.id, reason: "no-unique-work" },
|
||
});
|
||
result.outcomes.push({ taskId: task.id, result: "skipped", reason: "no-unique-work" });
|
||
}
|
||
|
||
return result;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
worktreeMetadataReconcileLog.error(`reconcileInReviewBranchRebind failed: ${errorMessage}`);
|
||
return result;
|
||
}
|
||
}
|
||
|
||
async reconcileTaskWorktreeMetadata(options?: { includeTaskIds?: Set<string> }): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
|
||
const allTasks = await this.store.listTasks({ slim: true, includeArchived: false });
|
||
const branchMap = await getRegisteredWorktreeBranchMap(this.options.rootDir);
|
||
const registeredPaths = new Set(branchMap.values());
|
||
let repaired = 0;
|
||
|
||
for (const task of allTasks) {
|
||
if (!task.worktree) continue;
|
||
if (!options?.includeTaskIds?.has(task.id) && (task.column === "done" || task.column === "archived")) {
|
||
continue;
|
||
}
|
||
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
if (executingIds.has(task.id)) continue;
|
||
if (activeSessionRegistry.isPathActive(task.worktree)) continue;
|
||
|
||
const normalizedBranch = canonicalFusionBranchName(task.id);
|
||
const canonicalTaskWorktree = resolve(task.worktree);
|
||
const stale = !existsSync(task.worktree) || !registeredPaths.has(canonicalTaskWorktree);
|
||
if (!stale) continue;
|
||
|
||
const previousWorktree = task.worktree;
|
||
const previousBranch = task.branch ?? null;
|
||
const liveWorktree = branchMap.get(normalizedBranch);
|
||
|
||
if (liveWorktree) {
|
||
await this.store.updateTask(task.id, { worktree: liveWorktree, branch: normalizedBranch });
|
||
await this.emitWorktreeMetadataAuditEvent({
|
||
taskId: task.id,
|
||
mutationType: "task:auto-recover-worktree-metadata-rebound",
|
||
previousWorktree,
|
||
newWorktree: liveWorktree,
|
||
previousBranch,
|
||
newBranch: normalizedBranch,
|
||
});
|
||
worktreeMetadataReconcileLog.log(
|
||
`rebound ${task.id}: ${previousWorktree} -> ${liveWorktree} (${previousBranch ?? "<none>"} -> ${normalizedBranch})`,
|
||
);
|
||
repaired++;
|
||
continue;
|
||
}
|
||
|
||
await this.store.updateTask(task.id, { worktree: null, branch: null });
|
||
await this.emitWorktreeMetadataAuditEvent({
|
||
taskId: task.id,
|
||
mutationType: "task:auto-recover-worktree-metadata-cleared",
|
||
previousWorktree,
|
||
newWorktree: null,
|
||
previousBranch,
|
||
newBranch: null,
|
||
});
|
||
worktreeMetadataReconcileLog.log(
|
||
`cleared ${task.id}: ${previousWorktree} (${previousBranch ?? "<none>"})`,
|
||
);
|
||
repaired++;
|
||
}
|
||
|
||
return repaired;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
worktreeMetadataReconcileLog.error(`reconcileTaskWorktreeMetadata failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
async autoReboundPausedScopeDecay(options?: { ignoreAgeGate?: boolean }): Promise<number> {
|
||
const result = await this.autoReboundPausedScopeDecayDetailed(options);
|
||
return result.count;
|
||
}
|
||
|
||
private async autoReboundPausedScopeDecayDetailed(options?: { ignoreAgeGate?: boolean }): Promise<{ count: number; reboundedIds: string[] }> {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return { count: 0, reboundedIds: [] };
|
||
|
||
const thresholdMs = Number(settings.pausedScopeDecayMs ?? 0);
|
||
if (!options?.ignoreAgeGate && (!Number.isFinite(thresholdMs) || thresholdMs <= 0)) {
|
||
return { count: 0, reboundedIds: [] };
|
||
}
|
||
|
||
const tasks = await this.store.listTasks({ slim: true, includeArchived: false });
|
||
const followersByHolder = new Map<string, number>();
|
||
for (const task of tasks) {
|
||
if (typeof task.blockedBy !== "string" || task.blockedBy.trim().length === 0) continue;
|
||
followersByHolder.set(task.blockedBy, (followersByHolder.get(task.blockedBy) ?? 0) + 1);
|
||
}
|
||
|
||
const now = Date.now();
|
||
const reboundedIds: string[] = [];
|
||
for (const task of tasks) {
|
||
if (task.column !== "in-progress" || task.paused !== true) continue;
|
||
if (SelfHealingManager.PAUSED_SCOPE_DECAY_EXCLUDED_REASONS.has(task.pausedReason ?? "")) continue;
|
||
const followerCount = followersByHolder.get(task.id) ?? 0;
|
||
if (followerCount <= 0) continue;
|
||
|
||
const movedAtMs = Date.parse(task.columnMovedAt ?? task.updatedAt ?? "");
|
||
const ageMs = Number.isFinite(movedAtMs) ? now - movedAtMs : Number.POSITIVE_INFINITY;
|
||
if (!options?.ignoreAgeGate && ageMs < thresholdMs) continue;
|
||
|
||
await this.store.moveTask(task.id, "todo", {
|
||
preserveProgress: true,
|
||
preserveWorktree: true,
|
||
preserveResumeState: true,
|
||
moveSource: "engine",
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-rebounded (FN-4890): paused in-progress holder exceeded scope-decay threshold with ${followerCount} blocked follower(s)`,
|
||
);
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("fn4890-paused-scope-decay", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
phase: "auto-rebound-paused-scope-decay",
|
||
});
|
||
await auditor.database({
|
||
type: "task:auto-rebound-paused-scope-decay",
|
||
target: task.id,
|
||
metadata: {
|
||
taskId: task.id,
|
||
followerCount,
|
||
ignoredAgeGate: options?.ignoreAgeGate === true,
|
||
thresholdMs,
|
||
ageMs,
|
||
},
|
||
});
|
||
reboundedIds.push(task.id);
|
||
}
|
||
|
||
return { count: reboundedIds.length, reboundedIds };
|
||
}
|
||
|
||
private classifyMetaTask(task: Task): { isMeta: boolean; targetTaskId: string | null } {
|
||
const title = task.title ?? "";
|
||
const description = task.description ?? "";
|
||
const targetTaskId = task.sourceParentTaskId ?? title.match(/\bFN-\d+\b/i)?.[0] ?? description.match(/\bFN-\d+\b/i)?.[0] ?? null;
|
||
const isMeta = Boolean(task.noCommitsExpected) || /\b(recover|unblock|finalize|meta)\b/i.test(`${title} ${description}`);
|
||
return { isMeta, targetTaskId: targetTaskId?.toUpperCase() ?? null };
|
||
}
|
||
|
||
private resolveMetaTargetTaskId(byId: Map<string, Task>, task: Task): string | null {
|
||
const classified = this.classifyMetaTask(task);
|
||
if (classified.targetTaskId) return classified.targetTaskId;
|
||
if (!classified.isMeta) return null;
|
||
|
||
const ordered = [...byId.values()].sort((a, b) => {
|
||
const aTime = Date.parse(a.createdAt ?? "");
|
||
const bTime = Date.parse(b.createdAt ?? "");
|
||
if (Number.isFinite(aTime) && Number.isFinite(bTime) && aTime !== bTime) return aTime - bTime;
|
||
return a.id.localeCompare(b.id);
|
||
});
|
||
const metaTasks = ordered.filter((candidate) => this.classifyMetaTask(candidate).isMeta);
|
||
const nonMetaTasks = ordered.filter((candidate) => !this.classifyMetaTask(candidate).isMeta);
|
||
const currentIndex = metaTasks.findIndex((candidate) => candidate.id === task.id);
|
||
const previousMeta = currentIndex > 0 ? metaTasks[currentIndex - 1] : null;
|
||
const firstNonMeta = nonMetaTasks[0] ?? null;
|
||
const action = (task.title ?? "").trim().split(/\s+/, 1)[0]?.toLowerCase() ?? "";
|
||
|
||
if (action === "recover" || action === "unblock") {
|
||
return previousMeta?.id ?? firstNonMeta?.id ?? null;
|
||
}
|
||
if (action === "finalize") {
|
||
return firstNonMeta?.id ?? previousMeta?.id ?? null;
|
||
}
|
||
return previousMeta?.id ?? firstNonMeta?.id ?? null;
|
||
}
|
||
|
||
private computeMetaChainDepth(byId: Map<string, Task>, targetTaskId: string): number {
|
||
let depth = 0;
|
||
const visited = new Set<string>();
|
||
let currentId: string | null = targetTaskId.toUpperCase();
|
||
while (currentId && !visited.has(currentId)) {
|
||
visited.add(currentId);
|
||
const task = byId.get(currentId);
|
||
if (!task) break;
|
||
if (!this.classifyMetaTask(task).isMeta) break;
|
||
const nextTargetId = this.resolveMetaTargetTaskId(byId, task);
|
||
if (!nextTargetId) break;
|
||
depth += 1;
|
||
currentId = nextTargetId.toUpperCase();
|
||
}
|
||
return depth;
|
||
}
|
||
|
||
private async archiveMetaTask(taskId: string): Promise<void> {
|
||
const task = await this.store.getTask(taskId);
|
||
if (!task || task.column === "archived") return;
|
||
if (task.column === "triage" || task.column === "todo") {
|
||
await this.store.moveTask(taskId, "in-progress", { moveSource: "engine" });
|
||
}
|
||
const progressed = await this.store.getTask(taskId);
|
||
if (progressed && progressed.column === "in-progress") {
|
||
await this.store.moveTask(taskId, "done", { moveSource: "engine", skipMergeBlocker: true });
|
||
}
|
||
if (typeof this.store.archiveTaskAndCleanup === "function") {
|
||
await this.store.archiveTaskAndCleanup(taskId);
|
||
return;
|
||
}
|
||
if (typeof this.store.archiveTask === "function") {
|
||
await this.store.archiveTask(taskId, true);
|
||
}
|
||
}
|
||
|
||
private countBlockedDepth(tasks: Task[]): number {
|
||
return tasks.filter((task) => typeof task.blockedBy === "string" && task.blockedBy.trim().length > 0).length;
|
||
}
|
||
|
||
private async evaluateMetaAutoArchiveGuards(task: Task): Promise<{ block: false } | { block: true; reasons: string[] }> {
|
||
const reasons: string[] = [];
|
||
|
||
try {
|
||
const ahead = await isBranchAheadOfBase(task, this.options.rootDir, task.baseBranch ?? task.mergeDetails?.mergeTargetBranch ?? "main");
|
||
if (ahead && ahead.aheadCount > 0) reasons.push("branch-has-unique-commits");
|
||
} catch (err: unknown) {
|
||
log.warn(`Meta auto-archive branch probe failed for ${task.id}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
|
||
const settings = await this.store.getSettings();
|
||
const graceMs = Number(settings.metaTaskActiveExecutionGraceMs ?? 30 * 60_000);
|
||
if (graceMs > 0) {
|
||
const now = Date.now();
|
||
const activityMs = Date.parse(task.executionStartedAt ?? task.columnMovedAt ?? task.updatedAt ?? "");
|
||
const ageMs = now - activityMs;
|
||
const columnMovedAtMs = Date.parse(task.columnMovedAt ?? "");
|
||
const executionStartedAtMs = Date.parse(task.executionStartedAt ?? "");
|
||
const transitionedRecentlyFromInProgress =
|
||
task.column !== "in-progress" &&
|
||
Number.isFinite(columnMovedAtMs) &&
|
||
Number.isFinite(executionStartedAtMs) &&
|
||
columnMovedAtMs >= executionStartedAtMs &&
|
||
now - columnMovedAtMs < graceMs;
|
||
const activeOrRecentlyInProgress = task.column === "in-progress" || transitionedRecentlyFromInProgress;
|
||
if (Number.isFinite(ageMs) && ageMs < graceMs && activeOrRecentlyInProgress) {
|
||
reasons.push("recent-executor-activity");
|
||
}
|
||
}
|
||
|
||
if ((task.taskDoneRetryCount ?? 0) > 0) reasons.push("task-done-retry-pending");
|
||
|
||
if (task.mergeDetails?.commitSha || task.status === "merging" || task.status === "merging-pr" || task.status === "failed") {
|
||
reasons.push("merge-in-progress");
|
||
}
|
||
|
||
if (task.worktree && activeSessionRegistry.isPathActive(task.worktree)) reasons.push("active-session");
|
||
|
||
return reasons.length > 0 ? { block: true, reasons } : { block: false };
|
||
}
|
||
|
||
async autoArchiveResolvedMetaTasks(reboundedTargets?: Set<string>): Promise<number> {
|
||
const tasks = await this.store.listTasks({ slim: false, includeArchived: true });
|
||
const byId = new Map(tasks.map((task) => [task.id.toUpperCase(), task]));
|
||
let archived = 0;
|
||
for (const task of tasks) {
|
||
if (task.column === "archived") continue;
|
||
const classified = this.classifyMetaTask(task);
|
||
const targetTaskId = this.resolveMetaTargetTaskId(byId, task);
|
||
if (!classified.isMeta || !targetTaskId) continue;
|
||
const chainDepth = this.computeMetaChainDepth(byId, targetTaskId);
|
||
const target = byId.get(targetTaskId.toUpperCase());
|
||
const resolved = Boolean(target && !this.classifyMetaTask(target).isMeta && (target.column === "done" || target.column === "archived" || target.column === "todo"));
|
||
const rebounded = Boolean(reboundedTargets?.has(targetTaskId));
|
||
if (!resolved && !rebounded && chainDepth < 2) continue;
|
||
const guardResult = await this.evaluateMetaAutoArchiveGuards(task);
|
||
if (guardResult.block) {
|
||
const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("fn4890-meta", task.id), agentId: "self-healing", taskId: task.id, phase: "auto-archive-meta-resolved-skipped" });
|
||
await auditor.database({
|
||
type: "task:auto-archive-meta-resolved-skipped",
|
||
target: task.id,
|
||
metadata: { taskId: task.id, targetTaskId, targetColumn: target?.column ?? "unknown", chainDepth, blockedBy: guardResult.reasons },
|
||
});
|
||
log.log(`[self-healing] skipped meta-resolved auto-archive for ${task.id}: ${guardResult.reasons.join(",")}`);
|
||
continue;
|
||
}
|
||
try {
|
||
await this.store.logEntry(task.id, `Auto-archived meta-task (FN-4890): target ${targetTaskId} resolved/superseded.`);
|
||
await this.archiveMetaTask(task.id);
|
||
const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("fn4890-meta", task.id), agentId: "self-healing", taskId: task.id, phase: "auto-archive-meta-resolved" });
|
||
await auditor.database({ type: "task:auto-archived-meta-resolved", target: task.id, metadata: { taskId: task.id, targetTaskId, targetColumn: target?.column ?? "unknown", chainDepth } });
|
||
archived++;
|
||
} catch (err: unknown) {
|
||
log.error(`autoArchiveResolvedMetaTasks failed for ${task.id}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
}
|
||
return archived;
|
||
}
|
||
|
||
async autoArchiveStalledMetaTasks(): Promise<number> {
|
||
const settings = await this.store.getSettings();
|
||
const thresholdMs = Number(settings.metaTaskStallAutoCloseMs ?? 2 * 60 * 60_000);
|
||
if (thresholdMs === 0) return 0;
|
||
const tasks = await this.store.listTasks({ slim: false, includeArchived: false });
|
||
const byId = new Map(tasks.map((task) => [task.id.toUpperCase(), task]));
|
||
let archived = 0;
|
||
const now = Date.now();
|
||
for (const task of tasks) {
|
||
if (task.column === "archived") continue;
|
||
const classified = this.classifyMetaTask(task);
|
||
const targetTaskId = this.resolveMetaTargetTaskId(byId, task);
|
||
if (!classified.isMeta || !targetTaskId) continue;
|
||
const chainDepth = this.computeMetaChainDepth(byId, targetTaskId);
|
||
const ageMs = now - Date.parse(task.columnMovedAt ?? task.updatedAt);
|
||
if (chainDepth < 2 && (!Number.isFinite(ageMs) || ageMs < thresholdMs)) continue;
|
||
const target = byId.get(targetTaskId.toUpperCase());
|
||
const targetMovedAtMs = Date.parse(target?.columnMovedAt ?? target?.updatedAt ?? "");
|
||
const targetStalled = !Number.isFinite(targetMovedAtMs) || (now - targetMovedAtMs >= thresholdMs);
|
||
if (chainDepth < 2 && !targetStalled) continue;
|
||
const guardResult = await this.evaluateMetaAutoArchiveGuards(task);
|
||
if (guardResult.block) {
|
||
const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("fn4890-meta", task.id), agentId: "self-healing", taskId: task.id, phase: "auto-archive-meta-stalled-skipped" });
|
||
await auditor.database({
|
||
type: "task:auto-archive-meta-stalled-skipped",
|
||
target: task.id,
|
||
metadata: { taskId: task.id, targetTaskId, chainDepth, stalledMs: Math.max(ageMs, 0), blockedBy: guardResult.reasons },
|
||
});
|
||
log.log(`[self-healing] skipped meta-stalled auto-archive for ${task.id}: ${guardResult.reasons.join(",")}`);
|
||
continue;
|
||
}
|
||
try {
|
||
await this.store.logEntry(task.id, `Auto-archived meta-task (FN-4890): superseded — not spawning further meta; rely on self-heal on target ${targetTaskId}`);
|
||
await this.archiveMetaTask(task.id);
|
||
const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("fn4890-meta", task.id), agentId: "self-healing", taskId: task.id, phase: "auto-archive-meta-stalled" });
|
||
await auditor.database({ type: "task:auto-archived-meta-stalled", target: task.id, metadata: { taskId: task.id, targetTaskId, chainDepth, stalledMs: Math.max(ageMs, 0) } });
|
||
archived++;
|
||
} catch (err: unknown) {
|
||
log.error(`autoArchiveStalledMetaTasks failed for ${task.id}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
}
|
||
return archived;
|
||
}
|
||
|
||
async runBoardStallAutoRecoverySweep(): Promise<{ holders: string[]; recovered: number; unrecovered: boolean }> {
|
||
const settings = await this.store.getSettings();
|
||
const windowMs = Number(settings.boardStallSweepWindowMs ?? 2 * 60 * 60_000);
|
||
const growthThreshold = Number(settings.boardStallBlockedGrowthThreshold ?? 3);
|
||
const now = Date.now();
|
||
const allTasks = await this.store.listTasks({ slim: true, includeArchived: false });
|
||
const blockedDepth = this.countBlockedDepth(allTasks);
|
||
|
||
if (!this.boardStallWindow || now - this.boardStallWindow.windowStartMs >= windowMs) {
|
||
this.boardStallWindow = {
|
||
windowStartMs: now,
|
||
windowStartBlockedDepth: blockedDepth,
|
||
transitionsOutOfInProgressInWindow: 0,
|
||
pendingVerification: null,
|
||
lastNtfyAt: this.boardStallWindow?.lastNtfyAt ?? null,
|
||
};
|
||
}
|
||
|
||
const window = this.boardStallWindow;
|
||
if (window.pendingVerification && this.maintenanceTickCounter > window.pendingVerification.tick) {
|
||
const noProgress = window.transitionsOutOfInProgressInWindow === 0;
|
||
if (noProgress) {
|
||
const ntfyAllowed = window.lastNtfyAt === null || now - window.lastNtfyAt >= BOARD_STALL_NOTIFICATION_COOLDOWN_MS;
|
||
let ntfyDispatched = false;
|
||
if (ntfyAllowed) {
|
||
try {
|
||
if (this.options.ntfyNotifier) {
|
||
await this.options.ntfyNotifier.notifyBoardStallUnrecovered({
|
||
holderIds: window.pendingVerification.holderIds,
|
||
followerCount: window.pendingVerification.followerCount,
|
||
});
|
||
window.lastNtfyAt = now;
|
||
ntfyDispatched = true;
|
||
} else {
|
||
const enabled = Boolean(settings.ntfyEnabled && settings.ntfyTopic);
|
||
const events = resolveNtfyEvents(settings.ntfyEvents);
|
||
if (enabled && isNtfyEventEnabled(events, "board-stall-unrecovered")) {
|
||
const clickUrl = buildNtfyClickUrl({ dashboardHost: settings.ntfyDashboardHost });
|
||
await sendNtfyNotification({
|
||
ntfyBaseUrl: settings.ntfyBaseUrl,
|
||
ntfyAccessToken: settings.ntfyAccessToken,
|
||
topic: settings.ntfyTopic!,
|
||
title: "Board stall unrecovered",
|
||
message: `Auto-recovery could not clear board stall. Holders: ${window.pendingVerification.holderIds.join(", ") || "none"}. Followers blocked: ${window.pendingVerification.followerCount}.`,
|
||
priority: "high",
|
||
clickUrl,
|
||
});
|
||
window.lastNtfyAt = now;
|
||
ntfyDispatched = true;
|
||
}
|
||
}
|
||
} catch {
|
||
ntfyDispatched = false;
|
||
}
|
||
}
|
||
const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("fn4890-board-stall", "global"), agentId: "self-healing", phase: "board-stall-unrecovered" });
|
||
await auditor.database({ type: "task:auto-board-stall-unrecovered", target: "board", metadata: { holderIds: window.pendingVerification.holderIds, followerCount: window.pendingVerification.followerCount, windowMs, ntfyDispatched } });
|
||
window.pendingVerification = null;
|
||
return { holders: [], recovered: 0, unrecovered: true };
|
||
}
|
||
window.pendingVerification = null;
|
||
}
|
||
|
||
const blockedGrowth = blockedDepth - window.windowStartBlockedDepth;
|
||
if (window.transitionsOutOfInProgressInWindow === 0 && blockedGrowth >= growthThreshold) {
|
||
const rebound = await this.autoReboundPausedScopeDecayDetailed({ ignoreAgeGate: true });
|
||
// Measure verification progress after intervention; don't count our own rebound moves.
|
||
window.transitionsOutOfInProgressInWindow = 0;
|
||
const followerCount = blockedDepth;
|
||
const auditor = createRunAuditor(this.store, { runId: generateSyntheticRunId("fn4890-board-stall", "global"), agentId: "self-healing", phase: "board-stall-broken" });
|
||
await auditor.database({ type: "task:auto-board-stall-broken", target: "board", metadata: { holderIds: rebound.reboundedIds, followerCount, windowMs, blockedGrowth } });
|
||
window.pendingVerification = { holderIds: rebound.reboundedIds, followerCount, startedAt: now, tick: this.maintenanceTickCounter };
|
||
return { holders: rebound.reboundedIds, recovered: rebound.count, unrecovered: false };
|
||
}
|
||
|
||
return { holders: [], recovered: 0, unrecovered: false };
|
||
}
|
||
|
||
async clearStaleBlockedBy(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
|
||
const staleMergingStatusMinAgeMs = this.options.staleMergingStatusMinAgeMs ?? DEFAULT_STALE_MERGING_STATUS_MIN_AGE_MS;
|
||
const configuredFanoutMinAgeMs = this.options.staleMergingFanoutMinAgeMs ?? DEFAULT_STALE_MERGING_FANOUT_MIN_AGE_MS;
|
||
const staleMergingFanoutMinAgeMs = Math.max(staleMergingStatusMinAgeMs, configuredFanoutMinAgeMs);
|
||
const activeMergeTaskId = this.options.getActiveMergeTaskId?.() ?? null;
|
||
const now = Date.now();
|
||
|
||
const todoTasks = await this.store.listTasks({ column: "todo" });
|
||
const inProgressTasks = await this.store.listTasks({ column: "in-progress" });
|
||
const inReviewTasks = await this.store.listTasks({ column: "in-review" });
|
||
const blockedTasks = [
|
||
...todoTasks,
|
||
...inProgressTasks,
|
||
...inReviewTasks.filter((task) => !task.paused),
|
||
].filter((task) => typeof task.blockedBy === "string" && task.blockedBy.trim().length > 0);
|
||
const queuedDependencyTasks = todoTasks.filter(
|
||
(task) => task.status === "queued" && (task.dependencies.length > 0 || Boolean(task.overlapBlockedBy)),
|
||
);
|
||
|
||
if (blockedTasks.length === 0 && queuedDependencyTasks.length === 0) return 0;
|
||
|
||
const allTasks = await this.store.listTasks({ includeArchived: true });
|
||
const taskById = new Map(allTasks.map((task) => [task.id, task]));
|
||
|
||
let recovered = 0;
|
||
const todoTaskIds = new Set(todoTasks.map((task) => task.id));
|
||
const blockedTaskIds = new Set(blockedTasks.map((task) => task.id));
|
||
const queuedDependencyTaskIds = new Set(queuedDependencyTasks.map((task) => task.id));
|
||
const candidates = new Map<string, typeof todoTasks[number]>();
|
||
for (const task of blockedTasks) candidates.set(task.id, task);
|
||
for (const task of queuedDependencyTasks) candidates.set(task.id, task);
|
||
|
||
for (const task of candidates.values()) {
|
||
const blockerId = task.blockedBy;
|
||
|
||
const unresolvedDeps = task.dependencies.filter((depId) => {
|
||
const dep = taskById.get(depId);
|
||
return dep && dep.column !== "done" && dep.column !== "in-review" && dep.column !== "archived";
|
||
});
|
||
const overlapBlocker = task.overlapBlockedBy ? taskById.get(task.overlapBlockedBy) : undefined;
|
||
const hasActiveOverlapBlocker = Boolean(
|
||
overlapBlocker
|
||
&& (overlapBlocker.column === "in-progress" || (overlapBlocker.column === "in-review" && !overlapBlocker.paused)),
|
||
);
|
||
|
||
if (blockedTaskIds.has(task.id)) {
|
||
if (!blockerId) continue;
|
||
|
||
const blocker = taskById.get(blockerId);
|
||
let reason: string | null = null;
|
||
|
||
if (!blocker) {
|
||
reason = `blocker ${blockerId} missing`;
|
||
} else if (blocker.column === "done") {
|
||
reason = `blocker ${blockerId} is done`;
|
||
} else if (blocker.column === "archived") {
|
||
reason = `blocker ${blockerId} is archived`;
|
||
} else if (blocker.column === "todo") {
|
||
reason = `blocker ${blockerId} moved to todo`;
|
||
} else if (blocker.column === "in-review" && blocker.paused) {
|
||
reason = `blocker ${blockerId} in-review + paused`;
|
||
} else if (
|
||
blocker.column === "in-review" &&
|
||
blocker.status === "failed" &&
|
||
(blocker.mergeRetries ?? 0) >= MAX_AUTO_MERGE_RETRIES
|
||
) {
|
||
reason = `blocker ${blockerId} in-review + failed (mergeRetries ${blocker.mergeRetries ?? 0}/${MAX_AUTO_MERGE_RETRIES})`;
|
||
} else if (
|
||
blocker.column === "in-review" &&
|
||
blocker.status === "failed" &&
|
||
isMissingWorktreeSessionStartFailure(blocker.error)
|
||
) {
|
||
reason = `blocker ${blockerId} in-review + failed (missing-worktree session start)`;
|
||
} else if (
|
||
blocker.column === "in-review" &&
|
||
(blocker.status === "merging" || blocker.status === "merging-pr" || blocker.status == null) &&
|
||
(!activeMergeTaskId || activeMergeTaskId !== blocker.id)
|
||
) {
|
||
const updatedAtMs = blocker.updatedAt ? Date.parse(blocker.updatedAt) : Number.NaN;
|
||
if (Number.isFinite(updatedAtMs)) {
|
||
const elapsedMs = now - updatedAtMs;
|
||
if (elapsedMs >= staleMergingFanoutMinAgeMs) {
|
||
const blockerStatus = blocker.status ?? "no-status";
|
||
reason = `blocker ${blockerId} in-review + ${blockerStatus} stale for ${elapsedMs}ms (threshold ${staleMergingFanoutMinAgeMs}ms)`;
|
||
}
|
||
}
|
||
} else if (task.dependencies.length > 0 && !unresolvedDeps.includes(blockerId)) {
|
||
reason = `blocker ${blockerId} not among unresolved dependencies`;
|
||
}
|
||
|
||
if (reason) {
|
||
try {
|
||
if (todoTaskIds.has(task.id)) {
|
||
if (unresolvedDeps.length > 0) {
|
||
const nextBlocker = unresolvedDeps[0]!;
|
||
await this.store.updateTask(task.id, { blockedBy: nextBlocker, status: "queued" });
|
||
await this.store.logEntry(task.id, `Auto-recovered: refreshed stale blockedBy — ${reason}; now blocked by ${nextBlocker}`);
|
||
} else if (hasActiveOverlapBlocker) {
|
||
await this.store.updateTask(task.id, { blockedBy: null, status: "queued" });
|
||
await this.store.logEntry(task.id, `Auto-recovered: preserved queued status — still blocked by file scope overlap with ${task.overlapBlockedBy}`);
|
||
} else {
|
||
await this.store.updateTask(task.id, { blockedBy: null, overlapBlockedBy: null, status: null });
|
||
await this.store.logEntry(task.id, `Auto-recovered: cleared stale blockedBy — ${reason}`);
|
||
}
|
||
} else {
|
||
await this.store.updateTask(task.id, { blockedBy: null });
|
||
await this.store.logEntry(task.id, `Auto-recovered (FN-4091): cleared stale blockedBy — ${reason}`);
|
||
}
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to clear stale blockedBy for ${task.id}: ${errorMessage}`);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (!todoTaskIds.has(task.id)) {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
if (unresolvedDeps.length === 0) {
|
||
if (queuedDependencyTaskIds.has(task.id)) {
|
||
try {
|
||
if (hasActiveOverlapBlocker) {
|
||
await this.store.updateTask(task.id, { blockedBy: null, status: "queued" });
|
||
await this.store.logEntry(task.id, `Auto-recovered: preserved queued status — still blocked by file scope overlap with ${task.overlapBlockedBy}`);
|
||
} else {
|
||
await this.store.updateTask(task.id, { blockedBy: null, overlapBlockedBy: null, status: null });
|
||
await this.store.logEntry(task.id, "Auto-recovered: cleared stale queued status — all dependencies satisfied");
|
||
}
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to clear stale queued status for ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
continue;
|
||
}
|
||
|
||
const nextBlocker = unresolvedDeps[0] ?? null;
|
||
if (nextBlocker && task.blockedBy !== nextBlocker) {
|
||
try {
|
||
await this.store.updateTask(task.id, { blockedBy: nextBlocker, status: "queued" });
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to refresh blockedBy for ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Stale blockedBy sweep failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
async reconcileSelfDefeatingDependencies(): Promise<number> {
|
||
const targetColumns: Array<Task["column"]> = ["triage", "todo"];
|
||
let recovered = 0;
|
||
|
||
for (const column of targetColumns) {
|
||
let tasks: Task[] = [];
|
||
try {
|
||
tasks = await this.store.listTasks({ column, slim: true });
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`reconcileSelfDefeatingDependencies: failed to list ${column} tasks: ${errorMessage}`);
|
||
continue;
|
||
}
|
||
|
||
for (const task of tasks) {
|
||
if (!task.dependencies.length) continue;
|
||
|
||
const match = detectSelfDefeatingDependency(task.title, task.dependencies);
|
||
if (!match) continue;
|
||
|
||
const originalDependencies = [...task.dependencies];
|
||
const nextDependencies = originalDependencies.filter((dep) => dep.toUpperCase() !== match.operandTaskId.toUpperCase());
|
||
if (nextDependencies.length === originalDependencies.length) continue;
|
||
|
||
try {
|
||
await this.store.updateTask(task.id, { dependencies: nextDependencies });
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-reconciled self-defeating dependency: removed ${match.operandTaskId} (matched verb: "${match.matchedVerb}") from dependencies.`,
|
||
);
|
||
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal-self-defeating-dep", task.id),
|
||
agentId: "system:self-healing",
|
||
taskId: task.id,
|
||
phase: "reconcile-self-defeating-dep",
|
||
});
|
||
await auditor.database({
|
||
type: "task:auto-reconciled-self-defeating-dep",
|
||
target: task.id,
|
||
metadata: {
|
||
matchedVerb: match.matchedVerb,
|
||
operandTaskId: match.operandTaskId,
|
||
originalDependencies,
|
||
nextDependencies,
|
||
},
|
||
});
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`reconcileSelfDefeatingDependencies: failed for ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
return recovered;
|
||
}
|
||
|
||
private async recordIntegrityAudit(taskId: string, mutationType: "task:finalize-unproven-blocked" | "task:integrity-reconcile-modified-files" | "task:integrity-warning" | "task:auto-recover-stale-merger-status", metadata: Record<string, unknown>): Promise<void> {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-healing-integrity", taskId),
|
||
agentId: "self-healing",
|
||
taskId,
|
||
phase: "self-healing",
|
||
});
|
||
await auditor.database({ type: mutationType, target: taskId, metadata });
|
||
}
|
||
|
||
/**
|
||
* FN-5092 watchdog: detect and repair tasks left in an impossible state where
|
||
* `column ∈ {done, archived}` but `status ∈ {merging, merging-pr}`.
|
||
*
|
||
* Cause: a recovery path (FN-4499 misbinding, FN-4500 already-on-main, manual
|
||
* finalization) moved the task to done WITHOUT going through the merger's
|
||
* `completeTask()`. The merger had previously set `status = "merging"` when it
|
||
* claimed `mergeActive[taskId]`; that slot is single-threaded and now leaks,
|
||
* stalling the entire merger queue for every subsequent in-review task.
|
||
*
|
||
* This watchdog catches the persistent-state half of the leak. The runtime
|
||
* in-memory `mergeActive` Map also has a periodic reconciler in
|
||
* `ProjectEngine.reconcileStaleMergeActive()`, but it skips entries that match
|
||
* the currently-active merge task; a status leak that survives across engine
|
||
* restarts can only be cleared at the storage layer.
|
||
*/
|
||
async reconcileStaleMergerStatus(): Promise<number> {
|
||
try {
|
||
const done = await this.store.listTasks({ column: "done", slim: true });
|
||
const archived = await this.store.listTasks({ column: "archived", slim: true });
|
||
const candidates = [...done, ...archived].filter((task) => {
|
||
const s = task.status;
|
||
return s === "merging" || s === "merging-pr";
|
||
});
|
||
if (candidates.length === 0) return 0;
|
||
|
||
let cleared = 0;
|
||
for (const task of candidates) {
|
||
try {
|
||
const previousStatus = task.status;
|
||
const updatedAtMs = Date.parse(task.updatedAt ?? "") || Date.now();
|
||
const ageMs = Math.max(0, Date.now() - updatedAtMs);
|
||
await this.store.updateTask(task.id, { status: null });
|
||
await this.recordIntegrityAudit(task.id, "task:auto-recover-stale-merger-status", {
|
||
previousColumn: task.column,
|
||
previousStatus,
|
||
ageMs,
|
||
mergeConfirmed: task.mergeDetails?.mergeConfirmed === true,
|
||
commitSha: task.mergeDetails?.commitSha ?? null,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovered: cleared stale status="${previousStatus}" on ${task.column} task (age ${Math.round(ageMs / 1000)}s) — was blocking merger queue`,
|
||
);
|
||
cleared++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`reconcileStaleMergerStatus: failed for ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
if (cleared > 0) {
|
||
log.warn(`Cleared ${cleared} stale merger-status leak${cleared === 1 ? "" : "s"} on done/archived tasks (FN-5092)`);
|
||
}
|
||
return cleared;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`reconcileStaleMergerStatus failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async finalizeNoOpReviewTasks(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const candidates = tasks.filter((t) =>
|
||
t.column === "in-review" &&
|
||
!t.paused &&
|
||
Boolean(t.worktree) &&
|
||
t.mergeDetails?.mergeConfirmed !== true &&
|
||
t.status !== "merging" &&
|
||
t.status !== "merging-pr" &&
|
||
t.status !== "awaiting-user-review" &&
|
||
t.status !== "failed" &&
|
||
getTaskMergeBlocker(t) === undefined,
|
||
);
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
let recovered = 0;
|
||
const mergeTargetBranch = typeof settings.baseBranch === "string" && settings.baseBranch.trim().length > 0
|
||
? settings.baseBranch
|
||
: "main";
|
||
for (const task of candidates) {
|
||
const ahead = await this.isBranchAheadOfBase(task, task.mergeDetails?.mergeTargetBranch || mergeTargetBranch);
|
||
if (!ahead || ahead.aheadCount !== 0) continue;
|
||
|
||
const classification = await classifyOwnedLandedEvidenceForSelfHealing(this.options.rootDir, task, ahead.baseRef);
|
||
|
||
if (classification.kind === "unproven") {
|
||
// FN-4811 follow-up: dedupe across engine restarts. The in-memory Set only
|
||
// dedupes within one process; persist the first-warning state on
|
||
// mergeDetails.integrityWarning so subsequent sweep runs (after restart) skip
|
||
// re-emitting an identical log entry.
|
||
const alreadyWarned =
|
||
this.finalizeUnprovenWarned.has(task.id) ||
|
||
(task.mergeDetails?.integrityWarning?.reason === classification.reason);
|
||
if (!alreadyWarned) {
|
||
this.finalizeUnprovenWarned.add(task.id);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Finalize blocked: unproven ownership evidence (${classification.reason}); no owned landed commit was found — auto-retrying via todo requeue`,
|
||
JSON.stringify(classification.details, null, 2),
|
||
);
|
||
await this.store.updateTask(task.id, {
|
||
mergeDetails: {
|
||
...(task.mergeDetails || {}),
|
||
integrityWarning: {
|
||
warnedAt: new Date().toISOString(),
|
||
reason: classification.reason,
|
||
},
|
||
},
|
||
});
|
||
} else {
|
||
// Hydrate the in-memory dedup Set from the persisted record so subsequent
|
||
// checks in this process don't have to re-query the task.
|
||
this.finalizeUnprovenWarned.add(task.id);
|
||
}
|
||
await this.recordIntegrityAudit(task.id, "task:finalize-unproven-blocked", {
|
||
reason: classification.reason,
|
||
details: classification.details,
|
||
autoRetry: true,
|
||
});
|
||
await this.store.moveTask(task.id, "todo", { preserveProgress: true, moveSource: "engine" });
|
||
continue;
|
||
}
|
||
|
||
const mergedAt = new Date().toISOString();
|
||
if (classification.kind === "owned-commit") {
|
||
const mergeDetails: MergeDetails = {
|
||
...(task.mergeDetails || {}),
|
||
commitSha: classification.commit.sha,
|
||
filesChanged: classification.commit.filesChanged,
|
||
insertions: classification.commit.insertions,
|
||
deletions: classification.commit.deletions,
|
||
mergeCommitMessage: classification.commit.subject,
|
||
mergeConfirmed: true,
|
||
mergedAt,
|
||
mergeTargetBranch: ahead.baseRef,
|
||
};
|
||
await this.store.updateTask(task.id, { mergeDetails });
|
||
await this.store.logEntry(task.id, `Auto-finalized: recovered owned landed commit ${classification.commit.sha.slice(0, 8)}`);
|
||
} else {
|
||
const noOpReason = `branch has zero commits ahead of ${classification.baseRef}`;
|
||
const mergeDetails: MergeDetails = {
|
||
...(task.mergeDetails || {}),
|
||
mergeConfirmed: true,
|
||
noOpMerge: true,
|
||
noOpReason,
|
||
landedFiles: [],
|
||
mergedAt,
|
||
mergeTargetBranch: classification.baseRef,
|
||
};
|
||
// FN-5092 hotfix: clear transient merger-queue status alongside mergeDetails
|
||
// patch so a leaked `mergeActive` slot from a prior merge attempt cannot survive
|
||
// this auto-finalize path. Same bug class as recoverBranchMisboundInReviewTasks().
|
||
await this.store.updateTask(task.id, { mergeDetails, modifiedFiles: [], status: null, error: null, paused: false });
|
||
await this.recordIntegrityAudit(task.id, "task:integrity-reconcile-modified-files", {
|
||
reason: "proven-no-op-finalize",
|
||
clearedCount: task.modifiedFiles?.length ?? 0,
|
||
});
|
||
await this.store.logEntry(task.id, `Auto-finalized no-op (proven): start point on ${classification.baseRef}; modifiedFiles cleared`);
|
||
}
|
||
|
||
const movedTask = await this.store.moveTask(task.id, "done");
|
||
this.emitTaskMerged(movedTask, { mergeConfirmed: true });
|
||
recovered++;
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} no-op review task(s) → done`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`No-op review finalization failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
async reconcileDoneTaskIntegrity(): Promise<number> {
|
||
try {
|
||
const tasks = await this.store.listTasks({ column: "done", slim: true });
|
||
const candidates = tasks.filter((task) =>
|
||
task.column === "done" &&
|
||
(!task.mergeDetails?.commitSha || task.mergeDetails.commitSha.trim().length === 0) &&
|
||
(task.modifiedFiles?.length ?? 0) > 0,
|
||
).slice(0, DONE_TASK_INTEGRITY_SWEEP_LIMIT);
|
||
|
||
if (candidates.length === 0) return 0;
|
||
const settings = await this.store.getSettings();
|
||
const mergeTargetBranch = typeof settings.baseBranch === "string" && settings.baseBranch.trim().length > 0
|
||
? settings.baseBranch
|
||
: "main";
|
||
|
||
let reconciled = 0;
|
||
for (const task of candidates) {
|
||
const classification = await classifyOwnedLandedEvidenceForSelfHealing(this.options.rootDir, task, mergeTargetBranch);
|
||
if (classification.kind === "owned-commit") {
|
||
await this.store.updateTask(task.id, {
|
||
mergeDetails: {
|
||
...(task.mergeDetails || {}),
|
||
commitSha: classification.commit.sha,
|
||
filesChanged: classification.commit.filesChanged,
|
||
insertions: classification.commit.insertions,
|
||
deletions: classification.commit.deletions,
|
||
mergeCommitMessage: classification.commit.subject,
|
||
},
|
||
});
|
||
await this.recordIntegrityAudit(task.id, "task:integrity-reconcile-modified-files", {
|
||
reason: "recovered-owned-commit",
|
||
commitSha: classification.commit.sha,
|
||
});
|
||
reconciled++;
|
||
continue;
|
||
}
|
||
|
||
if (classification.kind === "proven-no-op") {
|
||
await this.store.updateTask(task.id, {
|
||
modifiedFiles: [],
|
||
mergeDetails: {
|
||
...(task.mergeDetails || {}),
|
||
mergeConfirmed: true,
|
||
noOpMerge: true,
|
||
noOpReason: `branch has zero commits ahead of ${classification.baseRef}`,
|
||
landedFiles: [],
|
||
},
|
||
});
|
||
await this.recordIntegrityAudit(task.id, "task:integrity-reconcile-modified-files", {
|
||
reason: "proven-no-op",
|
||
clearedCount: task.modifiedFiles?.length ?? 0,
|
||
});
|
||
reconciled++;
|
||
continue;
|
||
}
|
||
|
||
if (classification.kind === "no-changes-finalized") {
|
||
await this.store.updateTask(task.id, {
|
||
modifiedFiles: [],
|
||
mergeDetails: {
|
||
...(task.mergeDetails || {}),
|
||
mergeConfirmed: true,
|
||
noOpMerge: true,
|
||
noOpReason: "verification-only finalize: no branch and no owned commits",
|
||
landedFiles: [],
|
||
},
|
||
});
|
||
await this.recordIntegrityAudit(task.id, "task:integrity-reconcile-modified-files", {
|
||
reason: "verification-only-finalize",
|
||
clearedCount: task.modifiedFiles?.length ?? 0,
|
||
baseRef: classification.baseRef,
|
||
details: classification.details,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
"Finalize: verification-only task — no owned commits and no branch; cleared stale modifiedFiles snapshot",
|
||
);
|
||
reconciled++;
|
||
continue;
|
||
}
|
||
|
||
// FN-4811 follow-up: dedupe across engine restarts (see matching block above).
|
||
const alreadyWarned =
|
||
this.finalizeUnprovenWarned.has(task.id) ||
|
||
(task.mergeDetails?.integrityWarning?.reason === classification.reason);
|
||
if (!alreadyWarned) {
|
||
this.finalizeUnprovenWarned.add(task.id);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Integrity warning: done-task finalize evidence is unproven (${classification.reason})`,
|
||
JSON.stringify(classification.details, null, 2),
|
||
);
|
||
await this.store.updateTask(task.id, {
|
||
mergeDetails: {
|
||
...(task.mergeDetails || {}),
|
||
integrityWarning: {
|
||
warnedAt: new Date().toISOString(),
|
||
reason: classification.reason,
|
||
},
|
||
},
|
||
});
|
||
} else {
|
||
this.finalizeUnprovenWarned.add(task.id);
|
||
}
|
||
await this.recordIntegrityAudit(task.id, "task:integrity-warning", {
|
||
reason: classification.reason,
|
||
modifiedFilesCount: task.modifiedFiles?.length ?? 0,
|
||
details: classification.details,
|
||
});
|
||
}
|
||
|
||
return reconciled;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Done-task integrity reconciliation failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover `in-review` tasks that are fully mergeable but never had
|
||
* `mergeTask()` invoked.
|
||
*
|
||
* This catches races where a task reached review, retained its worktree,
|
||
* and then got stranded without a merger loop to finish the branch.
|
||
*
|
||
* @returns Number of tasks merged or finalized to done
|
||
*/
|
||
async recoverMergeableReviewTasks(): Promise<number> {
|
||
try {
|
||
// Respect user merge intent. Without these gates the sweep would
|
||
// silently merge tasks even when the operator has opted into a
|
||
// PR-based review flow (`autoMerge: false`, `mergeStrategy:
|
||
// "pull-request"`) — see GitHub issue #21.
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
|
||
const mergeable = tasks.filter((t) =>
|
||
t.column === "in-review" &&
|
||
!t.paused &&
|
||
t.status !== "failed" &&
|
||
// Exclude transient merge statuses. Active merges should be left alone;
|
||
// stale ones are handled by recoverStaleMergingStatus().
|
||
t.status !== "merging" &&
|
||
t.status !== "merging-pr" &&
|
||
Boolean(t.worktree) &&
|
||
t.mergeDetails?.mergeConfirmed !== true &&
|
||
t.mergeDetails?.noOpMerge !== true &&
|
||
!hasTerminalInvalidDoneTransition(t) &&
|
||
// Mirror ProjectEngine.canMergeTask retry gate. If retries are already
|
||
// exhausted, re-enqueueing here is a no-op and each recovery log write
|
||
// refreshes updatedAt, preventing cooldown-based retries from ever
|
||
// becoming eligible. Also skip tasks explicitly tagged as no-op merges
|
||
// in case updateTask(moveTask) is briefly out-of-order during recovery.
|
||
(t.mergeRetries ?? 0) < MAX_AUTO_MERGE_RETRIES &&
|
||
getTaskMergeBlocker(t) === undefined,
|
||
);
|
||
|
||
const inReviewIds = new Set(tasks.map((task) => task.id));
|
||
const mergeableIds = new Set(mergeable.map((task) => task.id));
|
||
for (const taskId of [...this.mergeStarvationDrops.keys()]) {
|
||
if (!inReviewIds.has(taskId) || !mergeableIds.has(taskId)) {
|
||
this.mergeStarvationDrops.delete(taskId);
|
||
}
|
||
}
|
||
|
||
if (mergeable.length === 0) return 0;
|
||
|
||
log.warn(`Found ${mergeable.length} mergeable review task(s) stuck in in-review`);
|
||
|
||
// Prefer the engine's merge queue so `mergeStrategy` (direct vs.
|
||
// pull-request) is honored. Fall back to a direct store merge only
|
||
// when no enqueue callback is wired (standalone/tests).
|
||
const enqueueMerge = this.options.enqueueMerge;
|
||
let recovered = 0;
|
||
for (const task of mergeable) {
|
||
try {
|
||
if (enqueueMerge) {
|
||
const queued = enqueueMerge(task.id);
|
||
if (!queued) {
|
||
const drops = (this.mergeStarvationDrops.get(task.id) ?? 0) + 1;
|
||
this.mergeStarvationDrops.set(task.id, drops);
|
||
log.warn(
|
||
`Auto-recovery enqueue dropped for ${task.id} (${drops}/${MAX_STARVATION_DROPS}); engine merge queue rejected re-enqueue`,
|
||
);
|
||
if (drops >= MAX_STARVATION_DROPS) {
|
||
const error = `Auto-merge starvation: ${MAX_STARVATION_DROPS} consecutive enqueue attempts were dropped by the engine merge queue; task requires manual intervention.`;
|
||
await this.store.updateTask(task.id, { status: "failed", error });
|
||
await this.store.logEntry(task.id, error);
|
||
this.mergeStarvationDrops.delete(task.id);
|
||
recovered++;
|
||
}
|
||
continue;
|
||
}
|
||
this.mergeStarvationDrops.delete(task.id);
|
||
} else {
|
||
await this.store.mergeTask(task.id);
|
||
}
|
||
await this.store.logEntry(
|
||
task.id,
|
||
enqueueMerge
|
||
? "Auto-recovered: eligible in-review task re-enqueued for merge"
|
||
: "Auto-recovered: eligible in-review task was merged and moved to done",
|
||
);
|
||
log.log(`Recovered mergeable review task ${task.id}`);
|
||
recovered++;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover mergeable review task ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} mergeable review task(s) → done`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Mergeable review recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover `in-review` tasks parked by a failed pre-merge workflow step.
|
||
*
|
||
* When a pre-merge workflow step (e.g. Browser Verification) fails during an
|
||
* active executor run, `executor.handleWorkflowStepFailure` retries up to
|
||
* `MAX_WORKFLOW_STEP_RETRIES` times in-session. If all retries exhaust the
|
||
* task ends up in `in-review` with the failed workflow step result still on
|
||
* record, which `getTaskMergeBlocker` correctly treats as a merge block —
|
||
* leaving the task stranded with no live session to un-stick it.
|
||
*
|
||
* This scan delegates back to the executor's `recoverFailedPreMergeWorkflowStep`
|
||
* path (which reuses the same `sendTaskBackForFix` flow the executor uses
|
||
* internally) so the agent gets another attempt with the failure feedback
|
||
* injected into `PROMPT.md`. Bounded by `settings.maxPostReviewFixes` and the
|
||
* per-task `postReviewFixCount` so a persistently-failing verifier cannot
|
||
* ping-pong a task forever.
|
||
*
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
* @returns Number of tasks sent back for fix
|
||
*/
|
||
async recoverReviewTasksWithFailedPreMergeSteps(): Promise<number> {
|
||
const recoverFn = this.options.recoverFailedPreMergeStep;
|
||
if (!recoverFn) return 0;
|
||
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
const maxFixes = settings.maxPostReviewFixes ?? 1;
|
||
if (!Number.isFinite(maxFixes) || maxFixes <= 0) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
|
||
const candidates = tasks.filter((task) => {
|
||
if (task.column !== "in-review") return false;
|
||
if (task.paused) return false;
|
||
// Preserve terminal/human-handoff statuses (failed, awaiting-user-review,
|
||
// merging, etc.). Only revive tasks that are otherwise idle.
|
||
if (task.status) return false;
|
||
if (executingIds.has(task.id)) return false;
|
||
if ((task.postReviewFixCount ?? 0) >= maxFixes) return false;
|
||
|
||
// Must have at least one failed pre-merge workflow step result.
|
||
const hasFailedPreMerge = (task.workflowStepResults ?? []).some(
|
||
(r) => (r.phase || "pre-merge") === "pre-merge" && r.status === "failed",
|
||
);
|
||
if (!hasFailedPreMerge) return false;
|
||
|
||
// Merge must be blocked *specifically* by the failed pre-merge step —
|
||
// not by an unrelated condition (incomplete steps, etc.) that is
|
||
// already handled by a dedicated scan.
|
||
const blocker = getTaskMergeBlocker(task);
|
||
if (blocker !== "task has failed pre-merge workflow steps") return false;
|
||
|
||
// The retry flow injects into PROMPT.md + re-executes on the worktree.
|
||
// If the worktree was cleaned up we can't reliably resume here; leave
|
||
// such tasks for human intervention.
|
||
if (!task.worktree) return false;
|
||
|
||
return true;
|
||
});
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
log.warn(`Found ${candidates.length} in-review task(s) with failed pre-merge workflow steps — auto-reviving`);
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
const nextCount = (task.postReviewFixCount ?? 0) + 1;
|
||
try {
|
||
// Increment the counter BEFORE delegating so that even if the
|
||
// executor path crashes or races, the budget is still consumed and
|
||
// we can't enter an infinite revival loop.
|
||
await this.store.updateTask(task.id, { postReviewFixCount: nextCount });
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-reviving in-review task with failed pre-merge workflow step (attempt ${nextCount}/${maxFixes})`,
|
||
);
|
||
const sentBack = await recoverFn(task);
|
||
if (sentBack) {
|
||
log.log(`Revived ${task.id}: sent back for fix (${nextCount}/${maxFixes})`);
|
||
recovered++;
|
||
} else {
|
||
log.warn(`Revival of ${task.id} was skipped by executor — budget already consumed`);
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to revive ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Auto-revived ${recovered} in-review task(s) for pre-merge workflow step fix`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed pre-merge workflow step revival failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover tasks that reached `in-review` while a task step was still marked
|
||
* pending/in-progress. These tasks are not tracked by StuckTaskDetector
|
||
* anymore because the executor session is gone, and they are not mergeable
|
||
* because `getTaskMergeBlocker()` correctly blocks incomplete steps.
|
||
*
|
||
* Moving them back to `todo` lets the normal scheduler/executor resume the
|
||
* incomplete step instead of leaving the task stranded in review.
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async recoverStaleIncompleteReviewTasks(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
const timeoutMs = settings.taskStuckTimeoutMs;
|
||
if (!timeoutMs || timeoutMs <= 0) return 0;
|
||
|
||
const now = Date.now();
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const staleIncomplete = tasks.filter((task) =>
|
||
task.column === "in-review" &&
|
||
!task.paused &&
|
||
!task.status &&
|
||
task.steps.length > 0 &&
|
||
task.steps.some((step) => NON_TERMINAL_STEP_STATUSES.has(step.status)) &&
|
||
now - new Date(task.columnMovedAt ?? task.updatedAt).getTime() >= timeoutMs
|
||
);
|
||
|
||
if (staleIncomplete.length === 0) return 0;
|
||
|
||
log.warn(`Found ${staleIncomplete.length} stale in-review task(s) with incomplete steps`);
|
||
|
||
let recovered = 0;
|
||
for (const task of staleIncomplete) {
|
||
try {
|
||
await this.store.logEntry(
|
||
task.id,
|
||
"Auto-recovered: in-review task still had incomplete steps — moved back to todo for retry",
|
||
);
|
||
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||
log.log(`Recovered stale incomplete review task ${task.id}: moved back to todo`);
|
||
recovered++;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover stale incomplete review task ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
return recovered;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Stale incomplete review recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Final-fallback recovery for `in-review` tasks that fell through every other
|
||
* scan and have sat untouched longer than `taskStuckTimeoutMs`.
|
||
*
|
||
* When `settings.autoMerge` is disabled, this sweep is a no-op because
|
||
* PR-based manual review intentionally leaves tasks in `in-review`.
|
||
*
|
||
* The other review-recovery scans each require a specific shape (failed
|
||
* pre-merge step, incomplete steps, mergeable + worktree present, confirmed
|
||
* merge, transient merge status). A task whose state doesn't match any of
|
||
* those shapes — e.g. `status: "failed"` with no failed pre-merge step, or
|
||
* any other unanticipated combination — has no recovery path and stays
|
||
* silent in review forever.
|
||
*
|
||
* This catch-all kicks any such task back to `todo`, clearing transient
|
||
* `status` so the scheduler can pick it up. Worktree state is intentionally
|
||
* not considered: the executor will recreate one if needed.
|
||
*
|
||
* Preserved statuses (skipped):
|
||
* - `awaiting-user-review`, `awaiting-approval`: explicit human handoff
|
||
* - `merging`, `merging-pr`, `merging-fix`: handled by `recoverInterruptedMergingTasks`
|
||
*
|
||
* Rate-limiting comes from the `updatedAt >= taskStuckTimeoutMs` gate —
|
||
* each kick refreshes `updatedAt`, so a task that re-enters review and gets
|
||
* stuck again can only be kicked once per `taskStuckTimeoutMs` window.
|
||
*
|
||
* When `settings.autoMerge === false`, this sweep is a no-op because those
|
||
* projects intentionally use PR-based/manual in-review ownership.
|
||
*
|
||
* @returns Number of tasks kicked back to todo
|
||
*/
|
||
async surfaceInReviewStalls(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const cycleStartMs = Date.now();
|
||
const timeoutMs = settings.taskStuckTimeoutMs;
|
||
if (!timeoutMs || timeoutMs <= 0) return 0;
|
||
|
||
const activeMergeTaskId = this.options.getActiveMergeTaskId?.() ?? null;
|
||
const executingTaskIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: false });
|
||
let surfaced = 0;
|
||
|
||
for (const task of tasks) {
|
||
const signal = getInReviewStallReason(task, {
|
||
now: cycleStartMs,
|
||
activeMergeTaskId,
|
||
executingTaskIds,
|
||
staleMergingMinAgeMs: this.options.staleMergingStatusMinAgeMs ?? DEFAULT_STALE_MERGING_STATUS_MIN_AGE_MS,
|
||
maxAutoMergeRetries: MAX_AUTO_MERGE_RETRIES,
|
||
});
|
||
if (!signal) continue;
|
||
|
||
if (Date.parse(task.updatedAt) >= cycleStartMs) {
|
||
continue;
|
||
}
|
||
|
||
const previous = [...(task.log ?? [])]
|
||
.reverse()
|
||
.find((entry) => entry.action.startsWith(IN_REVIEW_STALL_LOG_PREFIX));
|
||
if (previous) {
|
||
const parsed = /^In-review stall surfaced \[([^\]]+)\]/.exec(previous.action);
|
||
const previousCode = parsed?.[1];
|
||
const previousAt = Date.parse(previous.timestamp);
|
||
if (Number.isFinite(previousAt) && previousAt >= cycleStartMs - timeoutMs && previousCode === signal.code) {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
const threshold = settings.inReviewStallDeadlockThreshold ?? 3;
|
||
const identicalCount = countRecentIdenticalStallEntries(task, { code: signal.code, reason: signal.reason });
|
||
const nextCount = identicalCount + 1;
|
||
const shouldDispose = threshold > 0 && task.userPaused !== true && nextCount >= threshold;
|
||
|
||
if (shouldDispose) {
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`${IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX}${signal.code}]: deadlock-prevention threshold reached after ${nextCount} identical stalls — pausing task. last reason: ${signal.reason}`,
|
||
);
|
||
await this.store.updateTask(task.id, {
|
||
paused: true,
|
||
pausedReason: "in-review-stall-deadlock",
|
||
status: "failed",
|
||
error: `In-review stall deadlock: ${signal.code} repeated ${nextCount}× without progress. ${signal.reason}`,
|
||
});
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-healing-stall-deadlock", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
phase: "self-healing",
|
||
});
|
||
await auditor.database({
|
||
type: "task:in-review-stall-deadlock-disposed",
|
||
target: task.id,
|
||
metadata: {
|
||
code: signal.code,
|
||
reason: signal.reason,
|
||
repetitionCount: nextCount,
|
||
threshold,
|
||
branch: task.branch ?? null,
|
||
worktree: task.worktree ?? null,
|
||
},
|
||
});
|
||
surfaced += 1;
|
||
continue;
|
||
}
|
||
|
||
await this.store.logEntry(task.id, `${IN_REVIEW_STALL_LOG_PREFIX}${signal.code}]: ${signal.reason}`);
|
||
surfaced += 1;
|
||
}
|
||
|
||
return surfaced;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`In-review stall surfacing failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
private getDependencyBlockedTodoReporter(): DependencyBlockedTodoReporter | null {
|
||
if (this.dependencyBlockedTodoReporter) {
|
||
return this.dependencyBlockedTodoReporter;
|
||
}
|
||
const projectId = this.options.getProjectId?.();
|
||
if (!projectId) {
|
||
return null;
|
||
}
|
||
this.dependencyBlockedTodoReporter = new DependencyBlockedTodoReporter({
|
||
store: this.store,
|
||
projectId,
|
||
now: () => Date.now(),
|
||
});
|
||
return this.dependencyBlockedTodoReporter;
|
||
}
|
||
|
||
async surfaceDependencyBlockedTodos(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.dependencyBlockedTodoReportEnabled === false) return 0;
|
||
|
||
const reporter = this.getDependencyBlockedTodoReporter();
|
||
if (!reporter) return 0;
|
||
const result = await reporter.report();
|
||
return result.groupCount ?? 0;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Dependency-blocked todo surfacing failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Surface quiet-window backlog-health diagnostics for unpaused in-review tasks.
|
||
*
|
||
* Non-overlap contract:
|
||
* - `surfaceStalePausedReviews()` owns paused in-review tasks.
|
||
* - `surfaceInReviewStalls()` owns reason-driven in-review stalls.
|
||
*
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async surfaceInReviewStalled(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const cycleStartMs = Date.now();
|
||
const thresholdMs = settings.inReviewStalledThresholdMs;
|
||
if (!thresholdMs || thresholdMs <= 0) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: false });
|
||
const activeMergeTaskId = this.options.getActiveMergeTaskId?.() ?? null;
|
||
const executingTaskIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
let surfaced = 0;
|
||
|
||
for (const task of tasks) {
|
||
if (task.paused === true) continue;
|
||
if (task.id === activeMergeTaskId || executingTaskIds.has(task.id)) continue;
|
||
|
||
const signal = getInReviewStalledSignal(task, {
|
||
now: cycleStartMs,
|
||
thresholdMs,
|
||
autoMerge: true,
|
||
activeMergeTaskId,
|
||
executingTaskIds,
|
||
});
|
||
if (!signal) continue;
|
||
|
||
if (Date.parse(task.updatedAt) >= cycleStartMs) {
|
||
continue;
|
||
}
|
||
|
||
const previous = [...(task.log ?? [])]
|
||
.reverse()
|
||
.find((entry) => entry.action.startsWith("In-review stalled surfaced ["));
|
||
if (previous) {
|
||
const parsed = /^In-review stalled surfaced \[([^\]]+)\]/.exec(previous.action);
|
||
const previousCode = parsed?.[1];
|
||
const previousAt = Date.parse(previous.timestamp);
|
||
if (Number.isFinite(previousAt) && previousAt >= cycleStartMs - thresholdMs && previousCode === signal.code) {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
const hours = (signal.quietMs / 3_600_000).toFixed(1);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`In-review stalled surfaced [${signal.code}]: quiet ${hours}h beyond ${(thresholdMs / 3_600_000).toFixed(1)}h threshold; disposition options — nudge review, retry, archive, or create follow-up task. lastActivitySource=${signal.lastActivitySource}`,
|
||
);
|
||
surfaced += 1;
|
||
}
|
||
|
||
return surfaced;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`In-review stalled surfacing failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
async surfaceStalePausedReviews(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
|
||
const cycleStartMs = Date.now();
|
||
const thresholdMs = settings.stalePausedReviewThresholdMs;
|
||
if (!thresholdMs || thresholdMs <= 0) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: false });
|
||
let surfaced = 0;
|
||
|
||
for (const task of tasks) {
|
||
if (task.paused !== true) continue;
|
||
const signal = getStalePausedReviewSignal(task, { now: cycleStartMs, thresholdMs });
|
||
if (!signal) continue;
|
||
if (Date.parse(task.updatedAt) >= cycleStartMs) continue;
|
||
|
||
const previous = [...(task.log ?? [])]
|
||
.reverse()
|
||
.find((entry) => entry.action.startsWith("Stale paused review surfaced ["));
|
||
if (previous) {
|
||
const parsed = /^Stale paused review surfaced \[([^\]]+)\]/.exec(previous.action);
|
||
const previousCode = parsed?.[1];
|
||
const previousAt = Date.parse(previous.timestamp);
|
||
if (Number.isFinite(previousAt) && previousAt >= cycleStartMs - thresholdMs && previousCode === signal.code) {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
const hours = (signal.ageMs / 3_600_000).toFixed(1);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Stale paused review surfaced [${signal.code}]: paused ${hours}h; disposition options — unpause, retry, archive, or create follow-up task. pausedReason=${signal.pausedReason ?? "none"}`,
|
||
);
|
||
surfaced += 1;
|
||
}
|
||
|
||
return surfaced;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Stale paused review surfacing failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
async surfaceStalePausedTodos(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
|
||
const cycleStartMs = Date.now();
|
||
const thresholdMs = settings.stalePausedTodoThresholdMs;
|
||
if (!thresholdMs || thresholdMs <= 0) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ column: "todo", slim: false });
|
||
let surfaced = 0;
|
||
|
||
for (const task of tasks) {
|
||
if (task.paused !== true) continue;
|
||
const signal = getStalePausedTodoSignal(task, { now: cycleStartMs, thresholdMs });
|
||
if (!signal) continue;
|
||
if (Date.parse(task.updatedAt) >= cycleStartMs) continue;
|
||
|
||
const previous = [...(task.log ?? [])]
|
||
.reverse()
|
||
.find((entry) => entry.action.startsWith("Stale paused todo surfaced ["));
|
||
if (previous) {
|
||
const parsed = /^Stale paused todo surfaced \[([^\]]+)\]/.exec(previous.action);
|
||
const previousCode = parsed?.[1];
|
||
const previousAt = Date.parse(previous.timestamp);
|
||
if (Number.isFinite(previousAt) && previousAt >= cycleStartMs - thresholdMs && previousCode === signal.code) {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
const hours = (signal.ageMs / 3_600_000).toFixed(1);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Stale paused todo surfaced [${signal.code}]: paused ${hours}h beyond ${(thresholdMs / 3_600_000).toFixed(1)}h threshold; disposition options — unpause, move to triage, archive, or create follow-up task. pausedReason=${signal.pausedReason ?? "none"}`,
|
||
);
|
||
surfaced += 1;
|
||
}
|
||
|
||
return surfaced;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Stale paused todo surfacing failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async recoverGhostReviewTasks(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
const timeoutMs = settings.taskStuckTimeoutMs;
|
||
if (!timeoutMs || timeoutMs <= 0) return 0;
|
||
|
||
const now = Date.now();
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const ghosts = tasks.filter((task) =>
|
||
task.column === "in-review" &&
|
||
!task.paused &&
|
||
!executingIds.has(task.id) &&
|
||
!(task.status && GHOST_REVIEW_PRESERVED_STATUSES.has(task.status)) &&
|
||
// Confirmed merges belong in `done` (handled by `recoverMergedReviewTasks`).
|
||
task.mergeDetails?.mergeConfirmed !== true &&
|
||
now - new Date(task.columnMovedAt ?? task.updatedAt).getTime() >= timeoutMs
|
||
);
|
||
|
||
if (ghosts.length === 0) return 0;
|
||
|
||
log.warn(`Found ${ghosts.length} ghost in-review task(s) — kicking back to todo`);
|
||
|
||
let recovered = 0;
|
||
for (const task of ghosts) {
|
||
try {
|
||
if (task.status) {
|
||
await this.store.updateTask(task.id, { status: null, error: null });
|
||
}
|
||
await this.store.logEntry(
|
||
task.id,
|
||
"Auto-recovered: in-review task idle past stuck-task timeout — kicked back to todo",
|
||
);
|
||
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||
log.log(`Kicked ghost review task ${task.id} back to todo`);
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to kick ghost review task ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Ghost review recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover stale `in-review` tasks left in a transient merge status.
|
||
*
|
||
* The direct AI merger can successfully create the final commit and then be
|
||
* interrupted before it stores mergeDetails and moves the task to `done`.
|
||
* When that happens no future task:moved event fires, so the merge queue has
|
||
* nothing to retry. This recovery confirms the task-specific commit exists on
|
||
* the current main lineage before finalizing the task.
|
||
*
|
||
* If no landed commit is found, it only clears the stale transient status so
|
||
* the normal mergeable-review recovery can retry the merge.
|
||
*
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
* @returns Number of tasks finalized or unblocked
|
||
*/
|
||
async recoverInterruptedMergingTasks(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
const timeoutMs = settings.taskStuckTimeoutMs;
|
||
if (!timeoutMs || timeoutMs <= 0) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const candidates = tasks.filter((task) =>
|
||
task.column === "in-review" &&
|
||
!task.paused &&
|
||
Boolean(task.status && ACTIVE_MERGE_STATUSES.has(task.status)) &&
|
||
this.isPastInterruptedMergeGrace(task, timeoutMs),
|
||
);
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
log.warn(`Found ${candidates.length} stale merging task(s) in in-review`);
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
try {
|
||
const landedCommit = await this.findLandedTaskCommit(task);
|
||
|
||
if (landedCommit) {
|
||
const mergeDetails: MergeDetails = {
|
||
commitSha: landedCommit.sha,
|
||
rebaseBaseSha: landedCommit.rebaseBaseSha,
|
||
filesChanged: landedCommit.filesChanged,
|
||
insertions: landedCommit.insertions,
|
||
deletions: landedCommit.deletions,
|
||
mergeCommitMessage: landedCommit.subject,
|
||
mergedAt: new Date().toISOString(),
|
||
mergeConfirmed: true,
|
||
prNumber: getPrimaryPrInfo(task)?.number,
|
||
};
|
||
|
||
await this.store.updateTask(task.id, {
|
||
status: null,
|
||
error: null,
|
||
mergeRetries: 0,
|
||
mergeDetails,
|
||
});
|
||
const movedTask = await this.store.moveTask(task.id, "done");
|
||
this.emitTaskMerged(movedTask, { mergeConfirmed: true });
|
||
await this.cleanupInterruptedMergeArtifacts(task);
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovered: stale merge status finalized from landed commit ${landedCommit.sha.slice(0, 8)}`,
|
||
);
|
||
log.log(`Recovered interrupted merge ${task.id}: finalized landed commit ${landedCommit.sha.slice(0, 8)}`);
|
||
recovered++;
|
||
continue;
|
||
}
|
||
|
||
await this.store.updateTask(task.id, { status: null, error: null });
|
||
await this.store.logEntry(
|
||
task.id,
|
||
"Auto-recovered: stale merge status cleared; merge will be retried",
|
||
);
|
||
log.log(`Recovered interrupted merge ${task.id}: cleared stale status for retry`);
|
||
try {
|
||
this.options.enqueueMerge?.(task.id);
|
||
} catch (enqueueErr: unknown) {
|
||
log.warn(
|
||
`Failed to re-enqueue ${task.id} after stale-merge recovery (will rely on polling sweep): ${enqueueErr instanceof Error ? enqueueErr.message : String(enqueueErr)}`,
|
||
);
|
||
}
|
||
recovered++;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover interrupted merge ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} interrupted merge task(s)`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Interrupted merge recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
private async readShortstatForSha(
|
||
sha: string,
|
||
rebaseBaseSha?: string,
|
||
): Promise<{ filesChanged: number; insertions: number; deletions: number } | null> {
|
||
try {
|
||
const command = rebaseBaseSha
|
||
? `git diff --shortstat ${shellQuote(`${rebaseBaseSha}..${sha}`)}`
|
||
: `git show --shortstat --format= ${shellQuote(sha)}`;
|
||
const stats = await execAsync(command, {
|
||
cwd: this.options.rootDir,
|
||
maxBuffer: 1024 * 1024,
|
||
});
|
||
const parsed = parseShortstat(stats.stdout);
|
||
return {
|
||
filesChanged: parsed.filesChanged ?? 0,
|
||
insertions: parsed.insertions ?? 0,
|
||
deletions: parsed.deletions ?? 0,
|
||
};
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private async readLandedFilesForSha(sha: string, rebaseBaseSha?: string): Promise<string[] | null> {
|
||
try {
|
||
const command = rebaseBaseSha
|
||
? `git diff --name-only ${shellQuote(`${rebaseBaseSha}..${sha}`)}`
|
||
: `git show --name-only --format= ${shellQuote(sha)}`;
|
||
const result = await execAsync(command, {
|
||
cwd: this.options.rootDir,
|
||
maxBuffer: 1024 * 1024,
|
||
});
|
||
const files = result.stdout
|
||
.split(/\r?\n/)
|
||
.map((line) => line.trim())
|
||
.filter(Boolean);
|
||
return files.length > 0 ? Array.from(new Set(files)) : [];
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async recoverDoneTaskMergeMetadata(): Promise<number> {
|
||
try {
|
||
const tasks = await this.store.listTasks({ column: "done", slim: true });
|
||
const candidates = tasks.filter((task) => task.column === "done" && !task.paused && Boolean(task.mergeDetails?.commitSha));
|
||
if (candidates.length === 0) return 0;
|
||
|
||
let repaired = 0;
|
||
for (const task of candidates) {
|
||
if (task.mergeDetails?.landedFilesAttributionRestricted || task.mergeDetails?.noOpVerifiedShortCircuit) {
|
||
log.log(`recoverDoneTaskMergeMetadata: skipped ${task.id} — attribution-restricted`);
|
||
continue;
|
||
}
|
||
try {
|
||
const storedSha = task.mergeDetails?.commitSha;
|
||
if (!storedSha) continue;
|
||
|
||
if (task.mergeDetails?.mergeConfirmed === true) {
|
||
const landed = await this.findLandedTaskCommit(task);
|
||
if (!landed || landed.sha !== storedSha) {
|
||
log.warn(
|
||
`Refusing to overwrite confirmed mergeDetails.commitSha for ${task.id} — stored SHA ${storedSha.slice(0, 8)} no longer reachable; preserving canonical attribution`,
|
||
);
|
||
continue;
|
||
}
|
||
|
||
const liveShortstat = await this.readShortstatForSha(storedSha, task.mergeDetails?.rebaseBaseSha);
|
||
const liveLandedFiles = await this.readLandedFilesForSha(storedSha, task.mergeDetails?.rebaseBaseSha);
|
||
const currentLandedFiles = task.mergeDetails?.landedFiles;
|
||
const landedFilesMismatch = Boolean(
|
||
liveLandedFiles && (
|
||
!currentLandedFiles ||
|
||
liveLandedFiles.length !== currentLandedFiles.length ||
|
||
liveLandedFiles.some((file, index) => currentLandedFiles[index] !== file)
|
||
),
|
||
);
|
||
const statsMismatch = Boolean(
|
||
liveShortstat && (
|
||
task.mergeDetails?.filesChanged !== liveShortstat.filesChanged ||
|
||
task.mergeDetails?.insertions !== liveShortstat.insertions ||
|
||
task.mergeDetails?.deletions !== liveShortstat.deletions
|
||
),
|
||
);
|
||
|
||
const needsMetadataRepair =
|
||
task.mergeDetails?.filesChanged === undefined ||
|
||
task.mergeDetails?.insertions === undefined ||
|
||
task.mergeDetails?.deletions === undefined ||
|
||
task.mergeDetails?.mergeCommitMessage === undefined ||
|
||
!currentLandedFiles ||
|
||
landedFilesMismatch ||
|
||
statsMismatch;
|
||
|
||
if (!needsMetadataRepair) continue;
|
||
|
||
const nextFilesChanged = liveShortstat?.filesChanged ?? task.mergeDetails?.filesChanged ?? landed.filesChanged;
|
||
const nextInsertions = liveShortstat?.insertions ?? task.mergeDetails?.insertions ?? landed.insertions;
|
||
const nextDeletions = liveShortstat?.deletions ?? task.mergeDetails?.deletions ?? landed.deletions;
|
||
|
||
await this.store.updateTask(task.id, {
|
||
mergeDetails: {
|
||
...task.mergeDetails,
|
||
filesChanged: nextFilesChanged,
|
||
insertions: nextInsertions,
|
||
deletions: nextDeletions,
|
||
landedFiles: liveLandedFiles ?? task.mergeDetails?.landedFiles,
|
||
mergeCommitMessage: task.mergeDetails?.mergeCommitMessage ?? landed.subject,
|
||
rebaseBaseSha: task.mergeDetails?.rebaseBaseSha ?? landed.rebaseBaseSha,
|
||
mergedAt: task.mergeDetails?.mergedAt ?? new Date().toISOString(),
|
||
prNumber: getPrimaryPrInfo(task)?.number,
|
||
},
|
||
modifiedFiles: liveLandedFiles && liveLandedFiles.length > 0 ? liveLandedFiles : undefined,
|
||
});
|
||
if ((statsMismatch && liveShortstat) || landedFilesMismatch) {
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovered: stale mergeDetails repaired (was ${task.mergeDetails?.filesChanged ?? "?"}/${task.mergeDetails?.insertions ?? "?"}/${task.mergeDetails?.deletions ?? "?"}, now ${liveShortstat?.filesChanged ?? nextFilesChanged}/${liveShortstat?.insertions ?? nextInsertions}/${liveShortstat?.deletions ?? nextDeletions})${landedFilesMismatch ? ` (files ${task.mergeDetails?.landedFiles?.length ?? 0} → ${liveLandedFiles?.length ?? task.mergeDetails?.landedFiles?.length ?? 0})` : ""} — sha unchanged ${storedSha.slice(0, 8)}`,
|
||
);
|
||
} else {
|
||
await this.store.logEntry(task.id, `Auto-recovered: reconciled done-task mergeDetails to owned commit ${landed.sha.slice(0, 8)}`);
|
||
}
|
||
repaired++;
|
||
continue;
|
||
}
|
||
|
||
const landed = await this.findLandedTaskCommit(task, { preferEarliestOwnedCommit: true });
|
||
if (!landed) {
|
||
await this.store.updateTask(task.id, { mergeDetails: undefined });
|
||
await this.store.logEntry(task.id, "Auto-recovered: cleared unowned done-task mergeDetails commitSha");
|
||
repaired++;
|
||
continue;
|
||
}
|
||
|
||
const landedStats = {
|
||
filesChanged: landed.filesChanged ?? 0,
|
||
insertions: landed.insertions ?? 0,
|
||
deletions: landed.deletions ?? 0,
|
||
};
|
||
const landedFiles = await this.readLandedFilesForSha(landed.sha, task.mergeDetails?.rebaseBaseSha ?? landed.rebaseBaseSha);
|
||
|
||
const needsRepair =
|
||
task.mergeDetails?.commitSha !== landed.sha ||
|
||
task.mergeDetails?.filesChanged === undefined ||
|
||
task.mergeDetails?.insertions === undefined ||
|
||
task.mergeDetails?.deletions === undefined ||
|
||
!task.mergeDetails?.landedFiles || (
|
||
task.mergeDetails?.commitSha === landed.sha && (
|
||
task.mergeDetails?.filesChanged !== landedStats.filesChanged ||
|
||
task.mergeDetails?.insertions !== landedStats.insertions ||
|
||
task.mergeDetails?.deletions !== landedStats.deletions ||
|
||
(landedFiles ? (
|
||
task.mergeDetails?.landedFiles?.length !== landedFiles.length ||
|
||
landedFiles.some((file, index) => task.mergeDetails?.landedFiles?.[index] !== file)
|
||
) : false)
|
||
)
|
||
);
|
||
|
||
if (!needsRepair) continue;
|
||
|
||
await this.store.updateTask(task.id, {
|
||
mergeDetails: {
|
||
...task.mergeDetails,
|
||
commitSha: landed.sha,
|
||
filesChanged: landedStats.filesChanged,
|
||
insertions: landedStats.insertions,
|
||
deletions: landedStats.deletions,
|
||
mergeCommitMessage: landed.subject,
|
||
rebaseBaseSha: task.mergeDetails?.rebaseBaseSha ?? landed.rebaseBaseSha,
|
||
landedFiles: landedFiles ?? task.mergeDetails?.landedFiles,
|
||
mergedAt: task.mergeDetails?.mergedAt ?? new Date().toISOString(),
|
||
mergeConfirmed: true,
|
||
prNumber: getPrimaryPrInfo(task)?.number,
|
||
},
|
||
modifiedFiles: landedFiles && landedFiles.length > 0 ? landedFiles : undefined,
|
||
});
|
||
await this.store.logEntry(task.id, `Auto-recovered: reconciled done-task mergeDetails to owned commit ${landed.sha.slice(0, 8)}`);
|
||
repaired++;
|
||
} catch (err: unknown) {
|
||
log.error(`Failed done-task merge metadata recovery for ${task.id}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
}
|
||
|
||
return repaired;
|
||
} catch (err: unknown) {
|
||
log.error(`Done-task merge metadata recovery failed: ${err instanceof Error ? err.message : String(err)}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
// ── Misclassified failure recovery ───────────────────────────────
|
||
|
||
/**
|
||
* Recover tasks that already merged successfully but never reached `done`.
|
||
*
|
||
* This catches races where the merge completed and merge metadata was stored,
|
||
* but a later transition failed or another process moved the task before the
|
||
* final `in-review` → `done` update completed.
|
||
*
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
* @returns Number of tasks recovered
|
||
*/
|
||
async recoverMergedReviewTasks(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
|
||
const mergedButNotDone = tasks.filter((t) =>
|
||
t.column === "in-review" &&
|
||
t.mergeDetails?.mergeConfirmed === true,
|
||
);
|
||
|
||
if (mergedButNotDone.length === 0) return 0;
|
||
|
||
log.warn(`Found ${mergedButNotDone.length} merged task(s) stuck in in-review`);
|
||
|
||
let recovered = 0;
|
||
for (const task of mergedButNotDone) {
|
||
try {
|
||
const hardBlocker = getTaskHardMergeBlocker({
|
||
...task,
|
||
steps: task.steps ?? [],
|
||
workflowStepResults: task.workflowStepResults,
|
||
});
|
||
if (hardBlocker) {
|
||
await this.store.updateTask(task.id, {
|
||
status: "failed",
|
||
error: `Merge confirmed but finalization blocked: ${hardBlocker}`,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovery skipped: merge confirmed but finalization blocked — ${hardBlocker}`,
|
||
);
|
||
continue;
|
||
}
|
||
|
||
const clearedFlags = {
|
||
paused: Boolean(task.paused),
|
||
status: Boolean(task.status),
|
||
error: Boolean(task.error),
|
||
};
|
||
await this.store.updateTask(task.id, {
|
||
paused: false,
|
||
status: null,
|
||
error: null,
|
||
mergeRetries: 0,
|
||
});
|
||
const movedTask = await this.store.moveTask(task.id, "done");
|
||
this.emitTaskMerged(movedTask, { mergeConfirmed: true });
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-finalized from in-review/paused: content proven via mergeConfirmed metadata. Cleared soft state paused=${clearedFlags.paused}, status=${clearedFlags.status}, error=${clearedFlags.error}`,
|
||
);
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "recover-merged-review",
|
||
});
|
||
await auditor.database({
|
||
type: "task:auto-recover-finalize-already-on-main",
|
||
target: task.id,
|
||
metadata: {
|
||
mergeSha: task.mergeDetails?.commitSha ?? null,
|
||
baseBranch: task.baseBranch || task.executionStartBranch || "main",
|
||
clearedFlags,
|
||
},
|
||
});
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`recoverMergedReviewTasks: failed to record run-audit event for ${task.id}: ${errorMessage}`);
|
||
}
|
||
log.log(`Recovered merged task ${task.id}: moved to done`);
|
||
recovered++;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover merged task ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} merged task(s) → done`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Merged review recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover deadlocked retry-exhausted merge failures that are still blocking
|
||
* dispatch via `blockedBy` or retained worktree ownership.
|
||
*/
|
||
/**
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async recoverStuckMergeDeadlocks(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const now = Date.now();
|
||
const inReview = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const triage = await this.store.listTasks({ column: "triage", slim: true });
|
||
const todo = await this.store.listTasks({ column: "todo", slim: true });
|
||
const inProgress = await this.store.listTasks({ column: "in-progress", slim: true });
|
||
|
||
const dependentsByBlocker = new Map<string, Task[]>();
|
||
for (const task of [...triage, ...todo, ...inProgress]) {
|
||
if (!task.blockedBy) continue;
|
||
const dependents = dependentsByBlocker.get(task.blockedBy) ?? [];
|
||
dependents.push(task);
|
||
dependentsByBlocker.set(task.blockedBy, dependents);
|
||
}
|
||
|
||
const candidates = inReview.filter((task) => {
|
||
const cooldownStart = this.deadlockRecoveryCooldown.get(task.id) ?? 0;
|
||
const cooldownElapsed = now - cooldownStart;
|
||
const hasBlockedDependents = (dependentsByBlocker.get(task.id) ?? []).some(
|
||
(dep) => dep.column === "triage" || dep.column === "todo",
|
||
);
|
||
return task.column === "in-review" &&
|
||
!task.paused &&
|
||
task.status === "failed" &&
|
||
(task.mergeRetries ?? 0) >= MAX_AUTO_MERGE_RETRIES &&
|
||
task.mergeDetails?.mergeConfirmed !== true &&
|
||
(hasBlockedDependents || Boolean(task.worktree)) &&
|
||
cooldownElapsed >= DEADLOCK_RECOVERY_COOLDOWN_MS;
|
||
});
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
const blockedDependents = dependentsByBlocker.get(task.id) ?? [];
|
||
const blockedTaskIds = blockedDependents.map((dep) => dep.id);
|
||
try {
|
||
const landedCommit = await this.findLandedTaskCommit(task);
|
||
if (landedCommit) {
|
||
const mergeDetails: MergeDetails = {
|
||
commitSha: landedCommit.sha,
|
||
rebaseBaseSha: landedCommit.rebaseBaseSha,
|
||
filesChanged: landedCommit.filesChanged,
|
||
insertions: landedCommit.insertions,
|
||
deletions: landedCommit.deletions,
|
||
mergeCommitMessage: landedCommit.subject,
|
||
mergedAt: new Date().toISOString(),
|
||
mergeConfirmed: true,
|
||
prNumber: getPrimaryPrInfo(task)?.number,
|
||
};
|
||
|
||
await this.store.updateTask(task.id, {
|
||
status: null,
|
||
error: null,
|
||
mergeRetries: 0,
|
||
worktree: null,
|
||
branch: null,
|
||
mergeDetails,
|
||
});
|
||
const movedTask = await this.store.moveTask(task.id, "done");
|
||
this.emitTaskMerged(movedTask, { mergeConfirmed: true });
|
||
await this.cleanupInterruptedMergeArtifacts(task);
|
||
|
||
const clearedDependents: string[] = [];
|
||
for (const dep of blockedDependents) {
|
||
try {
|
||
await this.store.updateTask(dep.id, { blockedBy: null });
|
||
await this.store.logEntry(dep.id, `Auto-recovered: cleared stale blockedBy ${task.id} after deadlock recovery`);
|
||
clearedDependents.push(dep.id);
|
||
} catch (depErr: unknown) {
|
||
const depErrMessage = depErr instanceof Error ? depErr.message : String(depErr);
|
||
log.warn(`self-heal:deadlock-recovery-dependent-error ${JSON.stringify({ blockerTaskId: task.id, dependentTaskId: dep.id, error: depErrMessage })}`);
|
||
}
|
||
}
|
||
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovered: merge deadlock resolved via landed commit ${landedCommit.sha.slice(0, 8)}${clearedDependents.length > 0 ? `; cleared blockedBy on ${clearedDependents.join(", ")}` : ""}`,
|
||
);
|
||
log.log(`self-heal:deadlock-recovered ${JSON.stringify({ stuckTaskId: task.id, blockedTaskIds, attributedSha: landedCommit.sha, action: "reattributed" })}`);
|
||
recovered++;
|
||
} else {
|
||
await this.store.updateTask(task.id, { paused: true });
|
||
await this.store.logEntry(task.id, "merge-deadlock-detected: requires manual intervention — verified content not on main");
|
||
log.warn(`self-heal:deadlock-recovered ${JSON.stringify({ stuckTaskId: task.id, blockedTaskIds, attributedSha: null, action: "paused-for-manual" })}`);
|
||
recovered++;
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`self-heal:deadlock-recovery-error ${JSON.stringify({ stuckTaskId: task.id, blockedTaskIds, error: errorMessage })}`);
|
||
} finally {
|
||
this.deadlockRecoveryCooldown.set(task.id, Date.now());
|
||
}
|
||
}
|
||
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Stuck merge deadlock recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
private parseScopeViolationPayload(detail: string): { declaredScope: string[]; stagedFiles: string[] } | null {
|
||
const lines = detail.split("\n");
|
||
const declaredScope: string[] = [];
|
||
const stagedFiles: string[] = [];
|
||
let section: "declared" | "staged" | null = null;
|
||
|
||
for (const rawLine of lines) {
|
||
const line = rawLine.trim();
|
||
if (line === "declaredScope:") {
|
||
section = "declared";
|
||
continue;
|
||
}
|
||
if (line === "stagedFiles:") {
|
||
section = "staged";
|
||
continue;
|
||
}
|
||
if (!line.startsWith("- ")) continue;
|
||
const value = line.slice(2).trim();
|
||
if (!value || value === "<none>") continue;
|
||
if (section === "declared") declaredScope.push(value);
|
||
if (section === "staged") stagedFiles.push(value);
|
||
}
|
||
|
||
if (declaredScope.length === 0 && stagedFiles.length === 0) return null;
|
||
return { declaredScope, stagedFiles };
|
||
}
|
||
|
||
private parseScopeViolationFromError(errorMessage: string | null | undefined): { declaredScope: string[]; stagedFiles: string[] } | null {
|
||
if (!errorMessage?.startsWith("File-scope invariant violation for ")) {
|
||
return null;
|
||
}
|
||
const stagedMatch = errorMessage.match(/staged files \[(.*?)\] have zero overlap/s);
|
||
const scopeMatch = errorMessage.match(/declared File Scope \[(.*?)\]\./s);
|
||
if (!stagedMatch || !scopeMatch) return null;
|
||
const stagedFiles = stagedMatch[1]
|
||
.split(",")
|
||
.map((entry) => entry.trim())
|
||
.filter((entry) => Boolean(entry) && entry !== "<none outside .changeset/>");
|
||
const declaredScope = scopeMatch[1]
|
||
.split(",")
|
||
.map((entry) => entry.trim())
|
||
.filter(Boolean);
|
||
return { declaredScope, stagedFiles };
|
||
}
|
||
|
||
/**
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async recoverOrphanOnlyScopeViolations(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const candidates = tasks.filter((task) =>
|
||
task.column === "in-review" &&
|
||
task.status === "failed" &&
|
||
task.scopeOverride !== true &&
|
||
task.mergeDetails?.mergeConfirmed !== true &&
|
||
!executingIds.has(task.id),
|
||
);
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
try {
|
||
const recentLogs = "getAgentLogs" in this.store && typeof this.store.getAgentLogs === "function"
|
||
? await this.store.getAgentLogs(task.id, { limit: 50 })
|
||
: [];
|
||
const scopeViolationLog = recentLogs.find((entry) =>
|
||
entry.type === "tool_error" &&
|
||
entry.detail?.includes("declaredScope:") &&
|
||
entry.detail?.includes("stagedFiles:"),
|
||
);
|
||
|
||
const parsed = (scopeViolationLog?.detail ? this.parseScopeViolationPayload(scopeViolationLog.detail) : null)
|
||
?? this.parseScopeViolationFromError(task.error);
|
||
if (!parsed) continue;
|
||
|
||
const { declaredScope, stagedFiles } = parsed;
|
||
if (declaredScope.length === 0) continue;
|
||
|
||
const orphanFiles = stagedFiles.filter((file) => !file.startsWith(".changeset/"));
|
||
if (orphanFiles.length === 0) continue;
|
||
const hasDeclaredOverlap = orphanFiles.some((file) => matchesScope(file, declaredScope));
|
||
if (hasDeclaredOverlap) continue;
|
||
|
||
const baseBranch = task.baseBranch || task.executionStartBranch || "main";
|
||
const landed = await this.findAlreadyMergedTaskCommit({
|
||
taskId: task.id,
|
||
lineageId: task.lineageId,
|
||
repoDir: this.options.rootDir,
|
||
baseBranch,
|
||
taskBranch: task.branch,
|
||
baseCommitSha: task.baseCommitSha,
|
||
});
|
||
if (!landed) continue;
|
||
|
||
const mergeDetails: MergeDetails = {
|
||
commitSha: landed.sha,
|
||
mergedAt: new Date().toISOString(),
|
||
mergeConfirmed: true,
|
||
resolutionStrategy: "orphan-discard-no-op",
|
||
};
|
||
|
||
const hardBlocker = getTaskHardMergeBlocker({
|
||
...task,
|
||
steps: task.steps ?? [],
|
||
workflowStepResults: task.workflowStepResults,
|
||
});
|
||
if (hardBlocker) {
|
||
await this.store.updateTask(task.id, {
|
||
status: "failed",
|
||
error: `Merge confirmed but finalization blocked: ${hardBlocker}`,
|
||
mergeDetails,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovery parked task in in-review: merged content found on ${baseBranch} (${landed.sha.slice(0, 8)}) but finalization blocked — ${hardBlocker}`,
|
||
);
|
||
continue;
|
||
}
|
||
|
||
const clearedFlags = {
|
||
paused: Boolean(task.paused),
|
||
status: Boolean(task.status),
|
||
error: Boolean(task.error),
|
||
};
|
||
await this.store.updateTask(task.id, {
|
||
paused: false,
|
||
status: null,
|
||
error: null,
|
||
mergeRetries: 0,
|
||
mergeDetails,
|
||
});
|
||
const movedTask = await this.store.moveTask(task.id, "done");
|
||
this.emitTaskMerged(movedTask, { mergeConfirmed: true });
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-finalized from in-review/paused: content proven on ${baseBranch} (${landed.sha.slice(0, 8)}). Cleared soft state paused=${clearedFlags.paused}, status=${clearedFlags.status}, error=${clearedFlags.error}`,
|
||
);
|
||
await this.cleanupWorktreeOnly(task);
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "recover-orphan-only-scope-violations",
|
||
});
|
||
await auditor.database({
|
||
type: "task:auto-recover-finalize-already-on-main",
|
||
target: task.id,
|
||
metadata: {
|
||
mergeSha: landed.sha,
|
||
baseBranch,
|
||
mergeStrategy: landed.strategy,
|
||
clearedFlags,
|
||
},
|
||
});
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`recoverOrphanOnlyScopeViolations: failed to record run-audit event for ${task.id}: ${errorMessage}`);
|
||
}
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`recoverOrphanOnlyScopeViolations: failed for ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} orphan-only scope-violation task(s) → done`);
|
||
}
|
||
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Orphan-only scope violation recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover retry-exhausted failed review tasks whose content already landed on
|
||
* the integration branch via a non-canonical merge lineage.
|
||
*
|
||
* Candidate filter:
|
||
* - `column === "in-review"`
|
||
* - `status === "failed"`
|
||
* - `(mergeRetries ?? 0) >= MAX_AUTO_MERGE_RETRIES`
|
||
* - `mergeDetails.mergeConfirmed !== true`
|
||
* - not actively executing
|
||
*
|
||
* Detection order (first match wins):
|
||
* 1. Fusion-Task-Id trailer lookup on the base branch
|
||
* 2. Task branch ancestry + task-id grep on first-parent base lineage
|
||
* 3. Patch-id match between task branch diff and recent base-branch commits
|
||
*
|
||
* Idempotency: recovered tasks are moved to `done`, status/error are cleared,
|
||
* and mergeRetries reset to 0, so subsequent sweeps will not match them.
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async recoverAlreadyMergedReviewTasks(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const candidates = tasks.filter((task) =>
|
||
task.column === "in-review" &&
|
||
task.status === "failed" &&
|
||
(task.mergeRetries ?? 0) >= MAX_AUTO_MERGE_RETRIES &&
|
||
task.mergeDetails?.mergeConfirmed !== true &&
|
||
!executingIds.has(task.id),
|
||
);
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
try {
|
||
const baseBranch = task.baseBranch || task.executionStartBranch || "main";
|
||
if (!baseBranch) continue;
|
||
|
||
const landed = await this.findAlreadyMergedTaskCommit({
|
||
taskId: task.id,
|
||
lineageId: task.lineageId,
|
||
repoDir: this.options.rootDir,
|
||
baseBranch,
|
||
taskBranch: task.branch,
|
||
baseCommitSha: task.baseCommitSha,
|
||
});
|
||
if (!landed) continue;
|
||
|
||
const mergeDetails: MergeDetails = {
|
||
commitSha: landed.sha,
|
||
mergedAt: new Date().toISOString(),
|
||
mergeConfirmed: true,
|
||
prNumber: getPrimaryPrInfo(task)?.number,
|
||
};
|
||
|
||
const hardBlocker = getTaskHardMergeBlocker({
|
||
...task,
|
||
steps: task.steps ?? [],
|
||
workflowStepResults: task.workflowStepResults,
|
||
});
|
||
if (hardBlocker) {
|
||
await this.store.updateTask(task.id, {
|
||
status: "failed",
|
||
error: `Merge confirmed but finalization blocked: ${hardBlocker}`,
|
||
mergeDetails,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovery parked task in in-review: merged content found on ${baseBranch} (${landed.sha.slice(0, 8)}) but finalization blocked — ${hardBlocker}`,
|
||
);
|
||
continue;
|
||
}
|
||
|
||
const clearedFlags = {
|
||
paused: Boolean(task.paused),
|
||
status: Boolean(task.status),
|
||
error: Boolean(task.error),
|
||
};
|
||
await this.store.updateTask(task.id, {
|
||
paused: false,
|
||
status: null,
|
||
error: null,
|
||
mergeRetries: 0,
|
||
mergeDetails,
|
||
});
|
||
const worktreeHint = task.worktree;
|
||
const movedTask = await this.store.moveTask(task.id, "done");
|
||
this.emitTaskMerged(movedTask, { mergeConfirmed: true });
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-finalized from in-review/paused: content proven on ${baseBranch} (${landed.sha.slice(0, 8)}). Cleared soft state paused=${clearedFlags.paused}, status=${clearedFlags.status}, error=${clearedFlags.error}`,
|
||
);
|
||
await this.reconcileCompletedTask(task.id, { worktreeHint });
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "recover-already-merged-review",
|
||
});
|
||
await auditor.database({
|
||
type: "task:auto-recover-finalize-already-on-main",
|
||
target: task.id,
|
||
metadata: {
|
||
mergeSha: landed.sha,
|
||
mergeStrategy: landed.strategy,
|
||
baseBranch,
|
||
mergeRetries: task.mergeRetries ?? 0,
|
||
clearedFlags,
|
||
},
|
||
});
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`recoverAlreadyMergedReviewTasks: failed to record run-audit event for ${task.id}: ${errorMessage}`);
|
||
}
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`recoverAlreadyMergedReviewTasks: failed for ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} already-merged retry-exhausted review task(s) → done`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Already-merged review recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async recoverCompletionHandoffLimbo(): Promise<void> {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return;
|
||
if (settings.autoMerge === false) return;
|
||
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: false });
|
||
const now = Date.now();
|
||
|
||
for (const task of tasks) {
|
||
if (task.column !== "in-review" || task.paused) continue;
|
||
if (task.status != null || task.mergeDetails != null || task.review != null || task.reviewState != null) continue;
|
||
if (this.options.isTaskActive?.(task.id)) continue;
|
||
if (getTaskMergeBlocker(task) !== undefined) continue;
|
||
|
||
const doneMarker = [...(task.log ?? [])].reverse().find((entry) => entry.action === "Task marked done by agent");
|
||
if (!doneMarker?.timestamp) continue;
|
||
const markerTs = Date.parse(doneMarker.timestamp);
|
||
if (!Number.isFinite(markerTs)) continue;
|
||
const ageMs = now - markerTs;
|
||
if (ageMs < COMPLETION_HANDOFF_LIMBO_GRACE_MS) continue;
|
||
|
||
const currentCount = task.completionHandoffLimboRecoveryCount ?? 0;
|
||
if (currentCount >= MAX_COMPLETION_HANDOFF_LIMBO_RECOVERIES) {
|
||
await this.store.updateTask(task.id, {
|
||
status: "failed",
|
||
error: "Completion handoff limbo recovery exhausted",
|
||
});
|
||
const exhaustedAudit = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "recover-completion-handoff-limbo",
|
||
});
|
||
await exhaustedAudit.database({
|
||
type: "task:auto-recover-completion-handoff-limbo-exhausted",
|
||
target: task.id,
|
||
metadata: { ageMs, attempts: currentCount },
|
||
});
|
||
continue;
|
||
}
|
||
|
||
await this.store.updateTask(task.id, {
|
||
completionHandoffLimboRecoveryCount: currentCount + 1,
|
||
});
|
||
|
||
const audit = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "recover-completion-handoff-limbo",
|
||
});
|
||
await audit.database({
|
||
type: "task:auto-recover-completion-handoff-limbo",
|
||
target: task.id,
|
||
metadata: { ageMs, source: "self-healing-in-review-sweep" },
|
||
});
|
||
|
||
await this.store.logEntry(task.id, "Auto-recovered (FN-4999): task in 'in-review' past handoff grace with no merge fan-out — re-emitting auto-merge handoff");
|
||
if (this.options.requeueForAutoMerge) {
|
||
await this.options.requeueForAutoMerge(task.id);
|
||
} else {
|
||
log.warn(`recoverCompletionHandoffLimbo: requeueForAutoMerge callback missing for ${task.id}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
private async isBranchTipMisboundToTask(input: {
|
||
branch: string;
|
||
taskId: string;
|
||
lineageId?: string;
|
||
baseBranch: string;
|
||
}): Promise<{ misbound: boolean; branchTip: string; landed: Awaited<ReturnType<typeof findAlreadyMergedTaskCommit>> }> {
|
||
const { branch, taskId, lineageId, baseBranch } = input;
|
||
const { stdout: bodyOut } = await execAsync(`git log -1 --format=%B ${shellQuote(branch)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
maxBuffer: 1024 * 1024,
|
||
});
|
||
const body = bodyOut;
|
||
const hasTaskId = body.includes(`Fusion-Task-Id: ${taskId}`);
|
||
const hasLineage = lineageId ? body.includes(`Fusion-Task-Lineage: ${lineageId}`) : false;
|
||
const { stdout: tipOut } = await execAsync(`git rev-parse ${shellQuote(branch)}`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
maxBuffer: 1024 * 1024,
|
||
});
|
||
const branchTip = tipOut.trim();
|
||
const landed = await this.findAlreadyMergedTaskCommit({
|
||
taskId,
|
||
lineageId,
|
||
repoDir: this.options.rootDir,
|
||
baseBranch,
|
||
taskBranch: branch,
|
||
});
|
||
return { misbound: !hasTaskId && !hasLineage, branchTip, landed };
|
||
}
|
||
|
||
async recoverBranchMisboundInReviewTasks(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const candidates = tasks.filter((task) =>
|
||
task.column === "in-review" &&
|
||
Boolean(task.branch) &&
|
||
task.mergeDetails?.mergeConfirmed !== true &&
|
||
!executingIds.has(task.id),
|
||
);
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
try {
|
||
const branch = task.branch;
|
||
if (!branch) continue;
|
||
const baseBranch = task.baseBranch || task.executionStartBranch || "main";
|
||
const check = await this.isBranchTipMisboundToTask({
|
||
branch,
|
||
taskId: task.id,
|
||
lineageId: task.lineageId,
|
||
baseBranch,
|
||
});
|
||
if (!check.misbound || !check.landed) continue;
|
||
|
||
const mergeDetails: MergeDetails = {
|
||
commitSha: check.landed.sha,
|
||
mergedAt: new Date().toISOString(),
|
||
mergeConfirmed: true,
|
||
prNumber: getPrimaryPrInfo(task)?.number,
|
||
};
|
||
|
||
await this.store.updateTask(task.id, {
|
||
mergeDetails,
|
||
branch: null,
|
||
worktree: null,
|
||
status: null,
|
||
error: null,
|
||
});
|
||
|
||
if (task.worktree && existsSync(task.worktree)) {
|
||
await removeWorktree({
|
||
rootDir: this.options.rootDir,
|
||
worktreePath: task.worktree,
|
||
settings,
|
||
taskId: task.id,
|
||
reason: RemovalReason.SelfHealingOrphanRescue,
|
||
}).catch(() => undefined);
|
||
}
|
||
|
||
await this.clearCompletionBranchIfSubsumed(task, branch).catch(() => false);
|
||
|
||
// FN-5092 hotfix: clear transient merger-queue status (`status: "merging"` set
|
||
// by the original merger attempt) before transitioning to done. Without this,
|
||
// a stale `mergeActive` slot for this task leaks indefinitely and blocks the
|
||
// entire merger queue until engine restart. Mirrors the pattern in
|
||
// merger.ts completeTask() and project-engine.ts auto-merge already-confirmed path.
|
||
await this.store.updateTask(task.id, { status: null, error: null, paused: false });
|
||
const movedTask = await this.store.moveTask(task.id, "done");
|
||
this.emitTaskMerged(movedTask, { mergeConfirmed: true });
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovered: branch tip misbound but content found on ${baseBranch} at ${check.landed.sha.slice(0, 8)} via ${check.landed.strategy}`,
|
||
);
|
||
await this.reconcileCompletedTask(task.id, { worktreeHint: task.worktree ?? undefined });
|
||
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "recover-branch-misbound-in-review",
|
||
});
|
||
await auditor.database({
|
||
type: "task:auto-recover-branch-misbound",
|
||
target: task.id,
|
||
metadata: {
|
||
branch,
|
||
branchTip: check.branchTip,
|
||
mergeSha: check.landed.sha,
|
||
mergeStrategy: check.landed.strategy,
|
||
lineageId: task.lineageId,
|
||
baseBranch,
|
||
},
|
||
});
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`recoverBranchMisboundInReviewTasks: failed to record run-audit event for ${task.id}: ${errorMessage}`);
|
||
}
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`recoverBranchMisboundInReviewTasks: failed for task ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Branch-misbound in-review recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async recoverForeignOnlyContaminatedInReviewTasks(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
const inReview = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const inProgress = await this.store.listTasks({ column: "in-progress", slim: true });
|
||
const candidates = [
|
||
...inReview.filter((task) =>
|
||
task.column === "in-review" &&
|
||
Boolean(task.branch) &&
|
||
Boolean(task.worktree) &&
|
||
task.mergeDetails?.mergeConfirmed !== true &&
|
||
!task.userPaused &&
|
||
!executingIds.has(task.id),
|
||
),
|
||
...inProgress.filter((task) =>
|
||
task.column === "in-progress" &&
|
||
task.paused === true &&
|
||
(task.pausedReason === "branch-cross-contamination" || task.pausedReason === "branch-conflict-unrecoverable") &&
|
||
Boolean(task.branch) &&
|
||
Boolean(task.worktree) &&
|
||
!task.userPaused &&
|
||
!executingIds.has(task.id),
|
||
),
|
||
];
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
if (!task.branch || !task.worktree) continue;
|
||
const baseSha = task.baseCommitSha ?? task.baseBranch ?? task.executionStartBranch ?? "main";
|
||
try {
|
||
const classification = await classifyForeignOnlyContamination({
|
||
repoDir: this.options.rootDir,
|
||
branchName: task.branch,
|
||
baseSha,
|
||
taskId: task.id,
|
||
});
|
||
|
||
if (classification.kind === "ambiguous" || classification.kind === "clean") {
|
||
await createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "recover-foreign-only-contamination-in-review",
|
||
}).database({
|
||
type: "task:auto-recover-foreign-only-contamination-skipped",
|
||
target: task.id,
|
||
metadata: { reason: classification.kind === "clean" ? "clean" : "ambiguous", kind: classification.kind },
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const result = await recoverForeignOnlyContamination(task, {
|
||
repoDir: this.options.rootDir,
|
||
taskStore: this.store,
|
||
runAudit: createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "recover-foreign-only-contamination-in-review",
|
||
}),
|
||
});
|
||
if (result.recovered) {
|
||
await this.store.logEntry(task.id, `Auto-recovered foreign-only contamination via ${result.subtype ?? "unknown"}`);
|
||
recovered += 1;
|
||
}
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`recoverForeignOnlyContaminatedInReviewTasks: failed for task ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Foreign-only contamination recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover tasks in `in-review` marked as `failed` where all steps are
|
||
* actually done. This catches the case where an agent completed all work
|
||
* but the session ended without calling `fn_task_done` (e.g., context
|
||
* overflow, compaction losing tool awareness). The executor marks these
|
||
* as failed, but the work is complete — clear the error so the normal
|
||
* review flow can proceed.
|
||
*
|
||
* @returns Number of tasks recovered
|
||
*/
|
||
async recoverMisclassifiedFailures(): Promise<number> {
|
||
try {
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
|
||
const misclassified = tasks.filter((t) =>
|
||
t.column === "in-review" &&
|
||
!t.paused &&
|
||
t.status === "failed" &&
|
||
isNoTaskDoneFailure(t) &&
|
||
t.steps.length > 0 &&
|
||
t.steps.every((s) => s.status === "done" || s.status === "skipped"),
|
||
);
|
||
|
||
if (misclassified.length === 0) return 0;
|
||
|
||
log.warn(`Found ${misclassified.length} misclassified failure(s) with all steps done`);
|
||
|
||
let recovered = 0;
|
||
for (const task of misclassified) {
|
||
try {
|
||
await this.store.updateTask(task.id, {
|
||
status: null,
|
||
error: null,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
"Auto-recovered: all steps complete despite 'no fn_task_done' failure — cleared error for normal review",
|
||
);
|
||
log.log(`Recovered misclassified failure ${task.id}: ${task.title || task.description?.slice(0, 60) || "(untitled)"}`);
|
||
recovered++;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover misclassified failure ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} misclassified failure(s) → cleared for review`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Misclassified failure recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
async auditNoCommitsExpectedCandidates(): Promise<number> {
|
||
try {
|
||
const inReviewTasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const allTasks = await this.store.listTasks({ slim: true });
|
||
const failedTasks = allTasks.filter((task) => task.status === "failed");
|
||
const candidateMap = new Map<string, Task>();
|
||
for (const task of [...inReviewTasks, ...failedTasks]) {
|
||
candidateMap.set(task.id, task);
|
||
}
|
||
const candidates = [...candidateMap.values()].filter((task) => {
|
||
if (task.noCommitsExpected === true) return false;
|
||
if (task.steps.length === 0 || !task.steps.every((step) => step.status === "done" || step.status === "skipped")) return false;
|
||
const noCommitsError = typeof task.error === "string" && /no_commits/i.test(task.error);
|
||
return task.column === "in-review" || noCommitsError;
|
||
});
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
const taskIds: string[] = [];
|
||
for (const task of candidates) {
|
||
const ahead = await isBranchAheadOfBase(task, this.options.rootDir, task.baseBranch || "main");
|
||
if (ahead && ahead.aheadCount === 0) {
|
||
taskIds.push(task.id);
|
||
}
|
||
}
|
||
|
||
if (taskIds.length > 0) {
|
||
log.warn(`no-commits-expected audit candidates: ${JSON.stringify({ taskIds })}`);
|
||
}
|
||
|
||
return taskIds.length;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`No-commits-expected audit failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover executor tasks stranded in `in-progress` before a real session was
|
||
* established, typically when the scheduler reserved a worktree path but the
|
||
* executor never materialized it or crashed before tracking the run.
|
||
*/
|
||
async recoverOrphanedExecutions(): Promise<number> {
|
||
try {
|
||
const tasks = await this.store.listTasks({ column: "in-progress", slim: true });
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
const now = Date.now();
|
||
|
||
const orphaned = tasks.filter((t) => {
|
||
if (t.column !== "in-progress" || t.paused || executingIds.has(t.id) || isTaskWorkComplete(t)) {
|
||
return false;
|
||
}
|
||
const staleness = now - new Date(t.updatedAt).getTime();
|
||
// Tasks with an existing worktree get a longer grace period to avoid
|
||
// racing with executor.resumeOrphaned() on engine startup.
|
||
const hasWorktree = t.worktree && existsSync(t.worktree);
|
||
const graceMs = hasWorktree ? ORPHANED_WITH_WORKTREE_GRACE_MS : ORPHANED_EXECUTION_RECOVERY_GRACE_MS;
|
||
return staleness >= graceMs;
|
||
});
|
||
|
||
if (orphaned.length === 0) return 0;
|
||
|
||
log.warn(`Found ${orphaned.length} orphaned executor task(s) stuck in in-progress`);
|
||
|
||
let recovered = 0;
|
||
for (const task of orphaned) {
|
||
try {
|
||
const hadWorktree = task.worktree && existsSync(task.worktree);
|
||
const reason = hadWorktree
|
||
? "worktree exists but no active session"
|
||
: "missing worktree/session";
|
||
|
||
if (this.options.leaseManager && task.checkedOutBy) {
|
||
const leaseRecovered = await this.options.leaseManager.recoverAbandonedLease(
|
||
task.id,
|
||
`orphaned execution: ${reason}`,
|
||
{ preserveProgress: true },
|
||
);
|
||
if (leaseRecovered) {
|
||
recovered++;
|
||
continue;
|
||
}
|
||
await this.options.leaseManager.reconcileLeaseRow(task.id);
|
||
}
|
||
|
||
// Reset steps whose work was never committed before clearing the worktree
|
||
await this.resetStepsIfWorkLost(task);
|
||
|
||
await this.store.updateTask(task.id, {
|
||
status: "stuck-killed",
|
||
worktree: null,
|
||
branch: null,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovered orphaned executor task — ${reason}, moved back to todo`,
|
||
);
|
||
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||
recovered++;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover orphaned executor task ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} orphaned executor task(s) → todo`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Orphaned executor recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
private getDurableAgentRecoveryState(agent: { metadata?: Record<string, unknown> | null }): {
|
||
attempts: number;
|
||
nextRetryAt?: string;
|
||
exhausted?: boolean;
|
||
lastMissingModulePath?: string;
|
||
consecutiveMissingModulePathCount: number;
|
||
} {
|
||
const metadata = agent.metadata ?? {};
|
||
const raw = metadata.durableErrorRecovery;
|
||
if (!raw || typeof raw !== "object") {
|
||
return { attempts: 0, consecutiveMissingModulePathCount: 0 };
|
||
}
|
||
const record = raw as Record<string, unknown>;
|
||
const attempts = typeof record.attempts === "number" && Number.isFinite(record.attempts)
|
||
? Math.max(0, Math.floor(record.attempts))
|
||
: 0;
|
||
const consecutiveMissingModulePathCount =
|
||
typeof record.consecutiveMissingModulePathCount === "number" && Number.isFinite(record.consecutiveMissingModulePathCount)
|
||
? Math.max(0, Math.floor(record.consecutiveMissingModulePathCount))
|
||
: 0;
|
||
return {
|
||
attempts,
|
||
nextRetryAt: typeof record.nextRetryAt === "string" ? record.nextRetryAt : undefined,
|
||
exhausted: record.exhausted === true,
|
||
lastMissingModulePath: typeof record.lastMissingModulePath === "string" ? record.lastMissingModulePath : undefined,
|
||
consecutiveMissingModulePathCount,
|
||
};
|
||
}
|
||
|
||
private computeDurableAgentRecoveryCooldownMs(attempts: number): number {
|
||
const clampedAttempts = Math.max(1, attempts);
|
||
const exponential = DURABLE_ERROR_RECOVERY_BASE_COOLDOWN_MS * Math.pow(2, clampedAttempts - 1);
|
||
return Math.min(exponential, DURABLE_ERROR_RECOVERY_MAX_COOLDOWN_MS);
|
||
}
|
||
|
||
async recoverAgentsRunningOnInactiveTasks(): Promise<number> {
|
||
const agentStore = this.options.agentStore;
|
||
if (!agentStore) {
|
||
return 0;
|
||
}
|
||
|
||
const now = Date.now();
|
||
const recoveredAgentIds = new Set<string>();
|
||
const runningAgents = await agentStore.listAgents({ state: "running", includeEphemeral: true });
|
||
|
||
for (const agent of runningAgents) {
|
||
if (isEphemeralAgent(agent) || !agent.taskId) {
|
||
continue;
|
||
}
|
||
|
||
const linkedTask = await this.store.getTask(agent.taskId);
|
||
if (linkedTask && (linkedTask.column === "in-progress" || linkedTask.column === "in-review" || linkedTask.column === "done" || linkedTask.column === "archived")) {
|
||
continue;
|
||
}
|
||
|
||
const activeRun = await agentStore.getActiveHeartbeatRun(agent.id);
|
||
const runStartedAt = activeRun?.startedAt;
|
||
const runAgeMs = runStartedAt ? now - Date.parse(runStartedAt) : Number.POSITIVE_INFINITY;
|
||
const hasFreshRun = Boolean(activeRun) && Number.isFinite(runAgeMs) && runAgeMs <= RUNNING_ON_INACTIVE_TASK_STALE_RUN_MS;
|
||
if (hasFreshRun || this.options.hasActiveAgentExecution?.(agent.id) === true) {
|
||
continue;
|
||
}
|
||
|
||
await agentStore.updateAgentState(agent.id, "active");
|
||
await agentStore.syncExecutionTaskLink(agent.id, undefined);
|
||
recoveredAgentIds.add(agent.id);
|
||
log.log(`Recovered running durable agent ${agent.id} on inactive task ${agent.taskId}`);
|
||
}
|
||
|
||
return recoveredAgentIds.size;
|
||
}
|
||
|
||
async recoverDriftedAgentTaskLinks(): Promise<number> {
|
||
const agentStore = this.options.agentStore;
|
||
if (!agentStore) {
|
||
return 0;
|
||
}
|
||
|
||
const now = Date.now();
|
||
const clearedAgentIds = new Set<string>();
|
||
const durableAgents = await agentStore.listAgents({ includeEphemeral: false });
|
||
|
||
for (const agent of durableAgents) {
|
||
if (!agent.taskId) {
|
||
continue;
|
||
}
|
||
|
||
const linkedTaskId = agent.taskId;
|
||
const linkedTask = await this.store.getTask(linkedTaskId);
|
||
let shouldClear = false;
|
||
let reason = "";
|
||
|
||
if (!linkedTask) {
|
||
shouldClear = true;
|
||
reason = "linked task missing";
|
||
} else if (linkedTask.column === "done" || linkedTask.column === "archived") {
|
||
shouldClear = true;
|
||
reason = `linked task in terminal column ${linkedTask.column}`;
|
||
} else if (linkedTask.assignedAgentId && linkedTask.assignedAgentId !== agent.id) {
|
||
shouldClear = true;
|
||
reason = `linked task assigned to ${linkedTask.assignedAgentId}`;
|
||
} else if (linkedTask.column === "todo" || linkedTask.column === "triage") {
|
||
const activeRun = await agentStore.getActiveHeartbeatRun(agent.id);
|
||
const runStartedAt = activeRun?.startedAt;
|
||
const runAgeMs = runStartedAt ? now - Date.parse(runStartedAt) : Number.POSITIVE_INFINITY;
|
||
const hasFreshRun = Boolean(activeRun) && Number.isFinite(runAgeMs) && runAgeMs <= RUNNING_ON_INACTIVE_TASK_STALE_RUN_MS;
|
||
const hasActiveExecution = this.options.hasActiveAgentExecution?.(agent.id) === true;
|
||
if (!hasFreshRun && !hasActiveExecution) {
|
||
shouldClear = true;
|
||
reason = `linked task in queued column ${linkedTask.column} without fresh run`;
|
||
}
|
||
}
|
||
|
||
if (!shouldClear) {
|
||
continue;
|
||
}
|
||
|
||
await agentStore.syncExecutionTaskLink(agent.id, undefined);
|
||
clearedAgentIds.add(agent.id);
|
||
log.log(`Cleared drifted durable agent task link for ${agent.id} (${linkedTaskId}): ${reason}`);
|
||
}
|
||
|
||
log.log(`Recovered ${clearedAgentIds.size} drifted durable agent task link(s)`);
|
||
return clearedAgentIds.size;
|
||
}
|
||
|
||
async recoverOrphanedAgents(): Promise<number> {
|
||
const agentStore = this.options.agentStore;
|
||
if (!agentStore) {
|
||
return 0;
|
||
}
|
||
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
const timeoutMs = settings.taskStuckTimeoutMs;
|
||
if (!Number.isFinite(timeoutMs) || timeoutMs === undefined || timeoutMs <= 0) {
|
||
return 0;
|
||
}
|
||
const recoveryTimeoutMs = timeoutMs;
|
||
|
||
const allAgents = await agentStore.listAgents();
|
||
const allAgentIds = new Set(allAgents.map((agent) => agent.id));
|
||
const now = Date.now();
|
||
|
||
const orphaned = allAgents.filter((agent) => {
|
||
if (isEphemeralAgent(agent)) {
|
||
return false;
|
||
}
|
||
if (agent.state !== "running" && agent.state !== "error") {
|
||
return false;
|
||
}
|
||
const managerMissing = !agent.reportsTo || !allAgentIds.has(agent.reportsTo);
|
||
if (!managerMissing) {
|
||
return false;
|
||
}
|
||
const updatedAt = Date.parse(agent.updatedAt ?? "");
|
||
if (!Number.isFinite(updatedAt) || now - updatedAt < recoveryTimeoutMs) {
|
||
return false;
|
||
}
|
||
|
||
if (agent.state === "error") {
|
||
const runtimeConfig = (agent.runtimeConfig ?? {}) as Record<string, unknown>;
|
||
if (runtimeConfig.enabled === false) {
|
||
return false;
|
||
}
|
||
if (this.options.hasActiveAgentExecution?.(agent.id) === true) {
|
||
return false;
|
||
}
|
||
if (classifyError(agent.lastError ?? "") !== "transient" && !isStaleWorktreeModuleResolutionError(agent.lastError ?? "")) {
|
||
return false;
|
||
}
|
||
if (isOperatorActionableAgentError(agent.lastError ?? "")) {
|
||
return false;
|
||
}
|
||
|
||
const recoveryState = this.getDurableAgentRecoveryState(agent);
|
||
if (recoveryState.exhausted) {
|
||
return false;
|
||
}
|
||
if (recoveryState.nextRetryAt) {
|
||
const nextRetryMs = Date.parse(recoveryState.nextRetryAt);
|
||
if (Number.isFinite(nextRetryMs) && nextRetryMs > now) {
|
||
log.log(`Durable agent ${agent.id} transient recovery delayed until ${recoveryState.nextRetryAt}`);
|
||
return false;
|
||
}
|
||
}
|
||
}
|
||
|
||
return true;
|
||
});
|
||
|
||
if (orphaned.length === 0) {
|
||
return 0;
|
||
}
|
||
|
||
let recovered = 0;
|
||
for (const agent of orphaned) {
|
||
const updatedAt = Date.parse(agent.updatedAt ?? "");
|
||
const stuckForMs = Math.max(0, now - updatedAt);
|
||
try {
|
||
if (agent.state === "error") {
|
||
const recoveryState = this.getDurableAgentRecoveryState(agent);
|
||
const isStaleMissingModule = isStaleWorktreeModuleResolutionError(agent.lastError ?? "");
|
||
if (isStaleMissingModule) {
|
||
const missingModulePath = extractMissingModulePath(agent.lastError ?? "");
|
||
const repeatedPath =
|
||
missingModulePath && recoveryState.lastMissingModulePath === missingModulePath
|
||
? recoveryState.consecutiveMissingModulePathCount + 1
|
||
: 1;
|
||
await agentStore.updateAgent(agent.id, {
|
||
metadata: {
|
||
...(agent.metadata ?? {}),
|
||
durableErrorRecovery: {
|
||
attempts: recoveryState.attempts,
|
||
nextRetryAt: recoveryState.nextRetryAt,
|
||
exhausted: recoveryState.exhausted,
|
||
lastReason: "stale-path-module-resolution",
|
||
lastMissingModulePath: missingModulePath ?? recoveryState.lastMissingModulePath,
|
||
consecutiveMissingModulePathCount: repeatedPath,
|
||
lastObservedAt: new Date().toISOString(),
|
||
},
|
||
},
|
||
});
|
||
log.warn(`Suppressed durable-agent auto-restart for ${agent.id}: stale module-resolution failure indicates stale host process/worktree path`);
|
||
if (missingModulePath && repeatedPath >= 3) {
|
||
log.warn(
|
||
`Durable agent ${agent.id} repeated missing-module path ${repeatedPath} times (${missingModulePath}). Hosting dashboard/engine process is likely stale (for example, zombie process from a deleted worktree); clean up stale process/worktree. FN-4013 tracks systemic prevention.`,
|
||
);
|
||
}
|
||
continue;
|
||
}
|
||
const nextAttempts = recoveryState.attempts + 1;
|
||
const exhausted = nextAttempts >= DURABLE_ERROR_RECOVERY_MAX_RETRIES;
|
||
const nextRetryAt = new Date(Date.now() + this.computeDurableAgentRecoveryCooldownMs(nextAttempts)).toISOString();
|
||
await agentStore.updateAgent(agent.id, {
|
||
metadata: {
|
||
...(agent.metadata ?? {}),
|
||
durableErrorRecovery: {
|
||
attempts: nextAttempts,
|
||
lastAttemptAt: new Date().toISOString(),
|
||
nextRetryAt,
|
||
exhausted,
|
||
lastReason: exhausted ? "retry-budget-exhausted" : "transient-error",
|
||
lastMissingModulePath: undefined,
|
||
consecutiveMissingModulePathCount: 0,
|
||
},
|
||
},
|
||
});
|
||
if (exhausted) {
|
||
log.warn(`Suppressed durable-agent auto-restart for ${agent.id}: retry budget exhausted`);
|
||
continue;
|
||
}
|
||
}
|
||
|
||
await agentStore.updateAgentState(agent.id, "active");
|
||
await agentStore.updateAgent(agent.id, {
|
||
lastError: undefined,
|
||
});
|
||
|
||
if (agent.state === "error" && this.options.restartDurableAgentHeartbeat) {
|
||
const restartOk = await this.options.restartDurableAgentHeartbeat(agent.id, {
|
||
reason: "transient-error",
|
||
attempt: this.getDurableAgentRecoveryState(agent).attempts + 1,
|
||
});
|
||
if (!restartOk) {
|
||
log.warn(`Durable-agent transient recovery heartbeat restart skipped for ${agent.id}`);
|
||
}
|
||
}
|
||
|
||
log.log(
|
||
`Auto-recovered: orphaned agent ${agent.id} stuck in ${agent.state} for ${Math.round(stuckForMs / 1000)}s — reset to active`,
|
||
);
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover orphaned agent ${agent.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} orphaned agent(s) → active`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Orphaned agent recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Default cap (in ms) on how long an active heartbeat run from the current
|
||
* process is allowed to remain open before self-healing will terminate it.
|
||
* Six hours is well past any legitimate heartbeat tick (default 1 h
|
||
* interval, configurable up to a few hours) so reaching this threshold
|
||
* means the run record was never closed — typically a process that died
|
||
* without our watchdog catching it.
|
||
*/
|
||
private static readonly STALE_ACTIVE_RUN_MAX_AGE_MS = 6 * 60 * 60 * 1000;
|
||
|
||
/**
|
||
* Terminate orphaned `agentRuns` rows left in `status = 'active'` by a
|
||
* process that crashed before calling endHeartbeatRun(). These rows
|
||
* silently break heartbeat scheduling: HeartbeatTriggerScheduler.onTimerTick
|
||
* skips every tick that finds an active run, so the agent never gets called
|
||
* again until something cleans up.
|
||
*
|
||
* A run is considered stale when:
|
||
* - `processPid` was recorded and does not match the current `process.pid`
|
||
* (i.e., the writer process is gone — guaranteed orphan), or
|
||
* - `processPid` is missing (legacy data), or
|
||
* - the run has been active for longer than STALE_ACTIVE_RUN_MAX_AGE_MS,
|
||
* even from the current process (defense in depth against a writer that
|
||
* leaks the row without crashing the whole runtime).
|
||
*
|
||
* The matching `processPid` + young run case is left alone — that is a
|
||
* legitimately in-flight heartbeat.
|
||
*/
|
||
async recoverStaleHeartbeatRuns(): Promise<number> {
|
||
const agentStore = this.options.agentStore;
|
||
if (!agentStore) {
|
||
return 0;
|
||
}
|
||
|
||
let activeRuns;
|
||
try {
|
||
activeRuns = await agentStore.listActiveHeartbeatRuns();
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Stale heartbeat run recovery — listing failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
|
||
if (activeRuns.length === 0) {
|
||
return 0;
|
||
}
|
||
|
||
const now = Date.now();
|
||
const currentPid = process.pid;
|
||
const maxAgeMs = SelfHealingManager.STALE_ACTIVE_RUN_MAX_AGE_MS;
|
||
let recovered = 0;
|
||
|
||
for (const run of activeRuns) {
|
||
const startedMs = Date.parse(run.startedAt);
|
||
const ageMs = Number.isFinite(startedMs) ? Math.max(0, now - startedMs) : Infinity;
|
||
const recordedPid = run.processPid;
|
||
|
||
const pidMismatch = typeof recordedPid === "number" && recordedPid !== currentPid;
|
||
const pidMissing = typeof recordedPid !== "number";
|
||
const tooOld = ageMs >= maxAgeMs;
|
||
|
||
if (!pidMismatch && !pidMissing && !tooOld) {
|
||
continue;
|
||
}
|
||
|
||
const reason = pidMismatch
|
||
? `writer pid ${recordedPid} is no longer this process (current pid ${currentPid})`
|
||
: pidMissing
|
||
? `no processPid recorded`
|
||
: `active for ${Math.round(ageMs / 1000)}s (>= ${Math.round(maxAgeMs / 1000)}s threshold)`;
|
||
|
||
try {
|
||
const detail = await agentStore.getRunDetail(run.agentId, run.id);
|
||
if (detail) {
|
||
await agentStore.saveRun({
|
||
...detail,
|
||
endedAt: new Date().toISOString(),
|
||
status: "terminated",
|
||
stderrExcerpt: `Auto-recovered orphaned heartbeat run: ${reason}`,
|
||
});
|
||
}
|
||
await agentStore.endHeartbeatRun(run.id, "terminated");
|
||
log.log(
|
||
`Auto-recovered: orphan heartbeat run ${run.id} for ${run.agentId} (${reason})`,
|
||
);
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover stale heartbeat run ${run.id} for ${run.agentId}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} stale heartbeat run(s)`);
|
||
}
|
||
return recovered;
|
||
}
|
||
|
||
/**
|
||
* Recover `in-progress` tasks that failed only because the agent exited
|
||
* without calling fn_task_done, and where there is no sign of work to preserve.
|
||
*
|
||
* These are safe to requeue automatically when no steps progressed and git
|
||
* has neither worktree changes nor branch commits. Cases with any evidence
|
||
* of work are left alone for manual inspection or the normal orphan recovery
|
||
* path.
|
||
*/
|
||
async recoverNoProgressNoTaskDoneFailures(): Promise<number> {
|
||
try {
|
||
const tasks = await this.store.listTasks({ column: "in-progress", slim: true });
|
||
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
|
||
|
||
const candidates = tasks.filter((task) =>
|
||
task.column === "in-progress" &&
|
||
task.status === "failed" &&
|
||
isNoTaskDoneFailure(task) &&
|
||
!task.paused &&
|
||
!executingIds.has(task.id) &&
|
||
!isTaskWorkComplete(task) &&
|
||
!hasStepProgress(task),
|
||
);
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
log.warn(`Found ${candidates.length} no-progress no-task_done failure(s) in in-progress`);
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
try {
|
||
if (await this.hasRecoverableGitWork(task)) {
|
||
log.log(`${task.id} has recoverable git work — leaving in-progress for inspection`);
|
||
continue;
|
||
}
|
||
|
||
await this.store.updateTask(task.id, {
|
||
status: "stuck-killed",
|
||
worktree: null,
|
||
branch: null,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
"Auto-recovered no-progress no-task_done failure — clean worktree, moved back to todo",
|
||
);
|
||
await this.store.moveTask(task.id, "todo");
|
||
recovered++;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover no-progress no-task_done failure ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} no-progress no-task_done failure(s) → todo`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`No-progress no-task_done recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover failed `in-review` retries that point at an unusable worktree path.
|
||
*
|
||
* This is a narrow guard for session-start failures thrown by
|
||
* `assertValidWorktreeSession()` in `pi.ts`, classified centrally via
|
||
* `MISSING_WORKTREE_SESSION_PREFIXES` /
|
||
* `classifyMissingWorktreeSessionStartFailure()` in
|
||
* `restart-recovery-coordinator.ts`.
|
||
* We clear stale worktree metadata and failure state, keep step progress and
|
||
* retry counters, then requeue to todo for a clean retry.
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
*/
|
||
async recoverMissingWorktreeReviewFailures(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
const candidates = tasks.filter((task) =>
|
||
isRecoverableMissingWorktreeReviewFailureWithProgress(task)
|
||
|| isRecoverableMissingWorktreeReviewFailureNoProgress(task),
|
||
);
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
log.warn(`Found ${candidates.length} in-review task(s) failed by unusable-worktree session start`);
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId,
|
||
phase: "maintenance",
|
||
});
|
||
const result = await autoRecoverWorktreeSessionStartFailure(this.store, task, {
|
||
failure: task.error,
|
||
source: "in-review-sweep",
|
||
auditor,
|
||
});
|
||
if (result.outcome === "requeue-todo") recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover unusable-worktree review failure ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} unusable-worktree review failure(s) → todo`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Unusable-worktree review recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover `in-review` tasks marked as `failed` because the agent exited
|
||
* without calling `fn_task_done` *with partial step progress* (some steps done,
|
||
* some still pending). The work-in-progress is valuable but incomplete —
|
||
* the existing worktree and branch are preserved and the task is moved back
|
||
* to `todo` so the scheduler re-dispatches it for a fresh execution that
|
||
* continues from where the previous attempt left off.
|
||
*
|
||
* Bounded by `MAX_TASK_DONE_RETRIES` (per-task `taskDoneRetryCount`) so a
|
||
* persistently-broken task cannot loop forever; when exhausted the task
|
||
* remains parked in `in-review` for manual intervention. The counter is
|
||
* cleared by the executor on successful completion.
|
||
*
|
||
* Distinct from sibling recoveries:
|
||
* - `recoverMisclassifiedFailures`: all steps done → clear error, leave for review.
|
||
* - `recoverNoProgressNoTaskDoneFailures`: `in-progress` with zero progress → clean requeue.
|
||
* - This one: `in-review` with partial progress → bounded requeue preserving work.
|
||
*
|
||
* No-op when `settings.autoMerge === false` — PR-based review flow owns lifecycle until human merge.
|
||
* @returns Number of tasks requeued for retry
|
||
*/
|
||
async recoverPartialProgressNoTaskDoneFailures(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.globalPause || settings.enginePaused) return 0;
|
||
if (settings.autoMerge === false) return 0;
|
||
|
||
const tasks = await this.store.listTasks({ column: "in-review", slim: true });
|
||
|
||
const candidates = tasks.filter((task) =>
|
||
task.column === "in-review" &&
|
||
task.status === "failed" &&
|
||
isNoTaskDoneFailure(task) &&
|
||
!task.paused &&
|
||
!isTaskWorkComplete(task) &&
|
||
hasStepProgress(task) &&
|
||
(task.taskDoneRetryCount ?? 0) < MAX_TASK_DONE_RETRIES,
|
||
);
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
log.warn(
|
||
`Found ${candidates.length} partial-progress no-task_done failure(s) eligible for auto-retry`,
|
||
);
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
try {
|
||
const nextCount = (task.taskDoneRetryCount ?? 0) + 1;
|
||
await this.store.updateTask(task.id, {
|
||
status: null,
|
||
error: null,
|
||
sessionFile: null,
|
||
taskDoneRetryCount: nextCount,
|
||
});
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-retry ${nextCount}/${MAX_TASK_DONE_RETRIES}: agent finished without fn_task_done — requeuing to todo to resume partial work`,
|
||
);
|
||
await this.store.moveTask(task.id, "todo", { preserveProgress: true });
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(
|
||
`Failed to auto-retry partial-progress no-task_done failure ${task.id}: ${errorMessage}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(
|
||
`Auto-retried ${recovered} partial-progress no-task_done failure(s) → todo`,
|
||
);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Partial-progress no-task_done recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
private async isBranchAheadOfBase(
|
||
task: Task,
|
||
baseRef?: string,
|
||
): Promise<{ aheadCount: number; baseRef: string } | null> {
|
||
return isBranchAheadOfBase(task, this.options.rootDir, baseRef);
|
||
}
|
||
|
||
private async hasRecoverableGitWork(task: Task): Promise<boolean> {
|
||
if (task.worktree && existsSync(task.worktree)) {
|
||
try {
|
||
const { stdout: status } = await execAsync("git status --porcelain", {
|
||
cwd: task.worktree,
|
||
timeout: 30_000,
|
||
});
|
||
if (status.trim().length > 0) return true;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(
|
||
`Failed to inspect worktree status for ${task.id} at ${task.worktree}: ${errorMessage} — preserving worktree`,
|
||
);
|
||
// If we cannot inspect an existing worktree, preserve it.
|
||
return true;
|
||
}
|
||
}
|
||
|
||
const branchName = task.branch || canonicalFusionBranchName(task.id);
|
||
try {
|
||
await execAsync(`git rev-parse --verify "${branchName}"`, {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
} catch {
|
||
// Intentional negative test: rev-parse exits non-zero when branch does not exist.
|
||
return false;
|
||
}
|
||
|
||
try {
|
||
const { stdout: uniqueCommits } = await execAsync(
|
||
`git rev-list --count HEAD.."${branchName}"`,
|
||
{ cwd: this.options.rootDir, timeout: 30_000 },
|
||
);
|
||
return Number.parseInt(uniqueCommits.trim(), 10) > 0;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(
|
||
`Failed to compare branch ${branchName} against HEAD for ${task.id}: ${errorMessage} — preserving branch`,
|
||
);
|
||
// If the branch exists but cannot be compared, preserve it.
|
||
return true;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover triage tasks that already have an approved specification but were
|
||
* left stuck in `status: "planning"` without an active triage session.
|
||
*
|
||
* This catches the mirror-image of executor recovery: the review completed,
|
||
* but the final transition to `todo` / `awaiting-approval` never happened.
|
||
*/
|
||
async recoverApprovedTriageTasks(): Promise<number> {
|
||
const recoverFn = this.options.recoverApprovedTriageTask;
|
||
if (!recoverFn) return 0;
|
||
|
||
try {
|
||
// Evict stale entries from the triage processor's in-memory set before
|
||
// checking — tasks with hung promises (from stuck kills) would otherwise
|
||
// block recovery indefinitely.
|
||
this.options.evictStaleTriageProcessing?.();
|
||
|
||
const tasks = await this.store.listTasks({ column: "triage" });
|
||
const planningIds = this.options.getPlanningTaskIds?.() ?? new Set<string>();
|
||
const now = Date.now();
|
||
|
||
const orphanedApproved = tasks.filter((t) =>
|
||
t.column === "triage" &&
|
||
t.status === "planning" &&
|
||
!t.paused &&
|
||
!planningIds.has(t.id) &&
|
||
now - new Date(t.updatedAt).getTime() >= APPROVED_TRIAGE_RECOVERY_GRACE_MS &&
|
||
hasLatestSpecReviewApproval(t),
|
||
);
|
||
|
||
if (orphanedApproved.length === 0) return 0;
|
||
|
||
log.warn(`Found ${orphanedApproved.length} approved triage task(s) stuck in planning`);
|
||
|
||
let recovered = 0;
|
||
for (const task of orphanedApproved) {
|
||
log.log(`Recovering approved triage task ${task.id}: ${task.title || task.description?.slice(0, 60) || "(untitled)"}`);
|
||
const success = await recoverFn(task);
|
||
if (success) recovered++;
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} approved triage task(s) out of planning`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Approved triage recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover refinement tasks that have sat in triage long enough to indicate
|
||
* starvation while the rest of the board keeps progressing.
|
||
*
|
||
* Recovery is a bounded priority nudge only; tasks still route through the
|
||
* normal triage specification + approval pipeline.
|
||
*/
|
||
async recoverStarvedRefinementTriageTasks(): Promise<number> {
|
||
try {
|
||
this.options.evictStaleTriageProcessing?.();
|
||
|
||
const tasks = await this.store.listTasks({ slim: true, includeArchived: false });
|
||
const planningIds = this.options.getPlanningTaskIds?.() ?? new Set<string>();
|
||
const now = Date.now();
|
||
|
||
const candidates = tasks.filter((task) => {
|
||
if (task.column !== "triage") return false;
|
||
if (task.sourceType !== "task_refine") return false;
|
||
if (task.paused) return false;
|
||
if (task.status !== null && task.status !== "planning") return false;
|
||
if (planningIds.has(task.id)) return false;
|
||
|
||
const createdAtMs = new Date(task.createdAt).getTime();
|
||
const updatedAtMs = new Date(task.updatedAt).getTime();
|
||
if (!Number.isFinite(createdAtMs) || !Number.isFinite(updatedAtMs)) return false;
|
||
if (now - createdAtMs < STARVED_REFINEMENT_RECOVERY_GRACE_MS) return false;
|
||
if (now - updatedAtMs < STARVED_REFINEMENT_ESCALATION_COOLDOWN_MS) return false;
|
||
|
||
const peerProgressCount = tasks.filter((peer) =>
|
||
peer.id !== task.id &&
|
||
peer.column === "todo" &&
|
||
peer.sourceType !== "task_refine" &&
|
||
new Date(peer.updatedAt).getTime() > createdAtMs,
|
||
).length;
|
||
|
||
return peerProgressCount >= STARVED_PEER_PROGRESS_THRESHOLD;
|
||
});
|
||
|
||
if (candidates.length === 0) return 0;
|
||
|
||
log.warn(`Found ${candidates.length} starved refinement triage task(s)`);
|
||
|
||
let recovered = 0;
|
||
for (const task of candidates) {
|
||
try {
|
||
const nextPriority = bumpTaskPriority(task.priority);
|
||
if (nextPriority === task.priority) continue;
|
||
|
||
const createdAtMs = new Date(task.createdAt).getTime();
|
||
const peerProgressCount = tasks.filter((peer) =>
|
||
peer.id !== task.id &&
|
||
peer.column === "todo" &&
|
||
peer.sourceType !== "task_refine" &&
|
||
new Date(peer.updatedAt).getTime() > createdAtMs,
|
||
).length;
|
||
|
||
await this.store.updateTask(task.id, { priority: nextPriority });
|
||
await this.store.logEntry(
|
||
task.id,
|
||
`Auto-recovered starved refinement triage task: priority ${task.priority ?? "normal"} -> ${nextPriority} (age=${Math.max(0, now - createdAtMs)}ms, peerProgress=${peerProgressCount})`,
|
||
);
|
||
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", task.id),
|
||
agentId: "self-healing",
|
||
taskId: task.id,
|
||
taskLineageId: task.lineageId ?? undefined,
|
||
phase: "triage-recovery",
|
||
});
|
||
await auditor.database({
|
||
type: "task:auto-recover-starved-refinement",
|
||
target: task.id,
|
||
metadata: {
|
||
taskId: task.id,
|
||
ageMs: Math.max(0, now - createdAtMs),
|
||
peerProgressCount,
|
||
escalation: "priority-bump",
|
||
previousPriority: task.priority ?? "normal",
|
||
nextPriority,
|
||
graceMs: STARVED_REFINEMENT_RECOVERY_GRACE_MS,
|
||
cooldownMs: STARVED_REFINEMENT_ESCALATION_COOLDOWN_MS,
|
||
peerThreshold: STARVED_PEER_PROGRESS_THRESHOLD,
|
||
},
|
||
});
|
||
} catch (auditErr: unknown) {
|
||
const auditErrMessage = auditErr instanceof Error ? auditErr.message : String(auditErr);
|
||
log.warn(`Failed to record starved refinement recovery audit for ${task.id}: ${auditErrMessage}`);
|
||
}
|
||
|
||
recovered++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover starved refinement task ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} starved refinement triage task(s)`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Starved refinement triage recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Recover triage tasks stuck in `status: "planning"` whose agent session
|
||
* died before producing an approved spec.
|
||
*
|
||
* These tasks fall through two cracks:
|
||
* - The stuck task detector only monitors tasks with active tracked sessions.
|
||
* If the session crashed or was never started, the task is never tracked.
|
||
* - `recoverApprovedTriageTasks` only handles tasks with an approved spec.
|
||
*
|
||
* Recovery clears the status back to `null` so the next triage poll picks
|
||
* them up for a fresh planning attempt.
|
||
*/
|
||
async recoverOrphanedPlanningTasks(): Promise<number> {
|
||
try {
|
||
// Evict stale entries from the triage processor's in-memory set before
|
||
// checking — tasks with hung promises (from stuck kills) would otherwise
|
||
// block recovery indefinitely.
|
||
this.options.evictStaleTriageProcessing?.();
|
||
|
||
const tasks = await this.store.listTasks({ column: "triage" });
|
||
const planningIds = this.options.getPlanningTaskIds?.() ?? new Set<string>();
|
||
const now = Date.now();
|
||
|
||
const orphaned = tasks.filter((t) =>
|
||
t.column === "triage" &&
|
||
t.status === "planning" &&
|
||
!t.paused &&
|
||
!planningIds.has(t.id) &&
|
||
now - new Date(t.updatedAt).getTime() >= APPROVED_TRIAGE_RECOVERY_GRACE_MS &&
|
||
!hasLatestSpecReviewApproval(t),
|
||
);
|
||
|
||
if (orphaned.length === 0) return 0;
|
||
|
||
log.warn(`Found ${orphaned.length} orphaned planning triage task(s) without approval`);
|
||
|
||
let recovered = 0;
|
||
for (const task of orphaned) {
|
||
try {
|
||
log.log(`Recovering orphaned planning task ${task.id}: ${task.title || task.description?.slice(0, 60) || "(untitled)"}`);
|
||
await this.store.updateTask(task.id, { status: null });
|
||
await this.store.logEntry(
|
||
task.id,
|
||
"Auto-recovered orphaned planning task — agent session lost, cleared for re-planning",
|
||
);
|
||
recovered++;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Failed to recover orphaned planning task ${task.id}: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
if (recovered > 0) {
|
||
log.log(`Recovered ${recovered} orphaned planning task(s) — cleared for re-planning`);
|
||
}
|
||
return recovered;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Orphaned planning task recovery failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/** Run `git worktree prune` to clean stale metadata. */
|
||
private async pruneWorktrees(): Promise<void> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
const worktrunkEnabled = settings.worktrunk?.enabled === true;
|
||
if (worktrunkEnabled) {
|
||
const backend = resolveWorktreeBackend(settings, { logger: log });
|
||
if (backend.kind === "worktrunk") {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", "worktrunk-prune"),
|
||
agentId: "self-healing",
|
||
phase: "maintenance-prune",
|
||
});
|
||
|
||
try {
|
||
await backend.prune({ rootDir: this.options.rootDir });
|
||
await auditor.git({ type: "worktree:worktrunk-prune", target: this.options.rootDir, metadata: { success: true } });
|
||
log.log("Worktree prune delegated to worktrunk backend");
|
||
return;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
await auditor.git({ type: "worktree:worktrunk-prune", target: this.options.rootDir, metadata: { success: false, error: errorMessage } });
|
||
if (settings.worktrunk?.onFailure === "fail") {
|
||
log.error(`Worktrunk prune failed (fail-hard): ${errorMessage}`);
|
||
return;
|
||
}
|
||
log.warn(`Worktrunk prune failed; falling back to native git prune: ${errorMessage}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
await execAsync("git worktree prune", {
|
||
cwd: this.options.rootDir,
|
||
timeout: 30_000,
|
||
});
|
||
log.log("Worktree prune completed");
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Worktree prune failed: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Remove orphaned worktrees not assigned to any active task.
|
||
*
|
||
* When `recycleWorktrees` is OFF: removes registered idle worktrees too —
|
||
* they would otherwise pile up since the pool isn't keeping them.
|
||
*
|
||
* When `recycleWorktrees` is ON: leaves registered idle worktrees alone
|
||
* (the pool wants them for reuse) but still reaps unregistered stale dirs
|
||
* left behind by killed runs (e.g., `clear-hawk-broken`, `*-bak`). Those
|
||
* dirs can never be recycled — they aren't git worktrees — so they only
|
||
* waste disk.
|
||
*/
|
||
private async cleanupOrphans(): Promise<number> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.worktrunk?.enabled === true) {
|
||
log.log("[self-healing] skipped native orphan cleanup — worktrunk backend owns layout");
|
||
const backend = resolveWorktreeBackend(settings, { logger: log });
|
||
if (backend.kind === "worktrunk") {
|
||
await backend.prune({ rootDir: this.options.rootDir });
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
if (settings.recycleWorktrees) {
|
||
// Recycle on: only sweep unregistered stale dirs.
|
||
return await this.reapUnregisteredOrphans();
|
||
}
|
||
|
||
const orphaned = await scanIdleWorktrees(this.options.rootDir, this.store, settings);
|
||
if (orphaned.length === 0) return 0;
|
||
|
||
let cleaned = 0;
|
||
for (const worktreePath of orphaned) {
|
||
try {
|
||
await removeWorktree({
|
||
rootDir: this.options.rootDir,
|
||
worktreePath,
|
||
settings,
|
||
reason: RemovalReason.SelfHealingIdleSweep,
|
||
});
|
||
cleaned++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`Failed to remove orphaned worktree ${worktreePath}: ${errorMessage} — non-fatal`);
|
||
// Individual failure is non-fatal
|
||
}
|
||
}
|
||
|
||
if (cleaned > 0) {
|
||
log.log(`Cleaned ${cleaned} orphaned worktree(s)`);
|
||
}
|
||
return cleaned;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Orphan cleanup failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Sweep unregistered stale directories under `<rootDir>/.worktrees/` —
|
||
* directories that exist on disk but are NOT registered git worktrees.
|
||
* Safe to run alongside `recycleWorktrees: true` because the pool only
|
||
* tracks registered idle worktrees, never these orphans.
|
||
*/
|
||
private async reapUnregisteredOrphans(): Promise<number> {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.worktrunk?.enabled === true) {
|
||
log.log("[self-healing] skipped native unregistered-orphan reap — worktrunk backend owns layout");
|
||
const backend = resolveWorktreeBackend(settings, { logger: log });
|
||
if (backend.kind === "worktrunk") {
|
||
await backend.prune({ rootDir: this.options.rootDir });
|
||
}
|
||
return 0;
|
||
}
|
||
const worktreesDir = resolveWorktreesDir(this.options.rootDir, settings);
|
||
if (!existsSync(worktreesDir)) return 0;
|
||
|
||
let dirs: string[];
|
||
try {
|
||
dirs = readdirSync(worktreesDir, { withFileTypes: true })
|
||
.filter((e) => e.isDirectory())
|
||
.map((e) => join(worktreesDir, e.name));
|
||
} catch (err: unknown) {
|
||
log.warn(`Failed to read .worktrees/ for unregistered orphan reap: ${err instanceof Error ? err.message : String(err)}`);
|
||
return 0;
|
||
}
|
||
if (dirs.length === 0) return 0;
|
||
|
||
const registered = await getRegisteredWorktreePaths(this.options.rootDir);
|
||
const unregistered = dirs.filter((d) => !registered.has(resolve(d)));
|
||
|
||
let cleaned = 0;
|
||
for (const path of unregistered) {
|
||
const rel = relative(worktreesDir, path);
|
||
if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) {
|
||
log.warn(`Refusing to remove path outside .worktrees: ${path}`);
|
||
continue;
|
||
}
|
||
// FN-4811 (restored by FN-5065): never rmSync a directory bound to a live
|
||
// executor/merger/step session. The worktree may be unregistered in git's
|
||
// admin file while the owning process is still running.
|
||
if (activeSessionRegistry.isPathActive(path)) {
|
||
log.log(`[self-healing] deferring unregistered-orphan reap for ${path}: active session present`);
|
||
continue;
|
||
}
|
||
try {
|
||
rmSync(path, { recursive: true, force: true });
|
||
log.log(`Cleaned unregistered worktree dir: ${path}`);
|
||
cleaned++;
|
||
} catch (err: unknown) {
|
||
log.warn(`Failed to remove unregistered worktree dir ${path}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
}
|
||
|
||
if (cleaned > 0) {
|
||
log.log(`Cleaned ${cleaned} unregistered worktree dir(s) (recycle mode preserves registered idle worktrees)`);
|
||
}
|
||
return cleaned;
|
||
}
|
||
|
||
private deriveTaskIdFromFusionBranch(branch: string): string | null {
|
||
const match = /^fusion\/(fn|kb)-(\d+)$/i.exec(branch.trim());
|
||
if (!match) return null;
|
||
return `${match[1].toUpperCase()}-${match[2]}`;
|
||
}
|
||
|
||
private async getRegisteredWorktreePathForBranch(branch: string): Promise<string | null> {
|
||
try {
|
||
const stdout = execSync("git worktree list --porcelain", {
|
||
cwd: this.options.rootDir,
|
||
encoding: "utf-8",
|
||
stdio: ["pipe", "pipe", "pipe"],
|
||
}) || "";
|
||
const lines = stdout.split("\n");
|
||
let currentPath: string | null = null;
|
||
for (const line of lines) {
|
||
if (line.startsWith("worktree ")) {
|
||
currentPath = line.slice("worktree ".length).trim();
|
||
continue;
|
||
}
|
||
if (line.startsWith("branch ")) {
|
||
const fullRef = line.slice("branch ".length).trim();
|
||
const branchName = fullRef.startsWith("refs/heads/") ? fullRef.slice("refs/heads/".length) : fullRef;
|
||
if (branchName === branch && currentPath) {
|
||
return currentPath;
|
||
}
|
||
}
|
||
}
|
||
} catch (err: unknown) {
|
||
log.warn(`Failed to inspect registered worktree for ${branch}: ${err instanceof Error ? err.message : String(err)}`);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
private async inspectOrphanedBranch(branch: string): Promise<OrphanBranchInspection | null> {
|
||
try {
|
||
const tipSha = String(execSync(`git rev-parse --verify ${shellQuote(branch)}`, {
|
||
cwd: this.options.rootDir,
|
||
encoding: "utf-8",
|
||
stdio: ["pipe", "pipe", "pipe"],
|
||
})).trim();
|
||
if (!tipSha) return null;
|
||
|
||
const uniqueCommitCount = Number.parseInt(String(execSync(`git rev-list --count ${shellQuote(branch)} --not ${shellQuote("main")}`, {
|
||
cwd: this.options.rootDir,
|
||
encoding: "utf-8",
|
||
stdio: ["pipe", "pipe", "pipe"],
|
||
})).trim(), 10) || 0;
|
||
|
||
let uniqueCommitSubjects: string[] = [];
|
||
if (uniqueCommitCount > 0) {
|
||
const subjectOutput = String(execSync(`git log --format=%s --max-count=${ORPHAN_RESCUE_SUBJECT_CAP} ${shellQuote(branch)} --not ${shellQuote("main")}`, {
|
||
cwd: this.options.rootDir,
|
||
encoding: "utf-8",
|
||
stdio: ["pipe", "pipe", "pipe"],
|
||
}));
|
||
uniqueCommitSubjects = subjectOutput.split("\n").map((line) => line.trim()).filter(Boolean);
|
||
}
|
||
|
||
return {
|
||
branch,
|
||
tipSha,
|
||
uniqueCommitCount,
|
||
uniqueCommitSubjects,
|
||
derivedTaskId: this.deriveTaskIdFromFusionBranch(branch),
|
||
registeredWorktreePath: await this.getRegisteredWorktreePathForBranch(branch),
|
||
};
|
||
} catch (err: unknown) {
|
||
log.warn(`Failed to inspect orphaned branch ${branch}: ${err instanceof Error ? err.message : String(err)}`);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Resolve orphaned `fusion/*` branches.
|
||
* - Subsumed branches are pruned (`git branch -d`).
|
||
* - Unique-commit branches with missing task rows are rescued into triage tasks.
|
||
* - Archived matching tasks are left untouched with one-time acknowledgement logging.
|
||
*/
|
||
async cleanupOrphanedBranches(): Promise<number> {
|
||
try {
|
||
const orphaned = await scanOrphanedBranches(this.options.rootDir, this.store);
|
||
if (orphaned.length === 0) return 0;
|
||
|
||
let cleaned = 0;
|
||
const prunedBranches: string[] = [];
|
||
const allTasks = await this.store.listTasks({ slim: true, includeArchived: true });
|
||
const taskById = new Map(allTasks.map((task) => [task.id.toUpperCase(), task]));
|
||
|
||
for (const branch of orphaned) {
|
||
const inspection = await this.inspectOrphanedBranch(branch);
|
||
if (!inspection) continue;
|
||
|
||
if (inspection.uniqueCommitCount <= 0) {
|
||
try {
|
||
execSync(`git branch -d ${shellQuote(branch)}`, {
|
||
cwd: this.options.rootDir,
|
||
stdio: ["pipe", "pipe", "pipe"],
|
||
});
|
||
prunedBranches.push(branch);
|
||
cleaned++;
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", "orphan-branch"),
|
||
agentId: "self-healing",
|
||
phase: "orphan-branch-rescue",
|
||
});
|
||
await auditor.git({
|
||
type: "branch:orphan-prune",
|
||
target: branch,
|
||
metadata: {
|
||
phase: "orphan-branch-rescue",
|
||
tipSha: inspection.tipSha,
|
||
uniqueCommitCount: inspection.uniqueCommitCount,
|
||
},
|
||
});
|
||
} catch (auditErr: unknown) {
|
||
log.warn(`Failed to write branch:orphan-prune run-audit event for ${branch}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
|
||
}
|
||
} catch (err: unknown) {
|
||
log.warn(`Failed to prune subsumed orphaned branch ${branch}: ${err instanceof Error ? err.message : String(err)} — non-fatal`);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
const derivedTaskId = inspection.derivedTaskId;
|
||
const matchedTask = derivedTaskId ? taskById.get(derivedTaskId.toUpperCase()) : undefined;
|
||
const existingBranchTask = allTasks.find((task) => task.branch === branch);
|
||
|
||
if (matchedTask?.column === "archived") {
|
||
if (!this.orphanArchivedAcknowledged.has(matchedTask.id)) {
|
||
this.orphanArchivedAcknowledged.add(matchedTask.id);
|
||
log.warn(`[recovery] orphan-rescue-archived-skip ${matchedTask.id} branch=${branch} tip=${inspection.tipSha.slice(0, 12)} unique=${inspection.uniqueCommitCount}`);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (!matchedTask && !existingBranchTask) {
|
||
const summaryLines = [
|
||
`Recovered orphaned branch: ${branch}`,
|
||
`Tip: ${inspection.tipSha}`,
|
||
`Unique commits vs main: ${inspection.uniqueCommitCount}`,
|
||
];
|
||
if (inspection.uniqueCommitSubjects.length > 0) {
|
||
summaryLines.push("Recent commit subjects:");
|
||
for (const subject of inspection.uniqueCommitSubjects) {
|
||
summaryLines.push(`- ${subject}`);
|
||
}
|
||
}
|
||
const rescueTask = await this.store.createTask({
|
||
title: `Recover orphaned branch ${branch}`,
|
||
description: summaryLines.join("\n"),
|
||
branch,
|
||
column: "triage",
|
||
});
|
||
allTasks.push({ ...rescueTask, branch, column: "triage" } as Task);
|
||
if (inspection.registeredWorktreePath) {
|
||
await this.store.updateTask(rescueTask.id, { worktree: inspection.registeredWorktreePath });
|
||
}
|
||
await this.store.logEntry(rescueTask.id, `[recovery] orphan-rescue-created ${rescueTask.id} from ${branch} (${inspection.uniqueCommitCount} unique commits)`);
|
||
|
||
try {
|
||
const auditor = createRunAuditor(this.store, {
|
||
runId: generateSyntheticRunId("self-heal", rescueTask.id),
|
||
agentId: "self-healing",
|
||
taskId: rescueTask.id,
|
||
taskLineageId: rescueTask.lineageId,
|
||
phase: "orphan-branch-rescue",
|
||
});
|
||
await auditor.git({
|
||
type: "branch:orphan-rescued",
|
||
target: branch,
|
||
metadata: {
|
||
phase: "orphan-branch-rescue",
|
||
rescueTaskId: rescueTask.id,
|
||
tipSha: inspection.tipSha,
|
||
uniqueCommitCount: inspection.uniqueCommitCount,
|
||
derivedTaskId: derivedTaskId ?? null,
|
||
},
|
||
});
|
||
} catch (auditErr: unknown) {
|
||
log.warn(`Failed to write branch:orphan-rescued run-audit event for ${branch}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (prunedBranches.length > 0) {
|
||
const cleared = this.store.clearStaleExecutionStartBranchReferences(prunedBranches);
|
||
if (cleared.length > 0) {
|
||
log.log(`Cleared stale baseBranch on ${cleared.length} task(s): ${cleared.join(", ")}`);
|
||
}
|
||
}
|
||
|
||
return cleaned;
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Orphaned branch cleanup failed: ${errorMessage}`);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/** Run a best-effort passive WAL checkpoint without forcing live writers to truncate. */
|
||
private checkpointWal(): void {
|
||
try {
|
||
const result = this.store.walCheckpoint("PASSIVE");
|
||
if (result.log > 0) {
|
||
log.log(`WAL checkpoint (passive): ${result.checkpointed}/${result.log} pages checkpointed` +
|
||
(result.busy > 0 ? ` (${result.busy} busy)` : ""));
|
||
}
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`WAL checkpoint failed: ${errorMessage}`);
|
||
}
|
||
}
|
||
|
||
/** Remove oldest idle worktrees if total count exceeds 2× maxWorktrees. */
|
||
private async enforceWorktreeCap(): Promise<void> {
|
||
try {
|
||
const settings = await this.store.getSettings();
|
||
if (settings.worktrunk?.enabled === true) {
|
||
log.log("[self-healing] skipped native worktree cap enforcement — worktrunk backend owns layout");
|
||
const backend = resolveWorktreeBackend(settings, { logger: log });
|
||
if (backend.kind === "worktrunk") {
|
||
await backend.prune({ rootDir: this.options.rootDir });
|
||
}
|
||
return;
|
||
}
|
||
const worktreesDir = resolveWorktreesDir(this.options.rootDir, settings);
|
||
if (!existsSync(worktreesDir)) return;
|
||
const cap = (settings.maxWorktrees ?? 4) * 2;
|
||
|
||
const entries = readdirSync(worktreesDir, { withFileTypes: true });
|
||
const dirs = entries.filter((e) => e.isDirectory());
|
||
|
||
if (dirs.length <= cap) return;
|
||
|
||
// Find idle worktrees that can be safely removed
|
||
const idle = await scanIdleWorktrees(this.options.rootDir, this.store, settings);
|
||
if (idle.length === 0) return;
|
||
|
||
// Sort by mtime ascending (oldest first)
|
||
const withMtime = idle.map((p) => {
|
||
try {
|
||
return { path: p, mtime: statSync(p).mtimeMs };
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`Failed to read mtime for worktree ${p}: ${errorMessage} — defaulting mtime to 0`);
|
||
return { path: p, mtime: 0 };
|
||
}
|
||
});
|
||
withMtime.sort((a, b) => a.mtime - b.mtime);
|
||
|
||
let removed = 0;
|
||
const excess = dirs.length - cap;
|
||
|
||
for (const { path: worktreePath } of withMtime) {
|
||
if (removed >= excess) break;
|
||
try {
|
||
await removeWorktree({
|
||
rootDir: this.options.rootDir,
|
||
worktreePath,
|
||
settings,
|
||
reason: RemovalReason.SelfHealingIdleSweep,
|
||
});
|
||
removed++;
|
||
} catch (err: unknown) {
|
||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.warn(`Failed to remove idle worktree ${worktreePath} during cap enforcement: ${errorMessage} — non-fatal`);
|
||
// Individual failure is non-fatal
|
||
}
|
||
}
|
||
|
||
if (removed > 0) {
|
||
log.warn(`Worktree cap: removed ${removed} idle worktree(s) (was ${dirs.length}, cap ${cap})`);
|
||
}
|
||
} catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err);
|
||
log.error(`Worktree cap enforcement failed: ${errorMessage}`);
|
||
}
|
||
}
|
||
}
|
||
|
||
function hasLatestSpecReviewApproval(task: Task): boolean {
|
||
for (let i = task.log.length - 1; i >= 0; i--) {
|
||
const action = task.log[i]?.action ?? "";
|
||
if (action.startsWith("Spec review: ")) {
|
||
return action === "Spec review: APPROVE";
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
function isTaskWorkComplete(task: Task): boolean {
|
||
if (task.steps.length === 0) return false;
|
||
return task.steps.every((step) => step.status === "done" || step.status === "skipped");
|
||
}
|
||
|
||
function isNoTaskDoneFailure(task: Task): boolean {
|
||
const error = task.error?.toLowerCase() ?? "";
|
||
return error.includes("without calling fn_task_done") || error.includes("without calling task_done");
|
||
}
|
||
|
||
function hasStepProgress(task: Task): boolean {
|
||
return task.steps.some((step) => step.status !== "pending");
|
||
}
|