FN-5704: escalate self-owned execution limbo instead of endless resume
Escalate reclaimable in-progress resume limbo into an explicit failure path to stop infinite self-healing loops. - add resume-limbo escalation handling in self-healing/executor flow so self-owned stuck execution is failed and surfaced - extend core task/run-audit types and retry-reset/store behavior to persist and expose the new escalation state - add reliability interaction coverage for reclaim self-owned resume limbo escalation and update related schema/store/CLI/plugin tests and docs Files changed: AGENTS.md | 1 + docs/architecture.md | 2 + packages/cli/src/commands/__tests__/task.test.ts | 2 + packages/core/src/__tests__/db-migrate.test.ts | 12 +- packages/core/src/__tests__/db.test.ts | 34 ++--- packages/core/src/__tests__/goals-schema.test.ts | 2 +- packages/core/src/__tests__/insight-store.test.ts | 10 +- packages/core/src/__tests__/mission-store.test.ts | 2 +- packages/core/src/__tests__/run-audit.test.ts | 2 +- packages/core/src/__tests__/secrets-schema.test.ts | 6 +- .../core/src/__tests__/store-merge-queue.test.ts | 2 +- packages/core/src/__tests__/task-documents.test.ts | 2 +- packages/core/src/db.ts | 13 +- packages/core/src/manual-retry-reset.ts | 1 + packages/core/src/store.ts | 37 ++++- packages/core/src/types.ts | 11 ++ ...laim-self-owned-resume-limbo-escalation.test.ts | 168 +++++++++++++++++++++ packages/engine/src/executor.ts | 5 + packages/engine/src/run-audit.ts | 1 + packages/engine/src/self-healing.ts | 72 +++++++++ .../src/store/__tests__/roadmap-store.test.ts | 4 +- 21 files changed, 345 insertions(+), 44 deletions(-) Fusion-Task-Id: FN-5704 Fusion-Task-Lineage: c8f73d16-d95b-450f-a514-e7d3c2f7aebe
This commit is contained in:
gsxdsm
@@ -72,6 +72,7 @@ const DB_CORRUPTION_NOTIFICATION_COOLDOWN_MS = 60 * 60 * 1000;
|
||||
export const STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS = 10 * 60_000;
|
||||
export const COMPLETION_HANDOFF_LIMBO_GRACE_MS = 5 * 60_000;
|
||||
export const MAX_COMPLETION_HANDOFF_LIMBO_RECOVERIES = 3;
|
||||
const MAX_NO_PROGRESS_RESUME_ATTEMPTS = 2;
|
||||
|
||||
// listTasks already enforces ACTIVE_TASKS_WHERE (`"deletedAt" IS NULL`), but
|
||||
// deadlock/stall sweeps still defensively skip soft-deleted rows in case a
|
||||
@@ -106,6 +107,13 @@ async function classifyOwnedLandedEvidenceForSelfHealing(rootDir: string, task:
|
||||
return classifyOwnedLandedEvidence(rootDir, task, { mergeTargetBranch });
|
||||
}
|
||||
|
||||
function buildResumeLimboStepSignature(task: Task): string {
|
||||
return JSON.stringify({
|
||||
currentStep: task.currentStep ?? null,
|
||||
steps: Array.isArray(task.steps) ? task.steps.map((step) => step.status) : [],
|
||||
});
|
||||
}
|
||||
|
||||
function formatRecoveryTimestamp(date = new Date()): string {
|
||||
const pad = (value: number) => String(value).padStart(2, "0");
|
||||
return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}-${pad(date.getUTCHours())}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}`;
|
||||
@@ -2328,6 +2336,67 @@ export class SelfHealingManager {
|
||||
const preservedCommitCount = inspection.kind === "fully-subsumed"
|
||||
? 0
|
||||
: inspection.taskAttributedCommitCount;
|
||||
const stepSignature = buildResumeLimboStepSignature(task);
|
||||
const hasActiveSessionSignal = Boolean(task.checkedOutBy) || activeTaskIds.has(task.id.toUpperCase());
|
||||
const hasPriorSnapshot = typeof task.resumeLimboTipSha === "string" && typeof task.resumeLimboStepSignature === "string";
|
||||
const unchangedSincePriorResume = hasPriorSnapshot
|
||||
&& task.resumeLimboTipSha === inspection.tipSha
|
||||
&& task.resumeLimboStepSignature === stepSignature;
|
||||
const isNoProgressResume = task.column === "in-progress"
|
||||
&& unchangedSincePriorResume
|
||||
&& !hasActiveSessionSignal;
|
||||
const resumeAttemptCount = isNoProgressResume ? (task.resumeLimboCount ?? 0) + 1 : 0;
|
||||
|
||||
if (task.column === "in-progress" && isNoProgressResume && resumeAttemptCount >= MAX_NO_PROGRESS_RESUME_ATTEMPTS) {
|
||||
const idleAnchor = task.executionStartedAt ?? task.columnMovedAt ?? task.updatedAt;
|
||||
const idleAnchorMs = Date.parse(idleAnchor ?? "");
|
||||
const idleMs = Number.isFinite(idleAnchorMs) ? Math.max(0, Date.now() - idleAnchorMs) : null;
|
||||
await this.store.moveTask(task.id, "todo", {
|
||||
moveSource: "engine",
|
||||
preserveWorktree: true,
|
||||
preserveProgress: true,
|
||||
preserveResumeState: true,
|
||||
});
|
||||
await this.store.updateTask(task.id, {
|
||||
resumeLimboCount: 0,
|
||||
resumeLimboTipSha: inspection.tipSha,
|
||||
resumeLimboStepSignature: stepSignature,
|
||||
});
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`[recovery] resume-limbo-escalated ${task.id} moved to todo after ${resumeAttemptCount} no-progress reclaim/resume attempts`,
|
||||
JSON.stringify({
|
||||
frozenTipSha: inspection.tipSha,
|
||||
idleMs,
|
||||
resumeAttemptCount,
|
||||
currentStep: task.currentStep ?? null,
|
||||
}),
|
||||
);
|
||||
try {
|
||||
await createRunAuditor(this.store, {
|
||||
runId: generateSyntheticRunId("self-heal", task.id),
|
||||
agentId: "self-healing",
|
||||
taskId: task.id,
|
||||
taskLineageId: task.lineageId,
|
||||
phase: "reclaim-self-owned-branch-conflicts",
|
||||
}).database({
|
||||
type: "task:resume-limbo-escalated",
|
||||
target: task.id,
|
||||
metadata: {
|
||||
taskId: task.id,
|
||||
frozenTipSha: inspection.tipSha,
|
||||
idleMs,
|
||||
resumeAttemptCount,
|
||||
currentStep: task.currentStep ?? null,
|
||||
},
|
||||
});
|
||||
} catch (auditErr: unknown) {
|
||||
log.warn(`Failed to write task:resume-limbo-escalated run-audit event for ${task.id}: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`);
|
||||
}
|
||||
recovered++;
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.store.updateTask(task.id, {
|
||||
worktree: inspection.livePath,
|
||||
branch: task.branch,
|
||||
@@ -2335,6 +2404,9 @@ export class SelfHealingManager {
|
||||
pausedReason: undefined,
|
||||
status: null,
|
||||
error: null,
|
||||
resumeLimboCount: resumeAttemptCount,
|
||||
resumeLimboTipSha: inspection.tipSha,
|
||||
resumeLimboStepSignature: stepSignature,
|
||||
});
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
|
||||
Reference in New Issue
Block a user