From 96ad20c93d7ae8cfd18193ffab6e6e4bbfcc3577 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Fri, 21 Aug 2026 08:22:58 -0700 Subject: [PATCH] FN-9187: Bound auto-archive failure retries Prevent stale done-task retention from retrying permanently blocked archives forever. - Pre-filter tasks with live lineage children while honoring renamed archive lanes. - Bound repeated archive failures by reason and reset budgets when candidates or failure classes change. - Emit one bounded audit event and operator-facing task log when retries are exhausted. - Cover retry, escalation, reset, lineage, and renamed-lane behavior and document the retention contract. Files changed: .changeset/fn-9187-bounded-auto-archive.md | 7 ++ AGENTS.md | 1 + docs/architecture.md | 2 + docs/run-audit.md | 1 + docs/settings-reference.md | 2 +- .../live-lineage-children-lanes.pg.test.ts | 3 +- .../core/src/task-store/archive-lifecycle-2.ts | 11 +- packages/engine/src/__tests__/self-healing.test.ts | 117 +++++++++++++++++++++ .../engine/src/run-audit/run-audit-catalogue.ts | 3 + packages/engine/src/self-healing.ts | 93 +++++++++++++++- packages/engine/src/util/run-audit.ts | 2 + 11 files changed, 231 insertions(+), 11 deletions(-) Fusion-Task-Id: FN-9187 Fusion-Task-Lineage: cc9a6adc-4b6d-44da-a84b-5633d4b80a10 Co-authored-by: Fusion (runfusion.ai) --- .changeset/fn-9187-bounded-auto-archive.md | 7 ++ AGENTS.md | 1 + docs/architecture.md | 2 + docs/run-audit.md | 1 + docs/settings-reference.md | 2 +- .../live-lineage-children-lanes.pg.test.ts | 3 +- .../src/task-store/archive-lifecycle-2.ts | 11 +- .../engine/src/__tests__/self-healing.test.ts | 117 ++++++++++++++++++ .../src/run-audit/run-audit-catalogue.ts | 3 + packages/engine/src/self-healing.ts | 93 +++++++++++++- packages/engine/src/util/run-audit.ts | 2 + 11 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 .changeset/fn-9187-bounded-auto-archive.md diff --git a/.changeset/fn-9187-bounded-auto-archive.md b/.changeset/fn-9187-bounded-auto-archive.md new file mode 100644 index 0000000000..e6e40301ae --- /dev/null +++ b/.changeset/fn-9187-bounded-auto-archive.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Stop retrying impossible auto-archives forever and surface abandoned archives on the task. +category: fix +dev: archiveStaleDoneTasks pre-filters live lineage parents and uses MAX_STARVATION_DROPS with task:auto-archive-failure-budget-exhausted. diff --git a/AGENTS.md b/AGENTS.md index 7b18fce4ab..86204fd9e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -323,6 +323,7 @@ canonical emitters remain explicit exclusions until their separately scoped hard - FN-7069: task-store open and self-healing housekeeping emit `task:reconcile-phantom-committed-reservation` when they prune orphaned child rows for a committed task-ID reservation that has no live/soft-deleted/archived task row and no task directory, while preserving the committed reservation so the ID is never reused. - FN-7074: task creation emits `task:reservation-commit-rolled-back` when a distributed reservation was committed atomically with a `tasks` row but a later create materialization step failed; metadata includes `reservationId`, `nodeId`, `reason: "failed-create"`, and `error`, and the reservation is moved to aborted so the sequence remains burned. - FN-6782/FN-6796: self-healing emits `task:auto-recover-paused-abort-park` when it clears a benign pause-abort operator park, requeueing safe `todo`/`in-progress` rows or preserving a clean auto-merge-eligible `in-review` row for review progression. +- FN-9187: self-healing emits `task:auto-archive-failure-budget-exhausted` once when a stale done-task archive exhausts `MAX_STARVATION_DROPS`; metadata remains ids/counts/fixed outcomes only (`taskId`, `attempts`, `maxAttempts`, `reason`), and the one-shot task log directs an operator to repair the archive guard. - FN-8908: self-healing reserves `task:auto-recover-terminal-failure` and `task:auto-recover-terminal-failure-exhausted` for generic terminal-failure budget recovery. Metadata must remain ids/counts/outcomes-only and never include failure prose or the rotating `wedgeNotification.autoRecovery.applyToken`; that durable budget is the backoff source, and its apply fence—not the grace heuristic—authorizes the single clear/requeue transition. - FN-6793/FN-6797: self-healing emits `task:reconcile-in-review-unmet-dependencies` when it rebounds an `in-review` task whose declared dependencies are still unmet, and `task:reconcile-in-review-unmet-dependencies-no-action` when pause/user-pause, `autoMerge:false`, live execution/checkout proof, or a failed rebound mutation blocks that backward move. - Workspace (Phase D U1): self-healing emits `task:reconcile-workspace-partial-land` when it re-enqueues a partial/zero-landed workspace task's per-repo land (or parks it `failed` for proven branch absence or exhausted `evidence-unavailable` branch reads), and `task:reconcile-workspace-partial-land-no-action` when `autoMerge:false`, user-pause, a live sub-repo worktree (workspace-aware liveness), or `evidence-unavailable` blocks that backward move. The bounded evidence-exhaustion reason is `evidence-unavailable-exhausted`; audit metadata remains ids/counts/outcomes-only. diff --git a/docs/architecture.md b/docs/architecture.md index 42d9b90d5a..f183b7122a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2167,6 +2167,8 @@ The GitHub tracking state listener now attaches to every registered project stor #### Automated follow-up dedup (FN-5232 — REMOVED 2026-07-26) `packages/engine/src/verification-followup-dedup.ts` and its `createAutomatedFollowup`/`decideAutomatedFollowup` engine are DELETED, together with the meta-task auto-archive sweeps that existed to garbage-collect the cards it filed. Do not reintroduce either. +The surviving retention-driven `archiveStaleDoneTasks()` sweep is intentionally narrower: it pre-filters live lineage parents rather than clearing child provenance with `removeLineageReferences`, and it bounds each remaining same-reason archive failure with `MAX_STARVATION_DROPS`. Exhaustion writes one task log entry and `task:auto-archive-failure-budget-exhausted` audit event for operator action instead of retrying indefinitely. + - The engine filed a `sourceType: "recovery"` card whenever auto-merge gave up on verification or merge conflicts. That card mostly restated state already durable on the parent, which is parked `failed` with a descriptive `error` (verification) or carries an "Auto-merge gave up after conflict retries exhausted" `logEntry` (conflict). Those parent-side signals are the contract now; the card was redundant. - Because the classifier that cleaned these cards up (`classifyMetaTask`) matched a regex over title+description, it also matched ordinary feature work — and `resolveMetaTargetTaskId` bound an unmatched card to an unrelated task by creation order. Auto-archiving live work was the failure mode that motivated deleting the whole layer rather than tuning it. - The autostash-orphan path was the one caller carrying information found nowhere else, so it survives as a log entry + task comment (see the Stash Recovery bullets above) rather than a task. diff --git a/docs/run-audit.md b/docs/run-audit.md index d8968bdc8d..a741cab09c 100644 --- a/docs/run-audit.md +++ b/docs/run-audit.md @@ -47,6 +47,7 @@ Reconciliation-scoped auto-recover/reclaim events the self-healing sweep surface | --- | --- | | `task:auto-recover-paused-abort-park` | Self-healing clears a benign pause-abort operator park and requeues the task. | | `task:auto-rebound-paused-scope-decay` | Self-healing rebounds a task whose paused scope decayed past its floor, unblocking followers. | +| `task:auto-archive-failure-budget-exhausted` | Self-healing abandons a repeatedly failing stale-task archive and surfaces it for operator action. | | `task:reclaim-phantom-executor-binding` | Self-healing proves an in-memory executor-active binding is stale and requeues the task. | | `task:reconcile-orphaned-pending-step-results` | Self-healing rewrites orphaned `pending` workflow-step results (no live session) to `failed`. | | `task:reconcile-stale-duplicate-decision` | Self-healing clears a recurring duplicate-decision pause with no canonical target. | diff --git a/docs/settings-reference.md b/docs/settings-reference.md index adb1837db6..df3dc16d2a 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -719,7 +719,7 @@ Default notes: | `maxSpawnedAgentsPerParent` | `number` | `5` | Max child agents per parent task. | | `maxSpawnedAgentsGlobal` | `number` | `20` | Max spawned agents across one executor instance. | | `maintenanceIntervalMs` | `number` | `300000` | Periodic maintenance interval in ms (5 min). | -| `autoArchiveDoneTasksEnabled` | `boolean` | `true` | Enable periodic auto-archiving of done tasks. | +| `autoArchiveDoneTasksEnabled` | `boolean` | `true` | Enable periodic auto-archiving of done tasks; tasks with live lineage children are retained for operator action. | | `autoArchiveDoneAfterMs` | `number` | `172800000` | Age in ms after entering done before auto-archive (48h). | | `doneAutoArchiveDays` | `number` | `0` | Integer day-based done-task retention. `0` disables day override; values `> 0` take precedence over `autoArchiveDoneAfterMs`. | | `autoArchiveDuplicateTasksEnabled` | `boolean` | `false` | FN-7658/FN-8401: gates whether same-agent duplicate intake on every create backend auto-archives the later/new task. Default `false` — the duplicate is flagged in place (`nearDuplicateOf`/`nearDuplicateScore` marker, yellow "Duplicate" chip with Keep/Archive actions), and no live sibling is deleted or archived automatically. Set `true` to restore opt-in archival of the new task only. Does not affect ghost-bug preflight or tombstone-resurrection blocking. | diff --git a/packages/core/src/__tests__/postgres/live-lineage-children-lanes.pg.test.ts b/packages/core/src/__tests__/postgres/live-lineage-children-lanes.pg.test.ts index f36cad2e7a..11a2850e71 100644 --- a/packages/core/src/__tests__/postgres/live-lineage-children-lanes.pg.test.ts +++ b/packages/core/src/__tests__/postgres/live-lineage-children-lanes.pg.test.ts @@ -87,11 +87,12 @@ pgDescribe("findLiveLineageChildren under a renamed board vocabulary", () => { expect(await h.store().findLiveLineageChildren("KB-PARENT")).toEqual([]); }); - it("renamed vocabulary: a child in the RENAMED archive lane does not count as live", async () => { + it("renamed vocabulary: a child in the RENAMED archive lane does not block the parent archive", async () => { await seedRenamedWorkflow(); await seedLineagePair("vaulted"); expect(await h.store().findLiveLineageChildren("KB-PARENT")).toEqual([]); + await expect(h.store().archiveTask("KB-PARENT", { cleanup: false })).resolves.toMatchObject({ id: "KB-PARENT" }); }); /* diff --git a/packages/core/src/task-store/archive-lifecycle-2.ts b/packages/core/src/task-store/archive-lifecycle-2.ts index 2853436f80..de34fd0548 100644 --- a/packages/core/src/task-store/archive-lifecycle-2.ts +++ b/packages/core/src/task-store/archive-lifecycle-2.ts @@ -500,11 +500,14 @@ export async function archiveTaskBackendImpl(store: TaskStore, id: string, optio const entry = await store.taskToArchiveEntry(task, archivedAt); /* - FNXC:SpecLockLineageInvalidation 2026-08-10-14:33: - Archive keeps its legacy archived-lane semantics (undefined -> "archived") but threads that - one value through pre-read and gate. The workspace reservation is created inside the locked body. + FNXC:SelfHealing 2026-08-21-15:11: + Runfusion/Fusion#3497 requires the retention sweep and this transactional lineage gate to share + the project archive vocabulary. A renamed archived child is already filed and must not make the + sweep issue a guaranteed TaskHasLineageChildrenError; resolution failure remains fail-soft to the + legacy `archived` id. */ - const archiveLineageArchivedLanes: ReadonlySet | undefined = undefined; + const archiveLineageArchivedLanes = await resolveProjectColumnsForRoles(store, ["archived"]) + .catch(() => undefined); // Resolve configuration before the transaction; only its durable row verdict is authoritative. const livenessWipLanes = liveExecutionGuard === "refuse" ? await resolveArchiveLivenessWipLanes(store, id) : undefined; const archiveRun = async (context?: { candidateIds: string[]; promptByChildId: ReadonlyMap; locksHeld: boolean; attempt: number }) => { diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index 531f402c93..383bdde09c 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -2652,6 +2652,123 @@ describe("SelfHealingManager", () => { expect(store.archiveTaskAndCleanup).toHaveBeenCalledWith("FN-101"); expect(store.archiveTaskAndCleanup).not.toHaveBeenCalledWith("FN-100"); }); + + it("bounds same-reason archive failures and resets the budget when the failure class changes", async () => { + vi.setSystemTime(new Date("2026-01-04T00:00:00.000Z")); + (store.getSettings as ReturnType).mockResolvedValue({ + autoArchiveDoneTasksEnabled: true, + autoArchiveDoneAfterMs: 24 * 60 * 60 * 1000, + doneAutoArchiveDays: 0, + } as unknown as Settings); + const stale = [{ id: "FN-RETRY", column: "done", columnMovedAt: "2026-01-02T00:00:00.000Z", updatedAt: "2026-01-02T00:00:00.000Z" }]; + (store.listTasks as ReturnType).mockResolvedValue(stale); + (store.archiveTaskAndCleanup as ReturnType).mockRejectedValue(new Error("disk busy")); + + for (let index = 0; index < 10; index++) await manager.archiveStaleDoneTasks(); + + expect(store.archiveTaskAndCleanup).toHaveBeenCalledTimes(3); + + (store.archiveTaskAndCleanup as ReturnType).mockClear(); + const taskLive = Object.assign(new Error("live"), { name: "TaskIsLiveError" }); + (store.archiveTaskAndCleanup as ReturnType) + .mockRejectedValueOnce(new Error("disk busy")) + .mockRejectedValueOnce(taskLive) + .mockRejectedValueOnce(new Error("disk busy")); + const managerWithChangingFailure = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + + await managerWithChangingFailure.archiveStaleDoneTasks(); + await managerWithChangingFailure.archiveStaleDoneTasks(); + await managerWithChangingFailure.archiveStaleDoneTasks(); + + expect(store.archiveTaskAndCleanup).toHaveBeenCalledTimes(3); + managerWithChangingFailure.stop(); + }); + + it("escalates an exhausted archive budget once without letting log or audit failures stop other archives", async () => { + vi.setSystemTime(new Date("2026-01-04T00:00:00.000Z")); + (store.getSettings as ReturnType).mockResolvedValue({ + autoArchiveDoneTasksEnabled: true, + autoArchiveDoneAfterMs: 24 * 60 * 60 * 1000, + doneAutoArchiveDays: 0, + } as unknown as Settings); + (store.listTasks as ReturnType).mockResolvedValue([ + { id: "FN-EXHAUSTED", column: "done", columnMovedAt: "2026-01-02T00:00:00.000Z", updatedAt: "2026-01-02T00:00:00.000Z" }, + { id: "FN-OTHER", column: "done", columnMovedAt: "2026-01-02T00:00:00.000Z", updatedAt: "2026-01-02T00:00:00.000Z" }, + ]); + (store.archiveTaskAndCleanup as ReturnType).mockImplementation(async (id: string) => { + if (id === "FN-EXHAUSTED") throw new Error("disk busy"); + return {}; + }); + (store.logEntry as ReturnType).mockRejectedValue(new Error("log unavailable")); + (store.recordRunAuditEvent as ReturnType).mockRejectedValue(new Error("audit unavailable")); + const priorErrorCalls = (getSelfHealingLogger().error as ReturnType).mock.calls.length; + + for (let index = 0; index < 10; index++) await manager.archiveStaleDoneTasks(); + + expect((store.archiveTaskAndCleanup as ReturnType).mock.calls.filter(([id]) => id === "FN-EXHAUSTED")).toHaveLength(3); + expect(store.logEntry).toHaveBeenCalledTimes(1); + const exhaustedEvents = (store.recordRunAuditEvent as ReturnType).mock.calls.filter( + ([event]) => (event as { mutationType?: string }).mutationType === "task:auto-archive-failure-budget-exhausted", + ); + expect(exhaustedEvents).toHaveLength(1); + expect((getSelfHealingLogger().error as ReturnType).mock.calls).toHaveLength(priorErrorCalls + 1); + expect(store.archiveTaskAndCleanup).toHaveBeenCalledWith("FN-OTHER"); + }); + + it("clears an archive failure budget after success and when a task leaves the candidate set", async () => { + vi.setSystemTime(new Date("2026-01-04T00:00:00.000Z")); + (store.getSettings as ReturnType).mockResolvedValue({ + autoArchiveDoneTasksEnabled: true, + autoArchiveDoneAfterMs: 24 * 60 * 60 * 1000, + doneAutoArchiveDays: 0, + } as unknown as Settings); + const stale = [{ id: "FN-RESET", column: "done", columnMovedAt: "2026-01-02T00:00:00.000Z", updatedAt: "2026-01-02T00:00:00.000Z" }]; + (store.listTasks as ReturnType) + .mockResolvedValueOnce(stale) + .mockResolvedValueOnce(stale) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce(stale); + (store.archiveTaskAndCleanup as ReturnType) + .mockRejectedValueOnce(new Error("disk busy")) + .mockResolvedValueOnce({}) + .mockRejectedValueOnce(new Error("disk busy")); + + await manager.archiveStaleDoneTasks(); + await manager.archiveStaleDoneTasks(); + await manager.archiveStaleDoneTasks(); + await manager.archiveStaleDoneTasks(); + + expect(store.archiveTaskAndCleanup).toHaveBeenCalledTimes(3); + }); + + it("skips stale done lineage parents, including complete children, without blocking unrelated archives", async () => { + vi.setSystemTime(new Date("2026-01-04T00:00:00.000Z")); + (store.getSettings as ReturnType).mockResolvedValue({ + autoArchiveDoneTasksEnabled: true, + autoArchiveDoneAfterMs: 24 * 60 * 60 * 1000, + doneAutoArchiveDays: 0, + } as unknown as Settings); + (store.listTasks as ReturnType).mockResolvedValue([ + { id: "FN-PARENT-TODO", column: "done", columnMovedAt: "2026-01-02T00:00:00.000Z", updatedAt: "2026-01-02T00:00:00.000Z" }, + { id: "FN-PARENT-DONE", column: "done", columnMovedAt: "2026-01-02T00:00:00.000Z", updatedAt: "2026-01-02T00:00:00.000Z" }, + { id: "FN-PARENT-MULTI", column: "done", columnMovedAt: "2026-01-02T00:00:00.000Z", updatedAt: "2026-01-02T00:00:00.000Z" }, + { id: "FN-UNRELATED", column: "done", columnMovedAt: "2026-01-02T00:00:00.000Z", updatedAt: "2026-01-02T00:00:00.000Z" }, + { id: "FN-CHILD-TODO", column: "todo", sourceParentTaskId: "FN-PARENT-TODO" }, + { id: "FN-CHILD-DONE", column: "done", sourceParentTaskId: "FN-PARENT-DONE", columnMovedAt: "2026-01-03T23:00:00.000Z", updatedAt: "2026-01-03T23:00:00.000Z" }, + { id: "FN-CHILD-ONE", column: "todo", sourceParentTaskId: "FN-PARENT-MULTI" }, + { id: "FN-CHILD-TWO", column: "in-progress", sourceParentTaskId: "FN-PARENT-MULTI" }, + ]); + + const priorErrorCalls = (getSelfHealingLogger().error as ReturnType).mock.calls.length; + for (let index = 0; index < 6; index++) await manager.archiveStaleDoneTasks(); + + expect(store.archiveTaskAndCleanup).toHaveBeenCalledTimes(6); + expect(store.archiveTaskAndCleanup).toHaveBeenCalledWith("FN-UNRELATED"); + expect(store.archiveTaskAndCleanup).not.toHaveBeenCalledWith("FN-PARENT-TODO"); + expect(store.archiveTaskAndCleanup).not.toHaveBeenCalledWith("FN-PARENT-DONE"); + expect(store.archiveTaskAndCleanup).not.toHaveBeenCalledWith("FN-PARENT-MULTI"); + expect((getSelfHealingLogger().error as ReturnType).mock.calls).toHaveLength(priorErrorCalls); + }); }); // ── Completed task recovery ───────────────────────────────────────── diff --git a/packages/engine/src/run-audit/run-audit-catalogue.ts b/packages/engine/src/run-audit/run-audit-catalogue.ts index f61d32016c..5a619d56b5 100644 --- a/packages/engine/src/run-audit/run-audit-catalogue.ts +++ b/packages/engine/src/run-audit/run-audit-catalogue.ts @@ -50,6 +50,7 @@ export const DELIVERY_PIPELINE_RUN_AUDIT_EVENTS_LITERALS = [ /* ── 2. Self-healing reconciliation events ─────────────────────────────── */ "task:auto-recover-paused-abort-park", "task:auto-rebound-paused-scope-decay", + "task:auto-archive-failure-budget-exhausted", "task:reclaim-phantom-executor-binding", "task:reconcile-orphaned-pending-step-results", "task:reconcile-stale-duplicate-decision", @@ -122,6 +123,8 @@ export const DELIVERY_PIPELINE_RUN_AUDIT_EVENT_NOTES: Readonly(MERGE_A import { classifyTransientMergeError } from "./errors/transient-merge-error-classifier.js"; export { classifyTransientMergeError } from "./errors/transient-merge-error-classifier.js"; const MAX_STARVATION_DROPS = 3; +type AutoArchiveFailureReason = "lineage-children" | "task-live" | "dependents" | "not-found" | "unknown"; + +function classifyAutoArchiveFailure(err: unknown): AutoArchiveFailureReason { + if (!(err instanceof Error)) return "unknown"; + switch (err.name) { + case "TaskHasLineageChildrenError": return "lineage-children"; + case "TaskIsLiveError": return "task-live"; + case "TaskHasDependentsError": return "dependents"; + case "TaskNotFoundError": return "not-found"; + default: return "unknown"; + } +} /* FNXC:Workspace 2026-08-15-05:13: Failed workspace tasks are routinely retried with their progress preserved. Terminal teardown therefore @@ -743,6 +755,14 @@ export class SelfHealingManager extends SelfHealingGitEvidence { private deadlockRecoveryCooldown: Map = new Map(); private mergeStarvationDrops: Map = new Map(); /* + FNXC:SelfHealing 2026-08-20-08:08: + Runfusion/Fusion#3497 requires a process-scoped budget for stale-archive failures: repeating a + permanent refusal floods logs and obscures actionable failures. Restarting gets a fresh budget + because an operator may have repaired the cause; the one-shot durable escalation carries the + unresolved finding across restarts. + */ + private readonly autoArchiveFailures: Map = new Map(); + /* FNXC:Workspace 2026-08-15-04:42: The partial-land reconciler separately bounds rejected merge enqueues and unavailable branch evidence. A clean `show-ref` exit 1 proves a branch is absent; timeout, missing directories, and @@ -3179,10 +3199,28 @@ export class SelfHealingManager extends SelfHealingGitEvidence { selection (docs/solutions/workflow-learnings/project-union-versus-per-task-lanes.md). */ const dependentTerminalColumns = await resolveProjectColumnsForRoles(this.store, TERMINAL_ROLES); + // FNXC:SelfHealing 2026-08-20-08:02: + // Runfusion/Fusion#3497 found this retention sweep reissuing TaskHasLineageChildrenError every + // interval. Archive lanes alone mirror the store guard: a complete child still preserves lineage, + // while clearing sourceParentTaskId via removeLineageReferences is destructive provenance editing + // that retention automation is not authorized to perform. + const archivedColumns = await resolveProjectColumnsForRoles(this.store, ["archived"]) + .catch(() => new Set()); + const tasksWithLiveLineageChildren = new Map(); for (const t of tasks) { - if (dependentTerminalColumns.has(t.column)) continue; - for (const depId of t.dependencies ?? []) { - tasksWithActiveDependents.add(depId); + if (!dependentTerminalColumns.has(t.column)) { + for (const depId of t.dependencies ?? []) { + tasksWithActiveDependents.add(depId); + } + } + if ( + !archivedColumns.has(t.column) + && typeof t.sourceParentTaskId === "string" + && t.sourceParentTaskId.length > 0 + ) { + const children = tasksWithLiveLineageChildren.get(t.sourceParentTaskId) ?? []; + children.push(t.id); + tasksWithLiveLineageChildren.set(t.sourceParentTaskId, children); } } @@ -3203,9 +3241,18 @@ export class SelfHealingManager extends SelfHealingGitEvidence { log.debug(`Skipping auto-archive of ${t.id}: has active dependents`); return false; } + const lineageChildren = tasksWithLiveLineageChildren.get(t.id); + if (lineageChildren) { + log.debug(`Skipping auto-archive of ${t.id}: has live lineage children ${lineageChildren.join(", ")}`); + return false; + } return true; }); + const staleTaskIds = new Set(stale.map((task) => task.id)); + for (const taskId of this.autoArchiveFailures.keys()) { + if (!staleTaskIds.has(taskId)) this.autoArchiveFailures.delete(taskId); + } if (stale.length === 0) return 0; log.debug(`Auto-archiving ${stale.length} done task(s) older than ${archiveAfterMs}ms`); @@ -3213,15 +3260,51 @@ export class SelfHealingManager extends SelfHealingGitEvidence { let archived = 0; const thresholdDays = Math.floor(archiveAfterMs / 86_400_000); for (const task of stale) { + if ((this.autoArchiveFailures.get(task.id)?.count ?? 0) >= MAX_STARVATION_DROPS) continue; try { await this.store.archiveTaskAndCleanup(task.id); + this.autoArchiveFailures.delete(task.id); archived++; const ts = task.columnMovedAt || task.updatedAt; const movedAt = ts ? Date.parse(ts) : NaN; const ageDays = Number.isFinite(movedAt) ? Math.floor((now - movedAt) / 86_400_000) : 0; log.debug(`auto-archive: archived ${task.id} (age ${ageDays}d, threshold ${thresholdDays}d)`); - } catch (err: unknown) { const errorMessage = err instanceof Error ? err.message : String(err); - log.error(`Failed to auto-archive ${task.id}: ${errorMessage}`); + } catch (err: unknown) { + const reason = classifyAutoArchiveFailure(err); + const prior = this.autoArchiveFailures.get(task.id); + const count = prior?.signature === reason ? prior.count + 1 : 1; + this.autoArchiveFailures.set(task.id, { count, signature: reason }); + if (count < MAX_STARVATION_DROPS) { + log.warn(`Failed to auto-archive ${task.id} (${count}/${MAX_STARVATION_DROPS}, ${reason})`); + } else { + log.error(`Auto-archive abandoned for ${task.id} after ${count}/${MAX_STARVATION_DROPS} failures (${reason})`); + /* + FNXC:SelfHealing 2026-08-20-08:13: + This one-shot log entry makes an abandoned retention action visible to operators. It bumps + updatedAt, but modern stale rows use columnMovedAt; legacy rows move out of retention once, + and the exhausted in-memory budget prevents further archive attempts or repeated escalation. + */ + const remedy = reason === "lineage-children" + ? "Archive or unlink the referencing child, or use fn_task_archive with removeLineageReferences: true." + : "Inspect the task and resolve the reported archive guard before retrying manually."; + try { + await this.store.logEntry( + task.id, + `[self-healing] Auto-archive abandoned after ${count} consecutive ${reason} failures. ${remedy}`, + ); + } catch (logErr: unknown) { + log.warn(`Could not record auto-archive escalation for ${task.id}: ${logErr instanceof Error ? logErr.message : String(logErr)}`); + } + await emitBoundedRunAudit(this.store, { + taskId: task.id, + agentId: "self-healing", + runId: generateSyntheticRunId("self-heal-auto-archive-exhausted", task.id), + domain: "database", + mutationType: "task:auto-archive-failure-budget-exhausted", + target: task.id, + metadata: { taskId: task.id, attempts: count, maxAttempts: MAX_STARVATION_DROPS, reason }, + }, { log }); + } } } diff --git a/packages/engine/src/util/run-audit.ts b/packages/engine/src/util/run-audit.ts index 4e290e233d..2688073a60 100644 --- a/packages/engine/src/util/run-audit.ts +++ b/packages/engine/src/util/run-audit.ts @@ -572,6 +572,8 @@ export type DatabaseMutationType = // task:auto-archived-duplicate metadata: { siblingTaskIds: string[]; scores: Record } | "task:auto-archived-ghost-bug" | "task:auto-archived-duplicate" + /** Metadata: { taskId, attempts, maxAttempts, reason: "lineage-children" | "task-live" | "dependents" | "not-found" | "unknown" } */ + | "task:auto-archive-failure-budget-exhausted" | "task:auto-reconciled-self-defeating-dep" | "task:soft-delete-column-reconciled" | "task:dependency-cycle-rejected"