From ccebe5c5cf152513459b60b88f68d27e4e3faa4d Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sun, 9 Aug 2026 13:48:39 -1000 Subject: [PATCH] fix(engine): keep orphaned planning recovery alive (#3399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Resolves merge conflicts from #3392 against current `main`. `main` already landed the core #3386 fix via FN-8909 (`includeArchived: false` live-row enumeration + per-task isolation). This PR rebases the remaining #3392 refinements on top of that: - Best-effort, secret-free audit emission (per-task and no-action) - Distinguish `no-eligible-orphan` vs `all-attempts-failed` / `no-finalization` no-action outcomes - Redacted `errorType=` warn logs so poisoned-row failures cannot abort the sweep or leak error prose Supersedes #3392 (fork head is not writable from this environment despite `maintainer_can_modify`). ## Test plan - [x] `pnpm --filter @fusion/engine exec vitest run src/__tests__/self-healing.test.ts --project engine-default --run -t "finalizeOrphanedPlanningSegments"` — 8 passed - [ ] CI PR checks green ## Summary by CodeRabbit * **Bug Fixes** * Improved recovery of orphaned planning segments when individual finalization attempts fail. * Recovery now continues successfully even if audit recording encounters an error. * Added clearer recovery outcomes, distinguishing cases where no segments qualify, all attempts fail, or only some segments are finalized. * Warning messages now provide structured error details without exposing sensitive information. Co-authored-by: Codex --- .changeset/orphaned-planning-segments.md | 7 ++ .../engine/src/__tests__/self-healing.test.ts | 54 ++++++++++++++- packages/engine/src/self-healing.ts | 66 +++++++++++++------ 3 files changed, 106 insertions(+), 21 deletions(-) create mode 100644 .changeset/orphaned-planning-segments.md diff --git a/.changeset/orphaned-planning-segments.md b/.changeset/orphaned-planning-segments.md new file mode 100644 index 0000000000..0666fe7175 --- /dev/null +++ b/.changeset/orphaned-planning-segments.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Harden orphaned planning recovery audits so failed rows cannot abort the sweep. +category: fix +dev: Keep FN-8909 live-row enumeration; isolate audit emission failures and distinguish all-attempts-failed no-action outcomes (PR #3392). diff --git a/packages/engine/src/__tests__/self-healing.test.ts b/packages/engine/src/__tests__/self-healing.test.ts index e4c76b96af..f87638fde8 100644 --- a/packages/engine/src/__tests__/self-healing.test.ts +++ b/packages/engine/src/__tests__/self-healing.test.ts @@ -8915,7 +8915,7 @@ describe("SelfHealingManager", () => { await expect(recovery.finalizeOrphanedPlanningSegments()).resolves.toBe(1); expect(healthy).toMatchObject({ planningStartedAt: null, cumulativePlanningMs: 1050 }); - expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(expect.stringContaining("Failed to finalize orphaned planning segment for FN-RACING")); + expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(expect.stringContaining("orphaned planning segment FN-RACING could not be finalized: errorType=")); recovery.stop(); }); @@ -8942,7 +8942,57 @@ describe("SelfHealingManager", () => { await expect(recovery.finalizeOrphanedPlanningSegments()).resolves.toBe(1); expect(updateTask).toHaveBeenCalledWith(healthy.id, expect.objectContaining({ planningStartedAt: null, cumulativePlanningMs: 1050 })); expect(getTask).not.toHaveBeenCalledWith(firstPoison.id); - expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(expect.stringContaining("Failed to finalize orphaned planning segment for FN-POISON-2")); + expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(expect.stringContaining("orphaned planning segment FN-POISON-2 could not be finalized: errorType=")); + + recovery.stop(); + }); + + it("reports when every eligible orphan attempt fails", async () => { + const failedTasks = [ + { id: "FN-PLAN-ERROR-1", planningStartedAt: "2026-01-01T00:00:00.000Z" }, + { id: "FN-PLAN-ERROR-2", planningStartedAt: "2026-01-01T00:00:00.000Z" }, + ] as Task[]; + const updateTaskAtomic = vi.fn(async () => { + throw new Error("reconciliation unavailable"); + }); + const recoveryStore = createMockStore({ + listTasks: vi.fn().mockResolvedValue(failedTasks), + updateTaskAtomic, + }); + const recovery = new SelfHealingManager(recoveryStore, { + rootDir: "/tmp/test-project", + getPlanningTaskIds: () => new Set(), + hasActivePlanningWorkflowSession: () => false, + }); + vi.setSystemTime(new Date("2026-01-01T00:00:01.000Z")); + + expect(await recovery.finalizeOrphanedPlanningSegments()).toBe(0); + expect(updateTaskAtomic).toHaveBeenCalledTimes(2); + expect(recoveryStore.recordRunAuditEvent).toHaveBeenLastCalledWith(expect.objectContaining({ + mutationType: "task:reconcile-orphaned-planning-segment-no-action", + metadata: { finalizedCount: 0, reason: "all-attempts-failed", attemptedCount: 2 }, + })); + + recovery.stop(); + }); + + it("keeps a no-action sweep successful when audit recording fails", async () => { + const task = { id: "FN-PLAN-AUDIT-ERROR", planningStartedAt: "2026-01-01T00:00:00.000Z" } as Task; + const recordRunAuditEvent = vi.fn().mockRejectedValue(new Error("audit unavailable")); + const recoveryStore = createMockStore({ + listTasks: vi.fn().mockResolvedValue([task]), + recordRunAuditEvent, + }); + const recovery = new SelfHealingManager(recoveryStore, { + rootDir: "/tmp/test-project", + getPlanningTaskIds: () => new Set([task.id]), + hasActivePlanningWorkflowSession: () => false, + }); + + await expect(recovery.finalizeOrphanedPlanningSegments()).resolves.toBe(0); + expect(recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "task:reconcile-orphaned-planning-segment-no-action", + })); recovery.stop(); }); diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index a8478ed6f6..67cb0d80ce 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -14781,9 +14781,18 @@ const movedTask = await this.store.moveTask(task.id, completeLane); * a deletedAt filter alone could not contain this archive path. Per-task and * sweep guards ensure one race or poisoned row cannot disable finalization * store-wide during startup or maintenance (Runfusion/Fusion#3386). + * + * FNXC:TaskTiming 2026-08-09-23:30: + * Audit emission is best-effort and secret-free: a failed per-task or no-action + * audit must not turn a successful recovery into a failed sweep. When nothing + * finalizes, distinguish no-eligible-orphan from all-attempts-failed so operators + * can tell "nothing to do" from "everything exploded" (PR #3392 refinements on + * top of FN-8909). */ async finalizeOrphanedPlanningSegments(): Promise { let finalized = 0; + let attempted = 0; + let failedAttempts = 0; try { const planningIds = this.options.getPlanningTaskIds?.() ?? new Set(); const tasks = await this.store.listTasks({ slim: true, includeArchived: false }); @@ -14797,6 +14806,7 @@ const movedTask = await this.store.moveTask(task.id, completeLane); planningIds.has(task.id) || this.options.hasActivePlanningWorkflowSession?.(task.id) ) continue; + attempted++; let applied = false; const endMs = Date.now(); if (typeof this.store.updateTaskAtomic === "function") { @@ -14818,29 +14828,47 @@ const movedTask = await this.store.moveTask(task.id, completeLane); // FNXC:TaskTiming 2026-07-30-21:40: this recovery is operator-auditable // without persisting duration prose; the atomically finalized task id // and fixed no-live-owner reason are sufficient forensic evidence. - await this.store.recordRunAuditEvent?.({ - taskId: task.id, - agentId: "self-healing", - runId: generateSyntheticRunId("orphaned-planning-segment", task.id), - domain: "database", - mutationType: "task:reconcile-orphaned-planning-segment", - target: task.id, - metadata: { taskId: task.id, finalizedCount: 1, reason: "no-live-planning-owner" }, - }); + try { + await this.store.recordRunAuditEvent?.({ + taskId: task.id, + agentId: "self-healing", + runId: generateSyntheticRunId("orphaned-planning-segment", task.id), + domain: "database", + mutationType: "task:reconcile-orphaned-planning-segment", + target: task.id, + metadata: { taskId: task.id, finalizedCount: 1, reason: "no-live-planning-owner" }, + }); + } catch (auditError: unknown) { + const errorType = auditError instanceof Error && auditError.name ? auditError.name : "unknown-error"; + log.warn(`[self-healing] orphaned planning segment ${task.id} audit could not be recorded: errorType=${errorType}`); + } } catch (err: unknown) { - const errorMessage = err instanceof Error ? err.message : String(err); - log.warn(`Failed to finalize orphaned planning segment for ${task.id}: ${errorMessage}`); + failedAttempts++; + // Keep recovery logs bounded and secret-free; the task id and stable + // error type are sufficient to correlate the failed row in run audit. + const errorType = err instanceof Error && err.name ? err.name : "unknown-error"; + log.warn(`[self-healing] orphaned planning segment ${task.id} could not be finalized: errorType=${errorType}`); } } if (finalized === 0) { - await this.store.recordRunAuditEvent?.({ - agentId: "self-healing", - runId: generateSyntheticRunId("orphaned-planning-segment", "global"), - domain: "database", - mutationType: "task:reconcile-orphaned-planning-segment-no-action", - target: "planning-segments", - metadata: { finalizedCount: 0, reason: "no-eligible-orphan" }, - }); + const noActionMetadata = attempted === 0 + ? { finalizedCount: 0, reason: "no-eligible-orphan" } + : failedAttempts === attempted + ? { finalizedCount: 0, reason: "all-attempts-failed", attemptedCount: attempted } + : { finalizedCount: 0, reason: "no-finalization", attemptedCount: attempted, failedAttemptCount: failedAttempts }; + try { + await this.store.recordRunAuditEvent?.({ + agentId: "self-healing", + runId: generateSyntheticRunId("orphaned-planning-segment", "global"), + domain: "database", + mutationType: "task:reconcile-orphaned-planning-segment-no-action", + target: "planning-segments", + metadata: noActionMetadata, + }); + } catch (auditError: unknown) { + const errorType = auditError instanceof Error && auditError.name ? auditError.name : "unknown-error"; + log.warn(`[self-healing] orphaned planning segment no-action audit could not be recorded: errorType=${errorType}`); + } } return finalized; } catch (err: unknown) {