fix(engine): keep orphaned planning recovery alive (#3399)
## Summary Resolves merge conflicts from #3392 against current `main`. `main` already landed the core #3386 fix via FN-8909 (`includeArchived: false` live-row enumeration + per-task isolation). This PR rebases the remaining #3392 refinements on top of that: - Best-effort, secret-free audit emission (per-task and no-action) - Distinguish `no-eligible-orphan` vs `all-attempts-failed` / `no-finalization` no-action outcomes - Redacted `errorType=` warn logs so poisoned-row failures cannot abort the sweep or leak error prose Supersedes #3392 (fork head is not writable from this environment despite `maintainer_can_modify`). ## Test plan - [x] `pnpm --filter @fusion/engine exec vitest run src/__tests__/self-healing.test.ts --project engine-default --run -t "finalizeOrphanedPlanningSegments"` — 8 passed - [ ] CI PR checks green <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Bug Fixes** * Improved recovery of orphaned planning segments when individual finalization attempts fail. * Recovery now continues successfully even if audit recording encounters an error. * Added clearer recovery outcomes, distinguishing cases where no segments qualify, all attempts fail, or only some segments are finalized. * Warning messages now provide structured error details without exposing sensitive information. <!-- end of auto-generated comment: release notes by coderabbit.ai --> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
7
.changeset/orphaned-planning-segments.md
Normal file
7
.changeset/orphaned-planning-segments.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
summary: Harden orphaned planning recovery audits so failed rows cannot abort the sweep.
|
||||||
|
category: fix
|
||||||
|
dev: Keep FN-8909 live-row enumeration; isolate audit emission failures and distinguish all-attempts-failed no-action outcomes (PR #3392).
|
||||||
@@ -8915,7 +8915,7 @@ describe("SelfHealingManager", () => {
|
|||||||
|
|
||||||
await expect(recovery.finalizeOrphanedPlanningSegments()).resolves.toBe(1);
|
await expect(recovery.finalizeOrphanedPlanningSegments()).resolves.toBe(1);
|
||||||
expect(healthy).toMatchObject({ planningStartedAt: null, cumulativePlanningMs: 1050 });
|
expect(healthy).toMatchObject({ planningStartedAt: null, cumulativePlanningMs: 1050 });
|
||||||
expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(expect.stringContaining("Failed to finalize orphaned planning segment for FN-RACING"));
|
expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(expect.stringContaining("orphaned planning segment FN-RACING could not be finalized: errorType="));
|
||||||
|
|
||||||
recovery.stop();
|
recovery.stop();
|
||||||
});
|
});
|
||||||
@@ -8942,7 +8942,57 @@ describe("SelfHealingManager", () => {
|
|||||||
await expect(recovery.finalizeOrphanedPlanningSegments()).resolves.toBe(1);
|
await expect(recovery.finalizeOrphanedPlanningSegments()).resolves.toBe(1);
|
||||||
expect(updateTask).toHaveBeenCalledWith(healthy.id, expect.objectContaining({ planningStartedAt: null, cumulativePlanningMs: 1050 }));
|
expect(updateTask).toHaveBeenCalledWith(healthy.id, expect.objectContaining({ planningStartedAt: null, cumulativePlanningMs: 1050 }));
|
||||||
expect(getTask).not.toHaveBeenCalledWith(firstPoison.id);
|
expect(getTask).not.toHaveBeenCalledWith(firstPoison.id);
|
||||||
expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(expect.stringContaining("Failed to finalize orphaned planning segment for FN-POISON-2"));
|
expect(getSelfHealingLogger().warn).toHaveBeenCalledWith(expect.stringContaining("orphaned planning segment FN-POISON-2 could not be finalized: errorType="));
|
||||||
|
|
||||||
|
recovery.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports when every eligible orphan attempt fails", async () => {
|
||||||
|
const failedTasks = [
|
||||||
|
{ id: "FN-PLAN-ERROR-1", planningStartedAt: "2026-01-01T00:00:00.000Z" },
|
||||||
|
{ id: "FN-PLAN-ERROR-2", planningStartedAt: "2026-01-01T00:00:00.000Z" },
|
||||||
|
] as Task[];
|
||||||
|
const updateTaskAtomic = vi.fn(async () => {
|
||||||
|
throw new Error("reconciliation unavailable");
|
||||||
|
});
|
||||||
|
const recoveryStore = createMockStore({
|
||||||
|
listTasks: vi.fn().mockResolvedValue(failedTasks),
|
||||||
|
updateTaskAtomic,
|
||||||
|
});
|
||||||
|
const recovery = new SelfHealingManager(recoveryStore, {
|
||||||
|
rootDir: "/tmp/test-project",
|
||||||
|
getPlanningTaskIds: () => new Set<string>(),
|
||||||
|
hasActivePlanningWorkflowSession: () => false,
|
||||||
|
});
|
||||||
|
vi.setSystemTime(new Date("2026-01-01T00:00:01.000Z"));
|
||||||
|
|
||||||
|
expect(await recovery.finalizeOrphanedPlanningSegments()).toBe(0);
|
||||||
|
expect(updateTaskAtomic).toHaveBeenCalledTimes(2);
|
||||||
|
expect(recoveryStore.recordRunAuditEvent).toHaveBeenLastCalledWith(expect.objectContaining({
|
||||||
|
mutationType: "task:reconcile-orphaned-planning-segment-no-action",
|
||||||
|
metadata: { finalizedCount: 0, reason: "all-attempts-failed", attemptedCount: 2 },
|
||||||
|
}));
|
||||||
|
|
||||||
|
recovery.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps a no-action sweep successful when audit recording fails", async () => {
|
||||||
|
const task = { id: "FN-PLAN-AUDIT-ERROR", planningStartedAt: "2026-01-01T00:00:00.000Z" } as Task;
|
||||||
|
const recordRunAuditEvent = vi.fn().mockRejectedValue(new Error("audit unavailable"));
|
||||||
|
const recoveryStore = createMockStore({
|
||||||
|
listTasks: vi.fn().mockResolvedValue([task]),
|
||||||
|
recordRunAuditEvent,
|
||||||
|
});
|
||||||
|
const recovery = new SelfHealingManager(recoveryStore, {
|
||||||
|
rootDir: "/tmp/test-project",
|
||||||
|
getPlanningTaskIds: () => new Set([task.id]),
|
||||||
|
hasActivePlanningWorkflowSession: () => false,
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(recovery.finalizeOrphanedPlanningSegments()).resolves.toBe(0);
|
||||||
|
expect(recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
mutationType: "task:reconcile-orphaned-planning-segment-no-action",
|
||||||
|
}));
|
||||||
|
|
||||||
recovery.stop();
|
recovery.stop();
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -14781,9 +14781,18 @@ const movedTask = await this.store.moveTask(task.id, completeLane);
|
|||||||
* a deletedAt filter alone could not contain this archive path. Per-task and
|
* a deletedAt filter alone could not contain this archive path. Per-task and
|
||||||
* sweep guards ensure one race or poisoned row cannot disable finalization
|
* sweep guards ensure one race or poisoned row cannot disable finalization
|
||||||
* store-wide during startup or maintenance (Runfusion/Fusion#3386).
|
* store-wide during startup or maintenance (Runfusion/Fusion#3386).
|
||||||
|
*
|
||||||
|
* FNXC:TaskTiming 2026-08-09-23:30:
|
||||||
|
* Audit emission is best-effort and secret-free: a failed per-task or no-action
|
||||||
|
* audit must not turn a successful recovery into a failed sweep. When nothing
|
||||||
|
* finalizes, distinguish no-eligible-orphan from all-attempts-failed so operators
|
||||||
|
* can tell "nothing to do" from "everything exploded" (PR #3392 refinements on
|
||||||
|
* top of FN-8909).
|
||||||
*/
|
*/
|
||||||
async finalizeOrphanedPlanningSegments(): Promise<number> {
|
async finalizeOrphanedPlanningSegments(): Promise<number> {
|
||||||
let finalized = 0;
|
let finalized = 0;
|
||||||
|
let attempted = 0;
|
||||||
|
let failedAttempts = 0;
|
||||||
try {
|
try {
|
||||||
const planningIds = this.options.getPlanningTaskIds?.() ?? new Set<string>();
|
const planningIds = this.options.getPlanningTaskIds?.() ?? new Set<string>();
|
||||||
const tasks = await this.store.listTasks({ slim: true, includeArchived: false });
|
const tasks = await this.store.listTasks({ slim: true, includeArchived: false });
|
||||||
@@ -14797,6 +14806,7 @@ const movedTask = await this.store.moveTask(task.id, completeLane);
|
|||||||
planningIds.has(task.id) ||
|
planningIds.has(task.id) ||
|
||||||
this.options.hasActivePlanningWorkflowSession?.(task.id)
|
this.options.hasActivePlanningWorkflowSession?.(task.id)
|
||||||
) continue;
|
) continue;
|
||||||
|
attempted++;
|
||||||
let applied = false;
|
let applied = false;
|
||||||
const endMs = Date.now();
|
const endMs = Date.now();
|
||||||
if (typeof this.store.updateTaskAtomic === "function") {
|
if (typeof this.store.updateTaskAtomic === "function") {
|
||||||
@@ -14818,29 +14828,47 @@ const movedTask = await this.store.moveTask(task.id, completeLane);
|
|||||||
// FNXC:TaskTiming 2026-07-30-21:40: this recovery is operator-auditable
|
// FNXC:TaskTiming 2026-07-30-21:40: this recovery is operator-auditable
|
||||||
// without persisting duration prose; the atomically finalized task id
|
// without persisting duration prose; the atomically finalized task id
|
||||||
// and fixed no-live-owner reason are sufficient forensic evidence.
|
// and fixed no-live-owner reason are sufficient forensic evidence.
|
||||||
await this.store.recordRunAuditEvent?.({
|
try {
|
||||||
taskId: task.id,
|
await this.store.recordRunAuditEvent?.({
|
||||||
agentId: "self-healing",
|
taskId: task.id,
|
||||||
runId: generateSyntheticRunId("orphaned-planning-segment", task.id),
|
agentId: "self-healing",
|
||||||
domain: "database",
|
runId: generateSyntheticRunId("orphaned-planning-segment", task.id),
|
||||||
mutationType: "task:reconcile-orphaned-planning-segment",
|
domain: "database",
|
||||||
target: task.id,
|
mutationType: "task:reconcile-orphaned-planning-segment",
|
||||||
metadata: { taskId: task.id, finalizedCount: 1, reason: "no-live-planning-owner" },
|
target: task.id,
|
||||||
});
|
metadata: { taskId: task.id, finalizedCount: 1, reason: "no-live-planning-owner" },
|
||||||
|
});
|
||||||
|
} catch (auditError: unknown) {
|
||||||
|
const errorType = auditError instanceof Error && auditError.name ? auditError.name : "unknown-error";
|
||||||
|
log.warn(`[self-healing] orphaned planning segment ${task.id} audit could not be recorded: errorType=${errorType}`);
|
||||||
|
}
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
failedAttempts++;
|
||||||
log.warn(`Failed to finalize orphaned planning segment for ${task.id}: ${errorMessage}`);
|
// Keep recovery logs bounded and secret-free; the task id and stable
|
||||||
|
// error type are sufficient to correlate the failed row in run audit.
|
||||||
|
const errorType = err instanceof Error && err.name ? err.name : "unknown-error";
|
||||||
|
log.warn(`[self-healing] orphaned planning segment ${task.id} could not be finalized: errorType=${errorType}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (finalized === 0) {
|
if (finalized === 0) {
|
||||||
await this.store.recordRunAuditEvent?.({
|
const noActionMetadata = attempted === 0
|
||||||
agentId: "self-healing",
|
? { finalizedCount: 0, reason: "no-eligible-orphan" }
|
||||||
runId: generateSyntheticRunId("orphaned-planning-segment", "global"),
|
: failedAttempts === attempted
|
||||||
domain: "database",
|
? { finalizedCount: 0, reason: "all-attempts-failed", attemptedCount: attempted }
|
||||||
mutationType: "task:reconcile-orphaned-planning-segment-no-action",
|
: { finalizedCount: 0, reason: "no-finalization", attemptedCount: attempted, failedAttemptCount: failedAttempts };
|
||||||
target: "planning-segments",
|
try {
|
||||||
metadata: { finalizedCount: 0, reason: "no-eligible-orphan" },
|
await this.store.recordRunAuditEvent?.({
|
||||||
});
|
agentId: "self-healing",
|
||||||
|
runId: generateSyntheticRunId("orphaned-planning-segment", "global"),
|
||||||
|
domain: "database",
|
||||||
|
mutationType: "task:reconcile-orphaned-planning-segment-no-action",
|
||||||
|
target: "planning-segments",
|
||||||
|
metadata: noActionMetadata,
|
||||||
|
});
|
||||||
|
} catch (auditError: unknown) {
|
||||||
|
const errorType = auditError instanceof Error && auditError.name ? auditError.name : "unknown-error";
|
||||||
|
log.warn(`[self-healing] orphaned planning segment no-action audit could not be recorded: errorType=${errorType}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return finalized;
|
return finalized;
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
|
|||||||
Reference in New Issue
Block a user