test(FN-4823): expand central-claim recovery coverage
Fusion-Task-Id: FN-4823 Fusion-Task-Lineage: 0034c04f-df82-4a1b-9f25-b93643a2c157
This commit is contained in:
Fusion (runfusion.ai)
committed by
gsxdsm
gsxdsm
parent
bbcc0a269c
commit
a9dfb17b3d
@@ -248,6 +248,74 @@ describe("MeshLeaseManager", () => {
|
||||
expect(recordRunAuditEvent.mock.calls.some((call) => call[0].mutationType === "task:auto-recover-lease-foreign-owner")).toBe(true);
|
||||
});
|
||||
|
||||
it("treats central not_found as already-healed and still clears local lease", async () => {
|
||||
const currentTask = task({ column: "in-progress" });
|
||||
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
|
||||
const centralClaimStore: CentralClaimStore = {
|
||||
tryClaimTask: vi.fn() as any,
|
||||
renewTaskClaim: vi.fn() as any,
|
||||
getTaskClaim: vi.fn().mockReturnValue(null),
|
||||
releaseTaskClaim: vi.fn().mockReturnValue({ ok: false, reason: "not_found", current: null }),
|
||||
};
|
||||
const taskStore = {
|
||||
getTask: vi.fn().mockResolvedValue(currentTask),
|
||||
updateTask: vi.fn().mockResolvedValue(currentTask),
|
||||
moveTask: vi.fn().mockResolvedValue(currentTask),
|
||||
logEntry: vi.fn().mockResolvedValue(undefined),
|
||||
recordRunAuditEvent,
|
||||
} as unknown as TaskStore;
|
||||
|
||||
const manager = new MeshLeaseManager({ taskStore, centralClaimStore, projectId: "project-1" });
|
||||
const ok = await manager.recoverAbandonedLease("FN-1", "stale-heartbeat");
|
||||
expect(ok).toBe(true);
|
||||
expect(taskStore.updateTask).toHaveBeenCalled();
|
||||
expect(recordRunAuditEvent.mock.calls.some((call) => call[0].mutationType === "task:auto-recover-lease-already-healed")).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false when central claim release remains unavailable after retry", async () => {
|
||||
const currentTask = task();
|
||||
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
|
||||
const centralClaimStore: CentralClaimStore = {
|
||||
tryClaimTask: vi.fn() as any,
|
||||
renewTaskClaim: vi.fn() as any,
|
||||
getTaskClaim: vi.fn().mockReturnValue(null),
|
||||
releaseTaskClaim: vi.fn().mockImplementation(() => {
|
||||
throw new Error("busy");
|
||||
}),
|
||||
};
|
||||
const taskStore = {
|
||||
getTask: vi.fn().mockResolvedValue(currentTask),
|
||||
updateTask: vi.fn().mockResolvedValue(currentTask),
|
||||
moveTask: vi.fn().mockResolvedValue(currentTask),
|
||||
logEntry: vi.fn().mockResolvedValue(undefined),
|
||||
recordRunAuditEvent,
|
||||
} as unknown as TaskStore;
|
||||
|
||||
const manager = new MeshLeaseManager({ taskStore, centralClaimStore, projectId: "project-1" });
|
||||
const ok = await manager.recoverAbandonedLease("FN-1", "stale-heartbeat");
|
||||
expect(ok).toBe(false);
|
||||
expect(taskStore.updateTask).not.toHaveBeenCalled();
|
||||
expect(recordRunAuditEvent.mock.calls.some((call) => call[0].mutationType === "task:auto-recover-lease-central-unavailable")).toBe(true);
|
||||
});
|
||||
|
||||
it("single-node fallback keeps recovery local-only", async () => {
|
||||
const currentTask = task({ currentStep: 2, steps: [{ status: "done" } as any] });
|
||||
const recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
|
||||
const taskStore = {
|
||||
getTask: vi.fn().mockResolvedValue(currentTask),
|
||||
updateTask: vi.fn().mockResolvedValue(currentTask),
|
||||
moveTask: vi.fn().mockResolvedValue(currentTask),
|
||||
logEntry: vi.fn().mockResolvedValue(undefined),
|
||||
recordRunAuditEvent,
|
||||
} as unknown as TaskStore;
|
||||
|
||||
const manager = new MeshLeaseManager({ taskStore });
|
||||
const ok = await manager.recoverAbandonedLease("FN-1", "stale-heartbeat", { preserveProgress: true });
|
||||
expect(ok).toBe(true);
|
||||
expect(taskStore.moveTask).toHaveBeenCalledWith("FN-1", "todo", { preserveProgress: true });
|
||||
expect(recordRunAuditEvent.mock.calls.some((call) => String(call[0].mutationType).includes("task:auto-recover-lease-"))).toBe(false);
|
||||
});
|
||||
|
||||
it("reconcileLeaseRow clears local owner when central claim is already gone", async () => {
|
||||
const currentTask = task();
|
||||
const centralClaimStore: CentralClaimStore = {
|
||||
|
||||
@@ -25,6 +25,57 @@ function makeTask(overrides: Partial<Task> = {}): Task {
|
||||
}
|
||||
|
||||
describe("reliability interactions: lease recovery central claim", () => {
|
||||
it("two-node contention yields one winner and one foreign-owner rejection", async () => {
|
||||
const taskA = makeTask({ id: "FN-CAS" });
|
||||
const taskB = makeTask({ id: "FN-CAS" });
|
||||
const auditA = vi.fn().mockResolvedValue(undefined);
|
||||
const auditB = vi.fn().mockResolvedValue(undefined);
|
||||
let first = true;
|
||||
const centralClaimStore: CentralClaimStore = {
|
||||
tryClaimTask: vi.fn() as any,
|
||||
renewTaskClaim: vi.fn() as any,
|
||||
getTaskClaim: vi.fn().mockReturnValue(null),
|
||||
releaseTaskClaim: vi.fn().mockImplementation(() => {
|
||||
if (first) {
|
||||
first = false;
|
||||
return { ok: true };
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
reason: "not_owner",
|
||||
current: {
|
||||
projectId: "project-1",
|
||||
taskId: "FN-CAS",
|
||||
ownerNodeId: "node-z",
|
||||
ownerAgentId: "agent-z",
|
||||
ownerRunId: null,
|
||||
leaseEpoch: 2,
|
||||
leaseRenewedAt: "2026-05-01T00:00:00.000Z",
|
||||
createdAt: "2026-05-01T00:00:00.000Z",
|
||||
updatedAt: "2026-05-01T00:00:00.000Z",
|
||||
},
|
||||
};
|
||||
}),
|
||||
};
|
||||
|
||||
const baseStore = (task: Task, recordRunAuditEvent: any) => ({
|
||||
getTask: vi.fn().mockResolvedValue(task),
|
||||
updateTask: vi.fn().mockResolvedValue(task),
|
||||
moveTask: vi.fn().mockResolvedValue(task),
|
||||
logEntry: vi.fn().mockResolvedValue(undefined),
|
||||
recordRunAuditEvent,
|
||||
}) as unknown as TaskStore;
|
||||
|
||||
const managerA = new MeshLeaseManager({ taskStore: baseStore(taskA, auditA), centralClaimStore, projectId: "project-1" });
|
||||
const managerB = new MeshLeaseManager({ taskStore: baseStore(taskB, auditB), centralClaimStore, projectId: "project-1" });
|
||||
|
||||
const [firstOk, secondOk] = await Promise.all([
|
||||
managerA.recoverAbandonedLease("FN-CAS", "stale-heartbeat"),
|
||||
managerB.recoverAbandonedLease("FN-CAS", "stale-heartbeat"),
|
||||
]);
|
||||
expect([firstOk, secondOk].sort()).toEqual([false, true]);
|
||||
expect(auditB.mock.calls.concat(auditA.mock.calls).some((call) => call[0].mutationType === "task:auto-recover-lease-foreign-owner")).toBe(true);
|
||||
});
|
||||
it("scheduler invokes reconcile once when lease recovery returns false", async () => {
|
||||
const task = makeTask();
|
||||
const store = {
|
||||
@@ -82,6 +133,41 @@ describe("reliability interactions: lease recovery central claim", () => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("rejects stale owner renewal attempts after recovery", async () => {
|
||||
const centralClaimStore: CentralClaimStore = {
|
||||
tryClaimTask: vi.fn() as any,
|
||||
releaseTaskClaim: vi.fn().mockReturnValue({ ok: true }),
|
||||
getTaskClaim: vi.fn().mockReturnValue(null),
|
||||
renewTaskClaim: vi.fn().mockReturnValue({
|
||||
ok: false,
|
||||
reason: "not_found",
|
||||
current: null,
|
||||
}),
|
||||
};
|
||||
const current = makeTask({ id: "FN-RENEW" });
|
||||
const taskStore = {
|
||||
getTask: vi.fn().mockResolvedValue(current),
|
||||
updateTask: vi.fn().mockResolvedValue(current),
|
||||
moveTask: vi.fn().mockResolvedValue(current),
|
||||
logEntry: vi.fn().mockResolvedValue(undefined),
|
||||
recordRunAuditEvent: vi.fn().mockResolvedValue(undefined),
|
||||
} as unknown as TaskStore;
|
||||
|
||||
const manager = new MeshLeaseManager({ taskStore, centralClaimStore, projectId: "project-1" });
|
||||
await expect(manager.recoverAbandonedLease("FN-RENEW", "stale-heartbeat")).resolves.toBe(true);
|
||||
|
||||
const renew = centralClaimStore.renewTaskClaim({
|
||||
projectId: "project-1",
|
||||
taskId: "FN-RENEW",
|
||||
nodeId: "node-a",
|
||||
agentId: "agent-1",
|
||||
runId: null,
|
||||
renewedAt: new Date().toISOString(),
|
||||
expectedEpoch: 1,
|
||||
});
|
||||
expect(renew.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("reconciles split-brain state after central release succeeds but local update initially fails", async () => {
|
||||
const current = makeTask({ column: "in-progress" });
|
||||
const updateTask = vi
|
||||
|
||||
Reference in New Issue
Block a user