root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-2951): enforce unavailable-node scheduling and improve active agents panel

Browse Source 
- Enforce unavailable-node routing policy in the scheduler and wire policy integration through engine startup
- Expand scheduler and node-routing policy test coverage for unavailable-node handling and policy integration behavior
- Hoist the Active Agents panel above the main agents list and display next-heartbeat ETA details
- Fix Active Agents panel UI issues by resolving stuck "Connecting..." cards and adding spacing adjustments
- Add changesets covering Active Agents panel hoist/heartbeat ETA and connecting-state fixes

Fusion-Task-Id: FN-2951
This commit is contained in:
Fusion
2026-04-29 14:36:46 -07:00
committed by gsxdsm
parent d6c8073ad1
commit 7cb9d20125
5 changed files with 115 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
packages/engine/src/__tests__/node-routing-policy.test.ts
View File

@@ -1,78 +1,68 @@
import { describe, expect, it } from "vitest";
import type { EffectiveNode } from "../effective-node.js";
import type { NodeStatus, UnavailableNodePolicy } from "@fusion/core";
import { applyUnavailableNodePolicy } from "../node-routing-policy.js";
function effectiveNode(nodeId: string | undefined, source: EffectiveNode["source"]): EffectiveNode {
return { nodeId, source };
}
describe("applyUnavailableNodePolicy", () => {
it.each<[UnavailableNodePolicy | undefined, NodeStatus | undefined]>([
["block", "online"],
["block", "offline"],
["block", "error"],
["block", "connecting"],
["block", undefined],
["fallback-local", "online"],
["fallback-local", "offline"],
["fallback-local", "error"],
["fallback-local", "connecting"],
["fallback-local", undefined],
[undefined, "online"],
[undefined, "offline"],
[undefined, "error"],
[undefined, "connecting"],
[undefined, undefined],
])("always allows local execution (policy=%s, status=%s)", (policy, status) => {
const result = applyUnavailableNodePolicy(status, policy, true);
expect(result).toEqual({
allowed: true,
fallbackToLocal: false,
reason: "local-execution",
it("always allows local execution regardless of policy or health", () => {
const result = applyUnavailableNodePolicy({
effectiveNode: effectiveNode(undefined, "local"),
nodeHealth: "offline",
policy: "block",
});
expect(result).toEqual({ allowed: true, fallbackToLocal: false });
});
it.each<[
NodeStatus | undefined,
{ allowed: boolean; fallbackToLocal: boolean },
]>([
it.each<[NodeStatus | undefined, { allowed: boolean; fallbackToLocal?: boolean; reason?: string }]>([
["online", { allowed: true, fallbackToLocal: false }],
["offline", { allowed: false, fallbackToLocal: false }],
["error", { allowed: false, fallbackToLocal: false }],
["connecting", { allowed: false, fallbackToLocal: false }],
["offline", { allowed: false, reason: "Node node-1 is offline; policy is block" }],
["error", { allowed: false, reason: "Node node-1 is error; policy is block" }],
["connecting", { allowed: false, reason: "Node node-1 is connecting; policy is block" }],
[undefined, { allowed: true, fallbackToLocal: false }],
])("applies block policy for status=%s", (status, expected) => {
const result = applyUnavailableNodePolicy(status, "block", false);
expect(result.allowed).toBe(expected.allowed);
expect(result.fallbackToLocal).toBe(expected.fallbackToLocal);
});
it.each<[
NodeStatus | undefined,
{ allowed: boolean; fallbackToLocal: boolean },
]>([
["online", { allowed: true, fallbackToLocal: false }],
["offline", { allowed: true, fallbackToLocal: true }],
["error", { allowed: true, fallbackToLocal: true }],
["connecting", { allowed: true, fallbackToLocal: true }],
[undefined, { allowed: true, fallbackToLocal: false }],
])("applies fallback-local policy for status=%s", (status, expected) => {
const result = applyUnavailableNodePolicy(status, "fallback-local", false);
expect(result.allowed).toBe(expected.allowed);
expect(result.fallbackToLocal).toBe(expected.fallbackToLocal);
});
it("defaults undefined policy to block behavior", () => {
const result = applyUnavailableNodePolicy("offline", undefined, false);
expect(result).toEqual({
allowed: false,
fallbackToLocal: false,
reason: "blocked:offline",
])("applies block policy for status=%s", (nodeHealth, expected) => {
const result = applyUnavailableNodePolicy({
effectiveNode: effectiveNode("node-1", "task-override"),
nodeHealth,
policy: "block",
});
expect(result).toEqual(expected);
});
it("includes status in blocked and fallback reason strings", () => {
expect(applyUnavailableNodePolicy("offline", "block", false).reason).toBe("blocked:offline");
expect(applyUnavailableNodePolicy("error", "fallback-local", false).reason).toBe("fallback-local:error");
it.each<[NodeStatus | undefined, { allowed: boolean; fallbackToLocal: boolean; reason?: string }]>([
["online", { allowed: true, fallbackToLocal: false }],
["offline", { allowed: true, fallbackToLocal: true, reason: "Node node-1 is offline; falling back to local per policy" }],
["error", { allowed: true, fallbackToLocal: true, reason: "Node node-1 is error; falling back to local per policy" }],
["connecting", { allowed: true, fallbackToLocal: true, reason: "Node node-1 is connecting; falling back to local per policy" }],
[undefined, { allowed: true, fallbackToLocal: false }],
])("applies fallback-local policy for status=%s", (nodeHealth, expected) => {
const result = applyUnavailableNodePolicy({
effectiveNode: effectiveNode("node-1", "project-default"),
nodeHealth,
policy: "fallback-local",
});
expect(result).toEqual(expected);
});
it.each<[NodeStatus | undefined, { allowed: boolean; fallbackToLocal?: boolean; reason?: string }]>([
["online", { allowed: true, fallbackToLocal: false }],
["offline", { allowed: false, reason: "Node node-1 is offline; policy is block" }],
["error", { allowed: false, reason: "Node node-1 is error; policy is block" }],
["connecting", { allowed: false, reason: "Node node-1 is connecting; policy is block" }],
[undefined, { allowed: true, fallbackToLocal: false }],
])("treats undefined policy as block for status=%s", (nodeHealth, expected) => {
const result = applyUnavailableNodePolicy({
effectiveNode: effectiveNode("node-1", "task-override"),
nodeHealth,
policy: undefined as UnavailableNodePolicy | undefined,
});
expect(result).toEqual(expected);
});
});

43
packages/engine/src/__tests__/scheduler-node-routing.test.ts
View File

@@ -61,9 +61,9 @@ function createMockStore(task: Task, settings: Record<string, unknown> = {}): Ta
} as unknown as TaskStore;
}
function createMockHealthMonitor(statusMap: Record<string, NodeStatus | undefined>) {
function createMockNodeHealthMonitor(healthMap: Record<string, NodeStatus | undefined>) {
return {
getNodeHealth: vi.fn((id: string) => statusMap[id]),
getNodeHealth: vi.fn((id: string) => healthMap[id]),
} as unknown as import("../node-health-monitor.js").NodeHealthMonitor;
}
@@ -137,7 +137,7 @@ describe("Scheduler node routing", () => {
it("blocks dispatch when node is unhealthy and policy is block", async () => {
const task = createMockTask({ id: "FN-104", nodeId: "node-offline" });
const store = createMockStore(task, { maxConcurrent: 1, maxWorktrees: 1, unavailableNodePolicy: "block" });
const healthMonitor = createMockHealthMonitor({ "node-offline": "offline" });
const healthMonitor = createMockNodeHealthMonitor({ "node-offline": "offline" });
const scheduler = new Scheduler(store, { nodeHealthMonitor: healthMonitor });
(scheduler as unknown as { running: boolean }).running = true;
@@ -145,13 +145,14 @@ describe("Scheduler node routing", () => {
expect(store.updateTask).not.toHaveBeenCalled();
expect(store.moveTask).not.toHaveBeenCalled();
expect(store.logEntry).toHaveBeenCalledWith(task.id, "Routing blocked: node node-offline is offline, policy=block");
expect(store.logEntry).toHaveBeenCalledWith(task.id, "Node node-offline is offline; policy is block");
expect(schedulerLog.warn).toHaveBeenCalledWith("Task FN-104 blocked: Node node-offline is offline; policy is block");
});
it("deduplicates blocked log entries across polling cycles", async () => {
const task = createMockTask({ id: "FN-105", nodeId: "node-offline" });
const store = createMockStore(task, { maxConcurrent: 1, maxWorktrees: 1, unavailableNodePolicy: "block" });
const healthMonitor = createMockHealthMonitor({ "node-offline": "offline" });
const healthMonitor = createMockNodeHealthMonitor({ "node-offline": "offline" });
const scheduler = new Scheduler(store, { nodeHealthMonitor: healthMonitor });
(scheduler as unknown as { running: boolean }).running = true;
@@ -159,7 +160,7 @@ describe("Scheduler node routing", () => {
await scheduler.schedule();
const blockLogs = vi.mocked(store.logEntry).mock.calls.filter(([, message]) =>
String(message).includes("Routing blocked: node node-offline is offline, policy=block"),
String(message).includes("Node node-offline is offline; policy is block"),
);
expect(blockLogs).toHaveLength(1);
});
@@ -167,7 +168,7 @@ describe("Scheduler node routing", () => {
it("falls back to local dispatch when node is unhealthy and policy is fallback-local", async () => {
const task = createMockTask({ id: "FN-106", nodeId: "node-error" });
const store = createMockStore(task, { maxConcurrent: 1, maxWorktrees: 1, unavailableNodePolicy: "fallback-local" });
const healthMonitor = createMockHealthMonitor({ "node-error": "error" });
const healthMonitor = createMockNodeHealthMonitor({ "node-error": "error" });
const scheduler = new Scheduler(store, { nodeHealthMonitor: healthMonitor });
(scheduler as unknown as { running: boolean }).running = true;
@@ -177,13 +178,13 @@ describe("Scheduler node routing", () => {
effectiveNodeId: null,
effectiveNodeSource: "local",
}));
expect(store.logEntry).toHaveBeenCalledWith(task.id, "Routing fallback to local: node node-error is error, policy=fallback-local");
expect(store.logEntry).toHaveBeenCalledWith(task.id, "Node node-error is error; falling back to local per policy");
});
it("dispatches normally when node is online with block policy", async () => {
const task = createMockTask({ id: "FN-107", nodeId: "node-online" });
const store = createMockStore(task, { maxConcurrent: 1, maxWorktrees: 1, unavailableNodePolicy: "block" });
const healthMonitor = createMockHealthMonitor({ "node-online": "online" });
const healthMonitor = createMockNodeHealthMonitor({ "node-online": "online" });
const scheduler = new Scheduler(store, { nodeHealthMonitor: healthMonitor });
(scheduler as unknown as { running: boolean }).running = true;
@@ -198,7 +199,7 @@ describe("Scheduler node routing", () => {
it("dispatches normally when node health is unknown", async () => {
const task = createMockTask({ id: "FN-108", nodeId: "node-unknown" });
const store = createMockStore(task, { maxConcurrent: 1, maxWorktrees: 1, unavailableNodePolicy: "block" });
const healthMonitor = createMockHealthMonitor({ "node-unknown": undefined });
const healthMonitor = createMockNodeHealthMonitor({ "node-unknown": undefined });
const scheduler = new Scheduler(store, { nodeHealthMonitor: healthMonitor });
(scheduler as unknown as { running: boolean }).running = true;
@@ -210,25 +211,27 @@ describe("Scheduler node routing", () => {
}));
});
it("clears block and dispatches after node recovers", async () => {
it("clears block dedup after successful dispatch", async () => {
const task = createMockTask({ id: "FN-109", nodeId: "node-flaky" });
const store = createMockStore(task, { maxConcurrent: 1, maxWorktrees: 1, unavailableNodePolicy: "block" });
const getNodeHealth = vi.fn()
const getNodeHealth = vi
.fn()
.mockReturnValueOnce("offline" satisfies NodeStatus)
.mockReturnValueOnce("online" satisfies NodeStatus);
.mockReturnValueOnce("online" satisfies NodeStatus)
.mockReturnValueOnce("offline" satisfies NodeStatus);
const scheduler = new Scheduler(store, {
nodeHealthMonitor: { getNodeHealth } as unknown as import("../node-health-monitor.js").NodeHealthMonitor,
});
(scheduler as unknown as { running: boolean }).running = true;
await scheduler.schedule();
expect(store.updateTask).not.toHaveBeenCalled();
await scheduler.schedule();
expect(store.updateTask).toHaveBeenCalledWith(task.id, expect.objectContaining({
effectiveNodeId: "node-flaky",
effectiveNodeSource: "task-override",
}));
await scheduler.schedule();
const blockLogs = vi.mocked(store.logEntry).mock.calls.filter(([, message]) =>
String(message).includes("Node node-flaky is offline; policy is block"),
);
expect(blockLogs).toHaveLength(2);
});
it("skips policy check when no health monitor is provided", async () => {
@@ -248,7 +251,7 @@ describe("Scheduler node routing", () => {
it("never queries health for local tasks", async () => {
const task = createMockTask({ id: "FN-111", nodeId: undefined });
const store = createMockStore(task, { maxConcurrent: 1, maxWorktrees: 1 });
const healthMonitor = createMockHealthMonitor({});
const healthMonitor = createMockNodeHealthMonitor({});
const scheduler = new Scheduler(store, { nodeHealthMonitor: healthMonitor });
(scheduler as unknown as { running: boolean }).running = true;