feat(FN-4824): forward task assignment events across remote runtime
Fusion-Task-Id: FN-4824 Fusion-Task-Lineage: 65f53a33-0939-4b6f-b62b-4a255ad14d9e
This commit is contained in:
committed by
gsxdsm
parent
39a87a8925
commit
46a50c123f
@@ -325,6 +325,36 @@ describe("createSSE client cleanup", () => {
|
|||||||
expect(getActiveSSEConnections()).toBe(baseline);
|
expect(getActiveSSEConnections()).toBe(baseline);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("emits task:assigned SSE event from agent assignments", () => {
|
||||||
|
let onAssigned: ((agent: unknown, taskId: string) => void) | undefined;
|
||||||
|
const store = createMockStore();
|
||||||
|
const agentStore = {
|
||||||
|
on: vi.fn((event: string, handler: (agent: unknown, taskId: string) => void) => {
|
||||||
|
if (event === "agent:assigned") onAssigned = handler;
|
||||||
|
}),
|
||||||
|
off: vi.fn(),
|
||||||
|
};
|
||||||
|
const socket = new MockSocket();
|
||||||
|
const req = new EventEmitter() as Request & { query: Record<string, string>; socket: MockSocket };
|
||||||
|
req.query = { clientId: "assigned-client" };
|
||||||
|
req.socket = socket;
|
||||||
|
const res = new MockResponse(socket);
|
||||||
|
|
||||||
|
createSSE(store, undefined, undefined, undefined, undefined, agentStore as never)(
|
||||||
|
req,
|
||||||
|
res as unknown as Response
|
||||||
|
);
|
||||||
|
|
||||||
|
onAssigned?.({ id: "agent-1" }, "FN-1");
|
||||||
|
|
||||||
|
expect(res.write).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining("event: task:assigned")
|
||||||
|
);
|
||||||
|
expect(res.write).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining('"taskId":"FN-1"')
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("closes the connection when the outbound buffer exceeds the backpressure threshold", () => {
|
it("closes the connection when the outbound buffer exceeds the backpressure threshold", () => {
|
||||||
// Capture the task:created listener so we can fire a send after the
|
// Capture the task:created listener so we can fire a send after the
|
||||||
// socket buffer has been bloated past the threshold.
|
// socket buffer has been bloated past the threshold.
|
||||||
|
|||||||
@@ -389,6 +389,17 @@ export function createSSE(
|
|||||||
const onUpdated = (task: unknown) => {
|
const onUpdated = (task: unknown) => {
|
||||||
send(`event: task:updated\ndata: ${JSON.stringify(stripTaskListHeavyFields(task))}\n\n`);
|
send(`event: task:updated\ndata: ${JSON.stringify(stripTaskListHeavyFields(task))}\n\n`);
|
||||||
};
|
};
|
||||||
|
const onTaskAssigned = (agent: unknown, taskId: string) => {
|
||||||
|
const payload = {
|
||||||
|
taskId,
|
||||||
|
agentId:
|
||||||
|
agent && typeof agent === "object" && "id" in agent
|
||||||
|
? String((agent as { id: unknown }).id)
|
||||||
|
: "",
|
||||||
|
assignedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
send(`event: task:assigned\ndata: ${JSON.stringify(payload)}\n\n`);
|
||||||
|
};
|
||||||
const onDeleted = (task: unknown) => {
|
const onDeleted = (task: unknown) => {
|
||||||
send(`event: task:deleted\ndata: ${JSON.stringify(stripTaskListHeavyFields(task))}\n\n`);
|
send(`event: task:deleted\ndata: ${JSON.stringify(stripTaskListHeavyFields(task))}\n\n`);
|
||||||
};
|
};
|
||||||
@@ -713,6 +724,7 @@ export function createSSE(
|
|||||||
agentStore.off("agent:updated", onAgentUpdated);
|
agentStore.off("agent:updated", onAgentUpdated);
|
||||||
agentStore.off("agent:deleted", onAgentDeleted);
|
agentStore.off("agent:deleted", onAgentDeleted);
|
||||||
agentStore.off("agent:stateChanged", onAgentStateChanged);
|
agentStore.off("agent:stateChanged", onAgentStateChanged);
|
||||||
|
agentStore.off("agent:assigned", onTaskAssigned);
|
||||||
}
|
}
|
||||||
if (messageStore) {
|
if (messageStore) {
|
||||||
messageStore.off("message:sent", onMessageSent);
|
messageStore.off("message:sent", onMessageSent);
|
||||||
@@ -822,6 +834,7 @@ export function createSSE(
|
|||||||
agentStore.on("agent:updated", onAgentUpdated);
|
agentStore.on("agent:updated", onAgentUpdated);
|
||||||
agentStore.on("agent:deleted", onAgentDeleted);
|
agentStore.on("agent:deleted", onAgentDeleted);
|
||||||
agentStore.on("agent:stateChanged", onAgentStateChanged);
|
agentStore.on("agent:stateChanged", onAgentStateChanged);
|
||||||
|
agentStore.on("agent:assigned", onTaskAssigned);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (messageStore) {
|
if (messageStore) {
|
||||||
|
|||||||
@@ -67,6 +67,8 @@ export interface ProjectRuntimeEvents {
|
|||||||
"task:moved": [data: { task: Task; from: string; to: string }];
|
"task:moved": [data: { task: Task; from: string; to: string }];
|
||||||
/** Emitted when a task is updated */
|
/** Emitted when a task is updated */
|
||||||
"task:updated": [task: Task];
|
"task:updated": [task: Task];
|
||||||
|
/** Emitted when a cross-node assignment event is observed */
|
||||||
|
"task:assigned": [data: { taskId: string; agentId: string; assignedAt: string; source?: string }];
|
||||||
/** Emitted when an error occurs in the runtime */
|
/** Emitted when an error occurs in the runtime */
|
||||||
"error": [error: Error];
|
"error": [error: Error];
|
||||||
/** Emitted when the runtime health status changes */
|
/** Emitted when the runtime health status changes */
|
||||||
|
|||||||
@@ -123,6 +123,25 @@ describe("RemoteNodeClient", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("pollPendingAssignments() sends since cursor query", async () => {
|
||||||
|
const fetchMock = vi.fn().mockResolvedValue(
|
||||||
|
new Response(JSON.stringify([{ taskId: "KB-1", agentId: "agent-1", assignedAt: "2026-04-08T00:00:00.000Z" }]), {
|
||||||
|
status: 200,
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
})
|
||||||
|
);
|
||||||
|
globalThis.fetch = fetchMock as unknown as typeof fetch;
|
||||||
|
|
||||||
|
const client = new RemoteNodeClient({ baseUrl: BASE_URL, apiKey: API_KEY });
|
||||||
|
const result = await client.pollPendingAssignments({ since: "2026-04-08T00:00:00.000Z" });
|
||||||
|
|
||||||
|
expect(result).toEqual([{ taskId: "KB-1", agentId: "agent-1", assignedAt: "2026-04-08T00:00:00.000Z" }]);
|
||||||
|
expect(fetchMock).toHaveBeenCalledWith(
|
||||||
|
`${BASE_URL}/api/events/assignments?since=2026-04-08T00%3A00%3A00.000Z`,
|
||||||
|
expect.objectContaining({ method: "GET" })
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("executeTask() posts to execute endpoint", async () => {
|
it("executeTask() posts to execute endpoint", async () => {
|
||||||
const fetchMock = vi.fn().mockResolvedValue(
|
const fetchMock = vi.fn().mockResolvedValue(
|
||||||
new Response(JSON.stringify({ acknowledged: true }), {
|
new Response(JSON.stringify({ acknowledged: true }), {
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ const mockClientConstructor = vi.hoisted(() => vi.fn());
|
|||||||
const mockHealth = vi.hoisted(() => vi.fn());
|
const mockHealth = vi.hoisted(() => vi.fn());
|
||||||
const mockGetMetrics = vi.hoisted(() => vi.fn());
|
const mockGetMetrics = vi.hoisted(() => vi.fn());
|
||||||
const mockStreamEvents = vi.hoisted(() => vi.fn());
|
const mockStreamEvents = vi.hoisted(() => vi.fn());
|
||||||
|
const mockPollPendingAssignments = vi.hoisted(() => vi.fn());
|
||||||
|
|
||||||
vi.mock("../remote-node-client.js", () => ({
|
vi.mock("../remote-node-client.js", () => ({
|
||||||
RemoteNodeClient: vi.fn().mockImplementation((options: unknown) => {
|
RemoteNodeClient: vi.fn().mockImplementation((options: unknown) => {
|
||||||
@@ -15,6 +16,7 @@ vi.mock("../remote-node-client.js", () => ({
|
|||||||
health: mockHealth,
|
health: mockHealth,
|
||||||
getMetrics: mockGetMetrics,
|
getMetrics: mockGetMetrics,
|
||||||
streamEvents: mockStreamEvents,
|
streamEvents: mockStreamEvents,
|
||||||
|
pollPendingAssignments: mockPollPendingAssignments,
|
||||||
};
|
};
|
||||||
}),
|
}),
|
||||||
}));
|
}));
|
||||||
@@ -60,6 +62,7 @@ describe("RemoteNodeRuntime", () => {
|
|||||||
mockHealth.mockReset();
|
mockHealth.mockReset();
|
||||||
mockGetMetrics.mockReset();
|
mockGetMetrics.mockReset();
|
||||||
mockStreamEvents.mockReset();
|
mockStreamEvents.mockReset();
|
||||||
|
mockPollPendingAssignments.mockReset();
|
||||||
|
|
||||||
mockHealth.mockResolvedValue({ status: "ok", version: "1.0.0", uptime: 100 });
|
mockHealth.mockResolvedValue({ status: "ok", version: "1.0.0", uptime: 100 });
|
||||||
mockGetMetrics.mockResolvedValue({
|
mockGetMetrics.mockResolvedValue({
|
||||||
@@ -70,6 +73,7 @@ describe("RemoteNodeRuntime", () => {
|
|||||||
mockStreamEvents.mockImplementation(({ signal }: { signal?: AbortSignal } = {}) =>
|
mockStreamEvents.mockImplementation(({ signal }: { signal?: AbortSignal } = {}) =>
|
||||||
idleStream(signal)
|
idleStream(signal)
|
||||||
);
|
);
|
||||||
|
mockPollPendingAssignments.mockResolvedValue([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(async () => {
|
afterEach(async () => {
|
||||||
@@ -173,6 +177,7 @@ describe("RemoteNodeRuntime", () => {
|
|||||||
const createdHandler = vi.fn();
|
const createdHandler = vi.fn();
|
||||||
const movedHandler = vi.fn();
|
const movedHandler = vi.fn();
|
||||||
const updatedHandler = vi.fn();
|
const updatedHandler = vi.fn();
|
||||||
|
const assignedHandler = vi.fn();
|
||||||
const errorHandler = vi.fn();
|
const errorHandler = vi.fn();
|
||||||
|
|
||||||
mockStreamEvents.mockImplementation(({ signal }: { signal?: AbortSignal } = {}) =>
|
mockStreamEvents.mockImplementation(({ signal }: { signal?: AbortSignal } = {}) =>
|
||||||
@@ -193,6 +198,11 @@ describe("RemoteNodeRuntime", () => {
|
|||||||
payload: { id: "KB-1", column: "done" },
|
payload: { id: "KB-1", column: "done" },
|
||||||
timestamp: NOW,
|
timestamp: NOW,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
type: "task:assigned",
|
||||||
|
payload: { taskId: "KB-1", agentId: "agent-1", assignedAt: NOW },
|
||||||
|
timestamp: NOW,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
type: "error",
|
type: "error",
|
||||||
payload: { message: "boom" },
|
payload: { message: "boom" },
|
||||||
@@ -212,6 +222,7 @@ describe("RemoteNodeRuntime", () => {
|
|||||||
runtime.on("task:created", createdHandler);
|
runtime.on("task:created", createdHandler);
|
||||||
runtime.on("task:moved", movedHandler);
|
runtime.on("task:moved", movedHandler);
|
||||||
runtime.on("task:updated", updatedHandler);
|
runtime.on("task:updated", updatedHandler);
|
||||||
|
runtime.on("task:assigned", assignedHandler);
|
||||||
runtime.on("error", errorHandler);
|
runtime.on("error", errorHandler);
|
||||||
|
|
||||||
await runtime.start();
|
await runtime.start();
|
||||||
@@ -224,6 +235,12 @@ describe("RemoteNodeRuntime", () => {
|
|||||||
to: "in-progress",
|
to: "in-progress",
|
||||||
});
|
});
|
||||||
expect(updatedHandler).toHaveBeenCalledWith({ id: "KB-1", column: "done" });
|
expect(updatedHandler).toHaveBeenCalledWith({ id: "KB-1", column: "done" });
|
||||||
|
expect(assignedHandler).toHaveBeenCalledWith({
|
||||||
|
taskId: "KB-1",
|
||||||
|
agentId: "agent-1",
|
||||||
|
assignedAt: NOW,
|
||||||
|
source: "cross-node-push",
|
||||||
|
});
|
||||||
expect(errorHandler).toHaveBeenCalledWith(expect.any(Error));
|
expect(errorHandler).toHaveBeenCalledWith(expect.any(Error));
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -256,6 +273,40 @@ describe("RemoteNodeRuntime", () => {
|
|||||||
await runtime.stop();
|
await runtime.stop();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("polls pending assignments while reconnecting and deduplicates by assignedAt", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
|
||||||
|
mockStreamEvents.mockImplementation(async function* () {
|
||||||
|
throw new Error("stream down");
|
||||||
|
});
|
||||||
|
|
||||||
|
mockPollPendingAssignments
|
||||||
|
.mockResolvedValueOnce([{ taskId: "KB-2", agentId: "agent-2", assignedAt: NOW }])
|
||||||
|
.mockResolvedValue([{ taskId: "KB-2", agentId: "agent-2", assignedAt: NOW }]);
|
||||||
|
|
||||||
|
const runtime = new RemoteNodeRuntime({
|
||||||
|
nodeConfig: createNode(),
|
||||||
|
projectId: "proj_poll",
|
||||||
|
projectName: "Project Poll",
|
||||||
|
});
|
||||||
|
|
||||||
|
(runtime as unknown as { reconnectBaseDelayMs: number }).reconnectBaseDelayMs = 1;
|
||||||
|
(runtime as unknown as { maxReconnectDelayMs: number }).maxReconnectDelayMs = 1;
|
||||||
|
(runtime as unknown as { maxReconnectAttempts: number }).maxReconnectAttempts = 2;
|
||||||
|
|
||||||
|
const assignedHandler = vi.fn();
|
||||||
|
runtime.on("task:assigned", assignedHandler);
|
||||||
|
|
||||||
|
await runtime.start();
|
||||||
|
await vi.advanceTimersByTimeAsync(5);
|
||||||
|
|
||||||
|
await vi.waitFor(() => {
|
||||||
|
expect(assignedHandler).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
await runtime.stop();
|
||||||
|
});
|
||||||
|
|
||||||
it("validates remote node config on start", async () => {
|
it("validates remote node config on start", async () => {
|
||||||
const runtime = new RemoteNodeRuntime({
|
const runtime = new RemoteNodeRuntime({
|
||||||
nodeConfig: createNode({ type: "local", url: undefined, apiKey: undefined }),
|
nodeConfig: createNode({ type: "local", url: undefined, apiKey: undefined }),
|
||||||
|
|||||||
@@ -2,8 +2,25 @@ import type { Task, TaskCreateInput } from "@fusion/core";
|
|||||||
import type { RuntimeMetrics } from "../project-runtime.js";
|
import type { RuntimeMetrics } from "../project-runtime.js";
|
||||||
import { remoteNodeLog } from "../logger.js";
|
import { remoteNodeLog } from "../logger.js";
|
||||||
|
|
||||||
|
export type RemoteNodeEventType =
|
||||||
|
| "task:created"
|
||||||
|
| "task:moved"
|
||||||
|
| "task:updated"
|
||||||
|
| "task:assigned"
|
||||||
|
| "error"
|
||||||
|
| (string & {});
|
||||||
|
|
||||||
|
export interface RemoteNodeTaskAssignedPayload {
|
||||||
|
taskId: string;
|
||||||
|
agentId: string;
|
||||||
|
fromNodeId?: string;
|
||||||
|
toNodeId?: string;
|
||||||
|
leaseEpoch?: number;
|
||||||
|
assignedAt: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface RemoteNodeEvent {
|
export interface RemoteNodeEvent {
|
||||||
type: string;
|
type: RemoteNodeEventType;
|
||||||
payload: unknown;
|
payload: unknown;
|
||||||
timestamp: string;
|
timestamp: string;
|
||||||
}
|
}
|
||||||
@@ -95,6 +112,21 @@ export class RemoteNodeClient {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async pollPendingAssignments(options?: { since?: string }): Promise<RemoteNodeTaskAssignedPayload[]> {
|
||||||
|
const query = new URLSearchParams();
|
||||||
|
if (options?.since) {
|
||||||
|
query.set("since", options.since);
|
||||||
|
}
|
||||||
|
const path = query.size > 0
|
||||||
|
? `/api/events/assignments?${query.toString()}`
|
||||||
|
: "/api/events/assignments";
|
||||||
|
return this.withRetry(() =>
|
||||||
|
this.requestJson<RemoteNodeTaskAssignedPayload[]>(path, {
|
||||||
|
method: "GET",
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
async *streamEvents(options?: { signal?: AbortSignal }): AsyncIterable<RemoteNodeEvent> {
|
async *streamEvents(options?: { signal?: AbortSignal }): AsyncIterable<RemoteNodeEvent> {
|
||||||
const response = await this.withRetry(
|
const response = await this.withRetry(
|
||||||
() => this.openStream("/api/events/stream", options?.signal),
|
() => this.openStream("/api/events/stream", options?.signal),
|
||||||
|
|||||||
@@ -8,7 +8,11 @@ import type {
|
|||||||
RuntimeStatus,
|
RuntimeStatus,
|
||||||
} from "../project-runtime.js";
|
} from "../project-runtime.js";
|
||||||
import { remoteNodeLog } from "../logger.js";
|
import { remoteNodeLog } from "../logger.js";
|
||||||
import { RemoteNodeClient, type RemoteNodeEvent } from "./remote-node-client.js";
|
import {
|
||||||
|
RemoteNodeClient,
|
||||||
|
type RemoteNodeEvent,
|
||||||
|
type RemoteNodeTaskAssignedPayload,
|
||||||
|
} from "./remote-node-client.js";
|
||||||
|
|
||||||
export interface RemoteNodeRuntimeConfig {
|
export interface RemoteNodeRuntimeConfig {
|
||||||
nodeConfig: NodeConfig;
|
nodeConfig: NodeConfig;
|
||||||
@@ -32,6 +36,8 @@ export class RemoteNodeRuntime
|
|||||||
private reconnectBaseDelayMs = 5_000;
|
private reconnectBaseDelayMs = 5_000;
|
||||||
private maxReconnectDelayMs = 60_000;
|
private maxReconnectDelayMs = 60_000;
|
||||||
private maxReconnectAttempts = 10;
|
private maxReconnectAttempts = 10;
|
||||||
|
private lastAssignmentCursor: string | null = null;
|
||||||
|
private readonly seenAssignmentKeys = new Set<string>();
|
||||||
|
|
||||||
constructor(private config: RemoteNodeRuntimeConfig) {
|
constructor(private config: RemoteNodeRuntimeConfig) {
|
||||||
super();
|
super();
|
||||||
@@ -222,6 +228,8 @@ export class RemoteNodeRuntime
|
|||||||
`(attempt ${reconnectAttempts}/${this.maxReconnectAttempts})`
|
`(attempt ${reconnectAttempts}/${this.maxReconnectAttempts})`
|
||||||
);
|
);
|
||||||
|
|
||||||
|
await this.pollPendingAssignments("cross-node-poll");
|
||||||
|
|
||||||
await this.sleep(delayMs, signal);
|
await this.sleep(delayMs, signal);
|
||||||
if (signal.aborted) {
|
if (signal.aborted) {
|
||||||
return;
|
return;
|
||||||
@@ -253,6 +261,9 @@ export class RemoteNodeRuntime
|
|||||||
case "task:updated":
|
case "task:updated":
|
||||||
this.emit("task:updated", event.payload as Task);
|
this.emit("task:updated", event.payload as Task);
|
||||||
break;
|
break;
|
||||||
|
case "task:assigned":
|
||||||
|
this.emitAssignmentWake(event.payload as RemoteNodeTaskAssignedPayload, "cross-node-push");
|
||||||
|
break;
|
||||||
case "error": {
|
case "error": {
|
||||||
const payload = event.payload;
|
const payload = event.payload;
|
||||||
if (payload instanceof Error) {
|
if (payload instanceof Error) {
|
||||||
@@ -272,6 +283,46 @@ export class RemoteNodeRuntime
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async reconcileAssignments(assignments: RemoteNodeTaskAssignedPayload[]): Promise<void> {
|
||||||
|
for (const assignment of assignments) {
|
||||||
|
this.emitAssignmentWake(assignment, "cross-node-reconcile");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async pollPendingAssignments(source: "cross-node-poll" | "cross-node-reconcile"): Promise<void> {
|
||||||
|
try {
|
||||||
|
const assignments = await this.client.pollPendingAssignments({
|
||||||
|
since: this.lastAssignmentCursor ?? undefined,
|
||||||
|
});
|
||||||
|
for (const assignment of assignments) {
|
||||||
|
this.emitAssignmentWake(assignment, source);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
this.emitRuntimeError(this.toError(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private emitAssignmentWake(
|
||||||
|
assignment: RemoteNodeTaskAssignedPayload,
|
||||||
|
source: "cross-node-push" | "cross-node-poll" | "cross-node-reconcile"
|
||||||
|
): void {
|
||||||
|
const key = `${assignment.taskId}:${assignment.agentId}:${assignment.assignedAt}`;
|
||||||
|
if (this.seenAssignmentKeys.has(key)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.seenAssignmentKeys.add(key);
|
||||||
|
this.lastAssignmentCursor = assignment.assignedAt;
|
||||||
|
remoteNodeLog.log(
|
||||||
|
`[wake-trigger-diagnostics] source=${source} taskId=${assignment.taskId} agentId=${assignment.agentId}`
|
||||||
|
);
|
||||||
|
this.emit("task:assigned", {
|
||||||
|
taskId: assignment.taskId,
|
||||||
|
agentId: assignment.agentId,
|
||||||
|
assignedAt: assignment.assignedAt,
|
||||||
|
source,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private async refreshMetrics(): Promise<RuntimeMetrics> {
|
private async refreshMetrics(): Promise<RuntimeMetrics> {
|
||||||
try {
|
try {
|
||||||
const metrics = await this.client.getMetrics();
|
const metrics = await this.client.getMetrics();
|
||||||
|
|||||||
Reference in New Issue
Block a user