feat(FN-1224): add peer gossip protocol for mesh network synchronization
- Add peer exchange types (MeshSyncState, PeerExchangeMessage, PeerGossipConfig) to core types - Add peer merge and sync methods to CentralCore (mergeProject, syncWithPeer, getMeshState) - Add mesh sync API endpoints (GET /api/mesh/state, POST /api/mesh/sync) - Implement PeerExchangeService background gossip engine for periodic peer synchronization - Add comprehensive tests for CentralCore mesh methods and PeerExchangeService - Update memory docs with peer gossip protocol documentation
This commit is contained in:
@@ -32,6 +32,7 @@ export { TokenCapDetector, type TokenCapCheckResult } from "./token-cap-detector
|
||||
export { SelfHealingManager, type SelfHealingOptions } from "./self-healing.js";
|
||||
export { ProjectManager } from "./project-manager.js";
|
||||
export { NodeHealthMonitor } from "./node-health-monitor.js";
|
||||
export { PeerExchangeService, type PeerExchangeServiceOptions, type SyncResult } from "./peer-exchange-service.js";
|
||||
export { RemoteNodeClient } from "./runtimes/remote-node-client.js";
|
||||
export { RemoteNodeRuntime, type RemoteNodeRuntimeConfig } from "./runtimes/remote-node-runtime.js";
|
||||
export { StepSessionExecutor } from "./step-session-executor.js";
|
||||
|
||||
@@ -88,3 +88,6 @@ export const remoteNodeLog = createLogger("remote-node");
|
||||
|
||||
/** Logger for periodic node health monitor subsystem. */
|
||||
export const nodeHealthMonitorLog = createLogger("node-health-monitor");
|
||||
|
||||
/** Logger for the peer exchange (gossip) subsystem. */
|
||||
export const peerExchangeLog = createLogger("peer-exchange");
|
||||
|
||||
278
packages/engine/src/peer-exchange-service.test.ts
Normal file
278
packages/engine/src/peer-exchange-service.test.ts
Normal file
@@ -0,0 +1,278 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import type { CentralCore, NodeConfig, PeerInfo } from "@fusion/core";
|
||||
import { PeerExchangeService } from "./peer-exchange-service.js";
|
||||
|
||||
function makeNode(overrides: Partial<NodeConfig> = {}): NodeConfig {
|
||||
return {
|
||||
id: "node_remote",
|
||||
name: "Remote Node",
|
||||
type: "remote",
|
||||
url: "https://remote.example.com",
|
||||
apiKey: undefined,
|
||||
status: "online",
|
||||
maxConcurrent: 2,
|
||||
createdAt: "2026-04-01T10:00:00.000Z",
|
||||
updatedAt: "2026-04-01T12:00:00.000Z",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makePeerInfo(overrides: Partial<PeerInfo> = {}): PeerInfo {
|
||||
return {
|
||||
nodeId: "node_peer",
|
||||
nodeName: "Peer Node",
|
||||
nodeUrl: "https://peer.example.com",
|
||||
status: "online",
|
||||
metrics: null,
|
||||
lastSeen: "2026-04-01T12:00:00.000Z",
|
||||
maxConcurrent: 2,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("PeerExchangeService", () => {
|
||||
let mockCentralCore: CentralCore;
|
||||
let mockFetch: ReturnType<typeof vi.fn>;
|
||||
let mockListNodes: ReturnType<typeof vi.fn>;
|
||||
let mockGetAllKnownPeerInfo: ReturnType<typeof vi.fn>;
|
||||
let mockMergePeers: ReturnType<typeof vi.fn>;
|
||||
let mockReportMeshState: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-04-01T12:00:00.000Z"));
|
||||
|
||||
// Create individual mocks
|
||||
mockListNodes = vi.fn();
|
||||
mockGetAllKnownPeerInfo = vi.fn();
|
||||
mockMergePeers = vi.fn();
|
||||
mockReportMeshState = vi.fn();
|
||||
|
||||
mockCentralCore = {
|
||||
listNodes: mockListNodes,
|
||||
getAllKnownPeerInfo: mockGetAllKnownPeerInfo,
|
||||
mergePeers: mockMergePeers,
|
||||
reportMeshState: mockReportMeshState,
|
||||
} as unknown as CentralCore;
|
||||
|
||||
mockFetch = vi.fn();
|
||||
globalThis.fetch = mockFetch;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("constructor", () => {
|
||||
it("should create service instance", () => {
|
||||
const service = new PeerExchangeService(mockCentralCore);
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
it("should accept custom sync interval", () => {
|
||||
const service = new PeerExchangeService(mockCentralCore, { syncIntervalMs: 30_000 });
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("syncWithNode()", () => {
|
||||
it("should send correct request body with auth header when apiKey is set", async () => {
|
||||
const node = makeNode({ apiKey: "secret-key" });
|
||||
mockListNodes.mockResolvedValue([
|
||||
makeNode({ id: "node_local", type: "local", status: "online" }),
|
||||
]);
|
||||
mockGetAllKnownPeerInfo.mockResolvedValue([
|
||||
makePeerInfo({ nodeId: "node_local", nodeName: "local" }),
|
||||
makePeerInfo({ nodeId: "node_remote" }),
|
||||
]);
|
||||
mockMergePeers.mockResolvedValue({ added: [], updated: [] });
|
||||
mockReportMeshState.mockResolvedValue({});
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
senderNodeId: "node_remote",
|
||||
senderNodeUrl: "https://remote.example.com",
|
||||
knownPeers: [],
|
||||
newPeers: [],
|
||||
timestamp: "2026-04-01T12:00:00.000Z",
|
||||
}),
|
||||
});
|
||||
|
||||
const service = new PeerExchangeService(mockCentralCore);
|
||||
const result = await service.syncWithNode(node);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
"https://remote.example.com/api/mesh/sync",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer secret-key",
|
||||
},
|
||||
body: expect.stringContaining('"senderNodeId":"node_local"'),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("should send request without auth header when no apiKey", async () => {
|
||||
const node = makeNode({ apiKey: undefined });
|
||||
mockListNodes.mockResolvedValue([
|
||||
makeNode({ id: "node_local", type: "local", status: "online" }),
|
||||
]);
|
||||
mockGetAllKnownPeerInfo.mockResolvedValue([]);
|
||||
mockMergePeers.mockResolvedValue({ added: [], updated: [] });
|
||||
mockReportMeshState.mockResolvedValue({});
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
senderNodeId: "node_remote",
|
||||
senderNodeUrl: "https://remote.example.com",
|
||||
knownPeers: [],
|
||||
newPeers: [],
|
||||
timestamp: "2026-04-01T12:00:00.000Z",
|
||||
}),
|
||||
});
|
||||
|
||||
const service = new PeerExchangeService(mockCentralCore);
|
||||
await service.syncWithNode(node);
|
||||
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
headers: expect.not.objectContaining({ "Authorization": expect.anything() }),
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("should merge response.knownPeers (not just newPeers)", async () => {
|
||||
const node = makeNode();
|
||||
mockListNodes.mockResolvedValue([
|
||||
makeNode({ id: "node_local", type: "local", status: "online" }),
|
||||
]);
|
||||
|
||||
const allPeersFromResponse = [
|
||||
makePeerInfo({ nodeId: "node_local", nodeName: "local", status: "online" }),
|
||||
makePeerInfo({ nodeId: "node_peer_a", status: "online" }),
|
||||
makePeerInfo({ nodeId: "node_peer_b", status: "offline" }),
|
||||
];
|
||||
|
||||
mockGetAllKnownPeerInfo.mockResolvedValue([
|
||||
makePeerInfo({ nodeId: "node_local", nodeName: "local" }),
|
||||
]);
|
||||
mockMergePeers.mockResolvedValue({ added: ["node_peer_a"], updated: ["node_peer_b"] });
|
||||
mockReportMeshState.mockResolvedValue({});
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
senderNodeId: "node_remote",
|
||||
senderNodeUrl: "https://remote.example.com",
|
||||
knownPeers: allPeersFromResponse,
|
||||
newPeers: [makePeerInfo({ nodeId: "node_peer_c" })],
|
||||
timestamp: "2026-04-01T12:00:00.000Z",
|
||||
}),
|
||||
});
|
||||
|
||||
const service = new PeerExchangeService(mockCentralCore);
|
||||
const result = await service.syncWithNode(node);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
// Verify merge was called with all knownPeers, not just newPeers
|
||||
expect(mockMergePeers).toHaveBeenCalledWith(allPeersFromResponse);
|
||||
});
|
||||
|
||||
it("should refresh local metrics before sending request", async () => {
|
||||
const node = makeNode();
|
||||
mockListNodes.mockResolvedValue([
|
||||
makeNode({ id: "node_local", type: "local", status: "online" }),
|
||||
]);
|
||||
mockGetAllKnownPeerInfo.mockResolvedValue([]);
|
||||
mockMergePeers.mockResolvedValue({ added: [], updated: [] });
|
||||
mockReportMeshState.mockResolvedValue({});
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
senderNodeId: "node_remote",
|
||||
senderNodeUrl: "https://remote.example.com",
|
||||
knownPeers: [],
|
||||
newPeers: [],
|
||||
timestamp: "2026-04-01T12:00:00.000Z",
|
||||
}),
|
||||
});
|
||||
|
||||
const service = new PeerExchangeService(mockCentralCore);
|
||||
await service.syncWithNode(node);
|
||||
|
||||
expect(mockReportMeshState).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should handle network error gracefully", async () => {
|
||||
const node = makeNode();
|
||||
mockListNodes.mockResolvedValue([
|
||||
makeNode({ id: "node_local", type: "local", status: "online" }),
|
||||
]);
|
||||
mockGetAllKnownPeerInfo.mockResolvedValue([]);
|
||||
mockReportMeshState.mockResolvedValue({});
|
||||
mockFetch.mockRejectedValue(new Error("Network error"));
|
||||
|
||||
const service = new PeerExchangeService(mockCentralCore);
|
||||
const result = await service.syncWithNode(node);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain("Network error");
|
||||
});
|
||||
|
||||
it("should handle non-2xx response", async () => {
|
||||
const node = makeNode();
|
||||
mockListNodes.mockResolvedValue([
|
||||
makeNode({ id: "node_local", type: "local", status: "online" }),
|
||||
]);
|
||||
mockGetAllKnownPeerInfo.mockResolvedValue([]);
|
||||
mockReportMeshState.mockResolvedValue({});
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: false,
|
||||
status: 401,
|
||||
statusText: "Unauthorized",
|
||||
});
|
||||
|
||||
const service = new PeerExchangeService(mockCentralCore);
|
||||
const result = await service.syncWithNode(node);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain("HTTP 401");
|
||||
});
|
||||
});
|
||||
|
||||
describe("triggerSync()", () => {
|
||||
it("should trigger sync when called", async () => {
|
||||
mockListNodes.mockResolvedValue([
|
||||
makeNode({ id: "node_local", type: "local", status: "online" }),
|
||||
makeNode({ id: "node_1", name: "Remote 1", status: "online", url: "https://remote1.example.com" }),
|
||||
]);
|
||||
mockGetAllKnownPeerInfo.mockResolvedValue([]);
|
||||
mockMergePeers.mockResolvedValue({ added: [], updated: [] });
|
||||
mockReportMeshState.mockResolvedValue({});
|
||||
mockFetch.mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
senderNodeId: "node_1",
|
||||
senderNodeUrl: "https://remote1.example.com",
|
||||
knownPeers: [],
|
||||
newPeers: [],
|
||||
timestamp: "2026-04-01T12:00:00.000Z",
|
||||
}),
|
||||
});
|
||||
|
||||
const service = new PeerExchangeService(mockCentralCore, { syncIntervalMs: 60_000 });
|
||||
const results = await service.triggerSync();
|
||||
|
||||
expect(mockFetch).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
267
packages/engine/src/peer-exchange-service.ts
Normal file
267
packages/engine/src/peer-exchange-service.ts
Normal file
@@ -0,0 +1,267 @@
|
||||
import type { CentralCore } from "@fusion/core";
|
||||
import type { NodeConfig, PeerInfo, PeerSyncRequest, PeerSyncResponse } from "@fusion/core";
|
||||
import { peerExchangeLog } from "./logger.js";
|
||||
|
||||
export interface PeerExchangeServiceOptions {
|
||||
/** Interval between peer sync cycles in milliseconds. Default: 60000 (1 minute) */
|
||||
syncIntervalMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of syncing with a single node.
|
||||
*/
|
||||
export interface SyncResult {
|
||||
/** Node ID that was synced with */
|
||||
nodeId: string;
|
||||
/** Whether the sync was successful */
|
||||
success: boolean;
|
||||
/** Number of new peers discovered */
|
||||
added: number;
|
||||
/** Number of peers updated */
|
||||
updated: number;
|
||||
/** Error message if sync failed */
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Background service that implements the peer gossip protocol.
|
||||
*
|
||||
* Periodically exchanges peer information with connected remote nodes
|
||||
* to keep the mesh state up-to-date across all nodes.
|
||||
*/
|
||||
export class PeerExchangeService {
|
||||
private centralCore: CentralCore;
|
||||
private syncIntervalMs: number;
|
||||
private interval: ReturnType<typeof setInterval> | null = null;
|
||||
private activeSync: Promise<void> | null = null;
|
||||
private stopped = false;
|
||||
|
||||
/**
|
||||
* Create a PeerExchangeService.
|
||||
*
|
||||
* @param centralCore - CentralCore instance for node registry access
|
||||
* @param options - Configuration options
|
||||
*/
|
||||
constructor(centralCore: CentralCore, options: PeerExchangeServiceOptions = {}) {
|
||||
this.centralCore = centralCore;
|
||||
this.syncIntervalMs = options.syncIntervalMs ?? 60_000; // 1 minute default
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the peer exchange service.
|
||||
* Begins periodic gossip with all online remote nodes.
|
||||
*/
|
||||
start(): void {
|
||||
if (this.stopped) {
|
||||
peerExchangeLog.warn("Cannot start - service has been stopped");
|
||||
return;
|
||||
}
|
||||
|
||||
// Get initial peer count for logging (async call)
|
||||
this.centralCore.listNodes().then((nodes) => {
|
||||
const onlineRemoteCount = nodes.filter(
|
||||
(n) => n.type === "remote" && n.status === "online" && n.url
|
||||
).length;
|
||||
|
||||
peerExchangeLog.log(`Starting peer exchange service (sync interval: ${this.syncIntervalMs}ms, ${onlineRemoteCount} online remote peers)`);
|
||||
}).catch((err) => {
|
||||
peerExchangeLog.warn(`Failed to get initial peer count: ${err}`);
|
||||
});
|
||||
|
||||
// Start periodic sync
|
||||
this.interval = setInterval(() => {
|
||||
void this.syncWithAllPeers();
|
||||
}, this.syncIntervalMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the peer exchange service.
|
||||
* Clears the sync interval and prevents further syncs.
|
||||
*/
|
||||
stop(): void {
|
||||
if (this.interval) {
|
||||
clearInterval(this.interval);
|
||||
this.interval = null;
|
||||
}
|
||||
|
||||
this.stopped = true;
|
||||
peerExchangeLog.log("Stopped peer exchange service");
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger an immediate sync with all peers, bypassing the interval.
|
||||
*
|
||||
* If a sync is already in progress, returns the in-progress sync.
|
||||
*
|
||||
* @returns Promise that resolves when the sync completes
|
||||
*/
|
||||
async triggerSync(): Promise<SyncResult[]> {
|
||||
return this.syncWithAllPeers();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync with all online remote nodes.
|
||||
*
|
||||
* Uses single-flight pattern to prevent overlapping syncs.
|
||||
* If a sync is already in progress, returns that sync's promise.
|
||||
*/
|
||||
async syncWithAllPeers(): Promise<SyncResult[]> {
|
||||
// Single-flight: if a sync is already running, return that
|
||||
if (this.activeSync) {
|
||||
peerExchangeLog.log("Sync already in progress, skipping");
|
||||
await this.activeSync;
|
||||
return [];
|
||||
}
|
||||
|
||||
this.activeSync = this.runSyncWithAllPeers();
|
||||
try {
|
||||
await this.activeSync;
|
||||
} finally {
|
||||
this.activeSync = null;
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
private async runSyncWithAllPeers(): Promise<void> {
|
||||
try {
|
||||
// Get all online remote nodes with URLs
|
||||
const nodes = await this.centralCore.listNodes();
|
||||
const onlineRemoteNodes = nodes.filter(
|
||||
(node) => node.type === "remote" && node.status === "online" && node.url
|
||||
);
|
||||
|
||||
if (onlineRemoteNodes.length === 0) {
|
||||
peerExchangeLog.log("No online remote nodes to sync with");
|
||||
return;
|
||||
}
|
||||
|
||||
peerExchangeLog.log(`Starting sync with ${onlineRemoteNodes.length} peers`);
|
||||
|
||||
// Sync with each node sequentially (not in parallel to avoid thundering herd)
|
||||
let totalAdded = 0;
|
||||
let totalUpdated = 0;
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const node of onlineRemoteNodes) {
|
||||
try {
|
||||
const result = await this.syncWithNode(node);
|
||||
totalAdded += result.added;
|
||||
totalUpdated += result.updated;
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
errors.push(`${node.name}: ${message}`);
|
||||
peerExchangeLog.warn(`Sync with ${node.name} failed: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Log summary
|
||||
if (errors.length > 0) {
|
||||
peerExchangeLog.log(
|
||||
`Sync complete: ${onlineRemoteNodes.length - errors.length} succeeded, ${errors.length} failed. ` +
|
||||
`${totalAdded} new peers discovered, ${totalUpdated} updated. Errors: ${errors.join("; ")}`
|
||||
);
|
||||
} else {
|
||||
peerExchangeLog.log(
|
||||
`Sync complete: ${onlineRemoteNodes.length} peers synced. ` +
|
||||
`${totalAdded} new peers discovered, ${totalUpdated} updated.`
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
peerExchangeLog.error("Unexpected error in sync loop:", error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync with a single remote node.
|
||||
*
|
||||
* Sends our known peers and merges the response.
|
||||
*
|
||||
* @param node - Remote node configuration
|
||||
* @returns Sync result with counts and any errors
|
||||
*/
|
||||
async syncWithNode(node: NodeConfig): Promise<SyncResult> {
|
||||
try {
|
||||
// Build the sync request
|
||||
// Refresh local metrics first to ensure freshness
|
||||
await this.centralCore.reportMeshState();
|
||||
|
||||
// Get local node info
|
||||
const nodes = await this.centralCore.listNodes();
|
||||
const localNode = nodes.find((n) => n.type === "local");
|
||||
if (!localNode) {
|
||||
return { nodeId: node.id, success: false, added: 0, updated: 0, error: "Local node not found" };
|
||||
}
|
||||
|
||||
// Get all known peers for the request
|
||||
const allKnownPeers = await this.centralCore.getAllKnownPeerInfo();
|
||||
|
||||
const request: PeerSyncRequest = {
|
||||
senderNodeId: localNode.id,
|
||||
senderNodeUrl: localNode.url || "",
|
||||
knownPeers: allKnownPeers,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
// Build headers
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
if (node.apiKey) {
|
||||
headers["Authorization"] = `Bearer ${node.apiKey}`;
|
||||
}
|
||||
|
||||
// Send the sync request with 10-second timeout
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 10_000);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${node.url}/api/mesh/sync`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(request),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
return {
|
||||
nodeId: node.id,
|
||||
success: false,
|
||||
added: 0,
|
||||
updated: 0,
|
||||
error: `HTTP ${response.status}: ${response.statusText}`,
|
||||
};
|
||||
}
|
||||
|
||||
const peerResponse: PeerSyncResponse = await response.json();
|
||||
|
||||
// Merge ALL known peers from the response (not just newPeers)
|
||||
// This ensures we get updates for existing peers too
|
||||
const mergeResult = await this.centralCore.mergePeers(peerResponse.knownPeers);
|
||||
|
||||
peerExchangeLog.log(
|
||||
`Synced with ${node.name}: ${mergeResult.added.length} new, ${mergeResult.updated.length} updated, ` +
|
||||
`${peerResponse.newPeers.length} new to sender`
|
||||
);
|
||||
|
||||
return {
|
||||
nodeId: node.id,
|
||||
success: true,
|
||||
added: mergeResult.added.length,
|
||||
updated: mergeResult.updated.length,
|
||||
};
|
||||
} catch (fetchError) {
|
||||
clearTimeout(timeoutId);
|
||||
throw fetchError;
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (message.includes("abort")) {
|
||||
return { nodeId: node.id, success: false, added: 0, updated: 0, error: "Timeout (10s)" };
|
||||
}
|
||||
return { nodeId: node.id, success: false, added: 0, updated: 0, error: message };
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user