feat(FN-4825): complete Step 3 — add scheduler unreachable-owner audits

Fusion-Task-Id: FN-4825
Fusion-Task-Lineage: dc4e633e-fe46-47a8-9ecc-f032073caee9
This commit is contained in:
Fusion (runfusion.ai)
2026-05-16 19:42:09 -07:00
committed by gsxdsm
parent ee121a9f79
commit 14cf263bc6
2 changed files with 185 additions and 0 deletions

View File

@@ -28,6 +28,7 @@ import type { MeshLeaseManager } from "./mesh-lease-manager.js";
import { selectPermanentAgentForTask } from "./agent-assignment.js";
import type { AutoClaimSnapshotManager } from "./auto-claim-snapshot.js";
import { StaleTaskReporter } from "./stale-task-reporter.js";
import { createRunAuditor, generateSyntheticRunId } from "./run-audit.js";
/**
* Check whether two sets of file scope paths overlap.
@@ -508,6 +509,43 @@ export class Scheduler {
await this.store.logEntry(taskId, reason);
}
private async emitNodeUnreachableRecoveryAudit(
task: Task,
metadata: {
ownerNodeId: string;
ownerNodeHealth: "offline" | "error";
handoffAction: "park" | "reassign-local" | "reassign-any";
handoffReason: string;
decisionPath: "scheduler-handoff-park" | "scheduler-handoff-reassign-local" | "scheduler-handoff-reassign-any";
newColumn: string;
dispatchNodeBefore?: string;
dispatchNodeAfter?: string;
},
): Promise<void> {
const auditor = createRunAuditor(this.store, {
runId: generateSyntheticRunId("scheduler", task.id),
agentId: "scheduler",
taskId: task.id,
taskLineageId: task.lineageId,
phase: "dispatch-owning-node-handoff",
});
try {
await auditor.database({
type: "task:auto-recover-node-unreachable",
target: task.id,
metadata: {
previousColumn: task.column,
...metadata,
},
});
} catch (error) {
schedulerLog.warn(
`Task ${task.id} failed to emit node-unreachable auto-recovery audit: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
private async emitHighOverlapFanoutWarnings(tasks: Task[]): Promise<void> {
const fanoutMap = computeBlockerFanoutMap(tasks, 3);
const seenBlockers = new Set<string>();
@@ -1015,6 +1053,16 @@ export class Scheduler {
if (handoffDecision.action === "park") {
if (!this.wasNodeBlocked.has(task.id)) {
this.wasNodeBlocked.add(task.id);
await this.emitNodeUnreachableRecoveryAudit(freshTask, {
ownerNodeId: freshTask.checkoutNodeId,
ownerNodeHealth,
handoffAction: handoffDecision.action,
handoffReason: handoffDecision.reason,
decisionPath: "scheduler-handoff-park",
newColumn: freshTask.column,
dispatchNodeBefore: effectiveNode.nodeId,
dispatchNodeAfter: effectiveNode.nodeId,
});
const reason = `Owning-node handoff parked dispatch: ${handoffDecision.reason}`;
schedulerLog.log(`Task ${task.id} dispatch blocked — ${reason}`);
await this.store.logEntry(task.id, reason);
@@ -1023,9 +1071,23 @@ export class Scheduler {
}
await this.store.logEntry(task.id, `Owning-node handoff applied: ${handoffDecision.reason}`);
const dispatchNodeBefore = effectiveNode.nodeId;
if (handoffDecision.action === "reassign-local") {
effectiveNode = { nodeId: undefined, source: "local" };
}
await this.emitNodeUnreachableRecoveryAudit(freshTask, {
ownerNodeId: freshTask.checkoutNodeId,
ownerNodeHealth,
handoffAction: handoffDecision.action,
handoffReason: handoffDecision.reason,
decisionPath:
handoffDecision.action === "reassign-local"
? "scheduler-handoff-reassign-local"
: "scheduler-handoff-reassign-any",
newColumn: freshTask.column,
dispatchNodeBefore,
dispatchNodeAfter: effectiveNode.nodeId,
});
}
}