feat(FN-4887): complete Step 2-3 — add foreign-only recovery sweep and shared handler

Fusion-Task-Id: FN-4887
Fusion-Task-Lineage: 559690d8-cea6-4323-a522-9ebb6aa25731
This commit is contained in:
Fusion (runfusion.ai)
2026-05-17 16:42:21 -07:00
committed by gsxdsm
parent 91df093502
commit b5ed3d0c9c
7 changed files with 380 additions and 14 deletions

View File

@@ -57,6 +57,29 @@ describe("reliability interaction: contamination auto-recovery precedence", () =
expect(issueRetry).toHaveBeenCalledOnce();
});
it("foreign-only no-own-work routes to retry, not pause", async () => {
const issueRetry = vi.fn(async () => {});
const dispatcher = new AutoRecoveryDispatcher({
taskStore: {} as never,
auditEmitter: { database: vi.fn(async () => {}), git: vi.fn(), filesystem: vi.fn(), sandbox: vi.fn() },
handlers: { issueRetry },
});
const decision = await dispatcher.dispatch({
class: "branch-cross-contamination",
taskId: "FN-1",
pausedReason: "branch-cross-contamination",
evidence: { ownCommits: 0, foreignAttributedCommits: 3, recoveryKind: "foreign-only" },
}, {
task: baseTask,
retryCount: 0,
settings: { mode: "programmatic", maxRetries: 2 },
});
expect(decision.action).toBe("retry");
expect(issueRetry).toHaveBeenCalledOnce();
});
it("mode off and destructive ambiguity preserve pause", () => {
const dispatcher = new AutoRecoveryDispatcher({
taskStore: {} as never,

View File

@@ -0,0 +1,75 @@
import { describe, expect, it, vi, beforeEach } from "vitest";
import type { Task } from "@fusion/core";
const mocked = vi.hoisted(() => ({
classifyForeignOnlyContamination: vi.fn(),
recoverForeignOnlyContamination: vi.fn(),
}));
vi.mock("../branch-conflicts.js", async () => {
const actual = await vi.importActual<typeof import("../branch-conflicts.js")>("../branch-conflicts.js");
return {
...actual,
classifyForeignOnlyContamination: mocked.classifyForeignOnlyContamination,
};
});
vi.mock("../recovery/foreign-only-contamination.js", () => ({
recoverForeignOnlyContamination: mocked.recoverForeignOnlyContamination,
}));
import { SelfHealingManager } from "../self-healing.js";
function mkTask(overrides: Partial<Task> = {}): Task {
return {
id: "FN-1",
column: "in-review",
branch: "fusion/fn-1",
worktree: "/tmp/wt",
baseCommitSha: "main",
paused: false,
userPaused: false,
mergeDetails: null,
steps: [],
...overrides,
} as Task;
}
describe("SelfHealingManager.recoverForeignOnlyContaminatedInReviewTasks", () => {
const store = {
getSettings: vi.fn(async () => ({ globalPause: false, enginePaused: false })),
listTasks: vi.fn(),
logEntry: vi.fn(async () => {}),
on: vi.fn(),
} as any;
beforeEach(() => {
vi.clearAllMocks();
});
it("recovers foreign-only in-review candidates", async () => {
store.listTasks.mockImplementation(async ({ column }: { column: string }) => column === "in-review" ? [mkTask()] : []);
mocked.classifyForeignOnlyContamination.mockResolvedValue({ kind: "foreign-only-no-own-work" });
mocked.recoverForeignOnlyContamination.mockResolvedValue({ recovered: true, subtype: "reanchor" });
const manager = new SelfHealingManager(store, { rootDir: process.cwd() });
const recovered = await manager.recoverForeignOnlyContaminatedInReviewTasks();
expect(recovered).toBe(1);
expect(mocked.recoverForeignOnlyContamination).toHaveBeenCalledOnce();
});
it("skips ambiguous and user-paused tasks", async () => {
store.listTasks.mockImplementation(async ({ column }: { column: string }) => {
if (column === "in-review") return [mkTask({ id: "FN-2", userPaused: true }), mkTask({ id: "FN-3" })];
return [];
});
mocked.classifyForeignOnlyContamination.mockResolvedValue({ kind: "ambiguous" });
const manager = new SelfHealingManager(store, { rootDir: process.cwd() });
const recovered = await manager.recoverForeignOnlyContaminatedInReviewTasks();
expect(recovered).toBe(0);
expect(mocked.recoverForeignOnlyContamination).not.toHaveBeenCalled();
});
});

View File

@@ -1,6 +1,8 @@
import type { TaskStore } from "@fusion/core";
import { classifyForeignOnlyContamination } from "../branch-conflicts.js";
import type { AutoRecoveryContext, AutoRecoveryDecision, AutoRecoveryFailure, AutoRecoveryHandlers } from "../auto-recovery.js";
import { createLogger, type Logger } from "../logger.js";
import { recoverForeignOnlyContamination } from "../recovery/foreign-only-contamination.js";
import type { RunAuditor } from "../run-audit.js";
const baseLog = createLogger("auto-recovery:contamination");
@@ -47,18 +49,45 @@ export class ContaminationAutoRecoveryHandler implements Pick<AutoRecoveryHandle
return;
}
await this.deps.taskStore.moveTask(task.id, "todo", {
moveSource: "engine",
preserveResumeState: true,
preserveProgress: true,
preserveWorktree: true,
});
let recoveryKind: "default" | "foreign-only" = "default";
let subtype: "reanchor" | "branch-discard" | undefined;
await this.deps.taskStore.updateTask(task.id, {
paused: false,
pausedReason: null,
error: null,
});
if (ownCommits === 0 && foreignAttributedCommits > 0 && task.branch && task.worktree) {
const baseSha = task.baseCommitSha ?? task.baseBranch ?? task.executionStartBranch ?? "main";
const classification = await classifyForeignOnlyContamination({
repoDir: this.deps.repoDir,
branchName: task.branch,
baseSha,
taskId: task.id,
}).catch(() => null);
if (classification && (classification.kind === "foreign-only-no-own-work" || classification.kind === "foreign-only-already-upstream")) {
const recovered = await recoverForeignOnlyContamination(task, {
repoDir: this.deps.repoDir,
taskStore: this.deps.taskStore,
runAudit: this.deps.runAudit,
});
if (recovered.recovered) {
recoveryKind = "foreign-only";
subtype = recovered.subtype;
}
}
}
if (recoveryKind === "default") {
await this.deps.taskStore.moveTask(task.id, "todo", {
moveSource: "engine",
preserveResumeState: true,
preserveProgress: true,
preserveWorktree: true,
});
await this.deps.taskStore.updateTask(task.id, {
paused: false,
pausedReason: null,
error: null,
});
}
await this.deps.runAudit.database({
type: "contamination:retry-issued",
@@ -69,6 +98,8 @@ export class ContaminationAutoRecoveryHandler implements Pick<AutoRecoveryHandle
ownCommits,
foreignAttributedCommits,
retryCount: ctx.retryCount,
recoveryKind,
subtype,
},
});
}

View File

@@ -58,6 +58,7 @@ import {
autoRecoverCrossContamination,
classifyBootstrapMisbinding,
classifyForeignCommits,
classifyForeignOnlyContamination,
isBranchConflictError,
reanchorBranchToBase,
inspectBranchConflict,
@@ -4638,14 +4639,25 @@ export class TaskExecutor {
}
const autoRecoveryDispatcher = this.getAutoRecoveryDispatcher(audit);
const ownCommits = err.foreignCommits.filter((commit) => commit.foreignTaskId === task.id).length;
const foreignAttributedCommits = err.foreignCommits.filter((commit) => commit.foreignTaskId !== task.id).length;
const foreignOnlyClassification = (task.branch && task.baseCommitSha)
? await classifyForeignOnlyContamination({
repoDir: this.rootDir,
branchName: task.branch,
baseSha: task.baseCommitSha,
taskId: task.id,
}).catch(() => null)
: null;
const decision = await autoRecoveryDispatcher.dispatch({
class: "branch-cross-contamination",
taskId: task.id,
runId: this.currentRunContext?.runId,
pausedReason: "branch-cross-contamination",
evidence: {
ownCommits: err.foreignCommits.filter((commit) => commit.foreignTaskId === task.id).length,
foreignAttributedCommits: err.foreignCommits.filter((commit) => commit.foreignTaskId !== task.id).length,
ownCommits,
foreignAttributedCommits,
foreignOnlyKind: foreignOnlyClassification?.kind,
},
underlyingError: err,
}, {

View File

@@ -0,0 +1,136 @@
import { exec } from "node:child_process";
import { existsSync } from "node:fs";
import { promisify } from "node:util";
import type { Task, TaskStore } from "@fusion/core";
import { activeSessionRegistry } from "../active-session-registry.js";
import {
classifyForeignOnlyContamination,
reanchorBranchToBase,
type ClassifyForeignOnlyContaminationResult,
} from "../branch-conflicts.js";
import type { RunAuditor } from "../run-audit.js";
import { isUsableTaskWorktree } from "../worktree-pool.js";
const execAsync = promisify(exec);
const GIT_TIMEOUT_MS = 30_000;
const GIT_MAX_BUFFER = 10 * 1024 * 1024;
function quote(value: string): string {
return `'${value.replace(/'/g, `'\\''`)}'`;
}
export interface RecoverForeignOnlyContaminationDeps {
repoDir: string;
taskStore: TaskStore;
runAudit: RunAuditor;
}
export interface RecoverForeignOnlyContaminationResult {
recovered: boolean;
subtype?: "reanchor" | "branch-discard";
reason?: string;
}
export async function recoverForeignOnlyContamination(
task: Task,
deps: RecoverForeignOnlyContaminationDeps,
): Promise<RecoverForeignOnlyContaminationResult> {
if (!task.branch || !task.worktree) return { recovered: false, reason: "missing-branch-or-worktree" };
const baseSha = task.baseCommitSha ?? task.baseBranch ?? task.executionStartBranch ?? "main";
if (!baseSha) {
await deps.runAudit.database({
type: "task:auto-recover-foreign-only-contamination-skipped",
target: task.id,
metadata: { reason: "baseSha-unresolved" },
});
return { recovered: false, reason: "baseSha-unresolved" };
}
const classification = await classifyForeignOnlyContamination({
repoDir: deps.repoDir,
branchName: task.branch,
baseSha,
taskId: task.id,
});
if (classification.kind !== "foreign-only-no-own-work" && classification.kind !== "foreign-only-already-upstream") {
await deps.runAudit.database({
type: "task:auto-recover-foreign-only-contamination-skipped",
target: task.id,
metadata: { reason: "ambiguous", kind: classification.kind },
});
return { recovered: false, reason: "ambiguous" };
}
if (await isUsableTaskWorktree(task.worktree)) {
await reanchorBranchToBase({
repoDir: deps.repoDir,
worktreePath: task.worktree,
branchName: task.branch,
baseSha,
taskId: task.id,
});
await deps.taskStore.moveTask(task.id, "todo", {
moveSource: "engine",
preserveResumeState: true,
preserveProgress: true,
preserveWorktree: true,
});
await deps.taskStore.updateTask(task.id, {
recoveryRetryCount: 0,
nextRecoveryAt: null,
error: null,
paused: false,
pausedReason: null,
});
await deps.runAudit.database({
type: "task:auto-recover-foreign-only-contamination",
target: task.id,
metadata: { subtype: "reanchor", kind: classification.kind, baseSha },
});
return { recovered: true, subtype: "reanchor" };
}
if (activeSessionRegistry.isPathActive(task.worktree)) {
await deps.runAudit.database({
type: "task:auto-recover-foreign-only-contamination-skipped",
target: task.id,
metadata: { reason: "active-session", kind: classification.kind },
});
return { recovered: false, reason: "active-session" };
}
await execAsync("git worktree prune", { cwd: deps.repoDir, timeout: GIT_TIMEOUT_MS, maxBuffer: GIT_MAX_BUFFER }).catch(() => undefined);
await execAsync(`git branch -D ${quote(task.branch)}`, { cwd: deps.repoDir, timeout: GIT_TIMEOUT_MS, maxBuffer: GIT_MAX_BUFFER }).catch(() => undefined);
await deps.taskStore.moveTask(task.id, "todo", {
moveSource: "engine",
preserveResumeState: true,
preserveProgress: true,
preserveWorktree: false,
});
await deps.taskStore.updateTask(task.id, {
recoveryRetryCount: 0,
nextRecoveryAt: null,
error: null,
paused: false,
pausedReason: null,
worktree: null,
branch: null,
baseCommitSha: null,
modifiedFiles: [],
});
await deps.runAudit.database({
type: "task:auto-recover-foreign-only-contamination",
target: task.id,
metadata: {
subtype: "branch-discard",
kind: classification.kind,
baseSha,
worktreePresent: existsSync(task.worktree),
},
});
return { recovered: true, subtype: "branch-discard" };
}

View File

@@ -162,6 +162,8 @@ export type DatabaseMutationType =
| "task:auto-recover-already-merged"
| "task:auto-recover-finalize-already-on-main"
| "task:auto-recover-branch-misbound"
| "task:auto-recover-foreign-only-contamination"
| "task:auto-recover-foreign-only-contamination-skipped"
| "task:auto-recover-node-unreachable"
| "task:auto-recover-worktree-metadata-rebound"
| "task:auto-recover-worktree-metadata-cleared"

View File

@@ -37,13 +37,14 @@ import {
isRecoverableMissingWorktreeReviewFailureWithProgress,
} from "./restart-recovery-coordinator.js";
import { classifyError, extractMissingModulePath, isOperatorActionableAgentError, isStaleWorktreeModuleResolutionError } from "./transient-error-detector.js";
import { deriveTaskIdFromFusionBranch, inspectBranchConflict, listUniqueBranchCommits } from "./branch-conflicts.js";
import { classifyForeignOnlyContamination, deriveTaskIdFromFusionBranch, inspectBranchConflict, listUniqueBranchCommits } from "./branch-conflicts.js";
import { createRunAuditor, generateSyntheticRunId, type RunAuditor } from "./run-audit.js";
import { AutoRecoveryDispatcher } from "./auto-recovery.js";
import { activeSessionRegistry } from "./active-session-registry.js";
import { findAlreadyMergedTaskCommit } from "./already-merged-detector.js";
import { resolveWorktreesDir } from "./worktree-paths.js";
import type { OwnedLandedClassification } from "./merger.js";
import { recoverForeignOnlyContamination } from "./recovery/foreign-only-contamination.js";
const log = createLogger("self-healing");
const worktreeMetadataReconcileLog = createLogger("worktree-metadata-reconcile");
@@ -540,6 +541,7 @@ export class SelfHealingManager {
{ name: "reconcile-done-task-integrity", fn: () => this.reconcileDoneTaskIntegrity().then(() => undefined) },
{ name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks().then(() => undefined) },
{ name: "recover-branch-misbound-in-review", fn: () => this.recoverBranchMisboundInReviewTasks().then(() => undefined) },
{ name: "recover-foreign-only-contamination-in-review", fn: () => this.recoverForeignOnlyContaminatedInReviewTasks().then(() => undefined) },
{ name: "recover-orphan-only-scope-violations", fn: () => this.recoverOrphanOnlyScopeViolations().then(() => undefined) },
{ name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks().then(() => undefined) },
{ name: "misclassified-failures", fn: () => this.recoverMisclassifiedFailures().then(() => undefined) },
@@ -1115,6 +1117,7 @@ export class SelfHealingManager {
{ name: "recover-merged-review", fn: () => this.recoverMergedReviewTasks() },
{ name: "recover-already-merged-review", fn: () => this.recoverAlreadyMergedReviewTasks() },
{ name: "recover-branch-misbound-in-review", fn: () => this.recoverBranchMisboundInReviewTasks() },
{ name: "recover-foreign-only-contamination-in-review", fn: () => this.recoverForeignOnlyContaminatedInReviewTasks() },
{ name: "recover-orphan-only-scope-violations", fn: () => this.recoverOrphanOnlyScopeViolations() },
{ name: "recover-stuck-merge-deadlocks", fn: () => this.recoverStuckMergeDeadlocks() },
{ name: "recover-misclassified-failures", fn: () => this.recoverMisclassifiedFailures() },
@@ -4283,6 +4286,90 @@ export class SelfHealingManager {
}
}
async recoverForeignOnlyContaminatedInReviewTasks(): Promise<number> {
try {
const settings = await this.store.getSettings();
if (settings.globalPause || settings.enginePaused) return 0;
const executingIds = this.options.getExecutingTaskIds?.() ?? new Set<string>();
const inReview = await this.store.listTasks({ column: "in-review", slim: true });
const inProgress = await this.store.listTasks({ column: "in-progress", slim: true });
const candidates = [
...inReview.filter((task) =>
task.column === "in-review" &&
Boolean(task.branch) &&
Boolean(task.worktree) &&
task.mergeDetails?.mergeConfirmed !== true &&
!task.userPaused &&
!executingIds.has(task.id),
),
...inProgress.filter((task) =>
task.column === "in-progress" &&
task.paused === true &&
(task.pausedReason === "branch-cross-contamination" || task.pausedReason === "branch-conflict-unrecoverable") &&
Boolean(task.branch) &&
Boolean(task.worktree) &&
!task.userPaused &&
!executingIds.has(task.id),
),
];
let recovered = 0;
for (const task of candidates) {
if (!task.branch || !task.worktree) continue;
const baseSha = task.baseCommitSha ?? task.baseBranch ?? task.executionStartBranch ?? "main";
try {
const classification = await classifyForeignOnlyContamination({
repoDir: this.options.rootDir,
branchName: task.branch,
baseSha,
taskId: task.id,
});
if (classification.kind === "ambiguous" || classification.kind === "clean") {
await createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-heal", task.id),
agentId: "self-healing",
taskId: task.id,
taskLineageId: task.lineageId,
phase: "recover-foreign-only-contamination-in-review",
}).database({
type: "task:auto-recover-foreign-only-contamination-skipped",
target: task.id,
metadata: { reason: classification.kind === "clean" ? "clean" : "ambiguous", kind: classification.kind },
});
continue;
}
const result = await recoverForeignOnlyContamination(task, {
repoDir: this.options.rootDir,
taskStore: this.store,
runAudit: createRunAuditor(this.store, {
runId: generateSyntheticRunId("self-heal", task.id),
agentId: "self-healing",
taskId: task.id,
taskLineageId: task.lineageId,
phase: "recover-foreign-only-contamination-in-review",
}),
});
if (result.recovered) {
await this.store.logEntry(task.id, `Auto-recovered foreign-only contamination via ${result.subtype ?? "unknown"}`);
recovered += 1;
}
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.warn(`recoverForeignOnlyContaminatedInReviewTasks: failed for task ${task.id}: ${errorMessage}`);
}
}
return recovered;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`Foreign-only contamination recovery failed: ${errorMessage}`);
return 0;
}
}
/**
* Recover tasks in `in-review` marked as `failed` where all steps are
* actually done. This catches the case where an agent completed all work