feat(FN-3816): add merger self-healing for missing workspace and environmen

Implements merger self-healing with a missing workspace entry detector, bootstrap recovery path, and environment fault retry handling, plus supporting tests for merge error recovery and verification utilities, alongside documentation updates.

Fusion-Task-Id: FN-3816
This commit is contained in:
Fusion
2026-05-10 21:14:34 -07:00
committed by gsxdsm
parent 86df0a0f2d
commit a4617beb03
8 changed files with 424 additions and 11 deletions

View File

@@ -1,13 +1,27 @@
import { beforeEach, describe, expect, it, vi, type MockInstance } from "vitest";
import type { Settings } from "@fusion/core";
const testState = vi.hoisted(() => ({
currentStore: null as MockTaskStore | null,
aiMergeTask: vi.fn(),
}));
const testState = vi.hoisted(() => {
class MockVerificationError extends Error {
verificationResult: unknown;
constructor(message: string, verificationResult: unknown) {
super(message);
this.name = "VerificationError";
this.verificationResult = verificationResult;
}
}
return {
currentStore: null as MockTaskStore | null,
aiMergeTask: vi.fn(),
VerificationError: MockVerificationError,
};
});
vi.mock("../merger.js", () => ({
aiMergeTask: testState.aiMergeTask,
VerificationError: testState.VerificationError,
}));
vi.mock("../runtimes/in-process-runtime.js", () => ({
@@ -26,7 +40,7 @@ vi.mock("../runtimes/in-process-runtime.js", () => ({
import { ProjectEngine } from "../project-engine.js";
import { runtimeLog } from "../logger.js";
import { aiMergeTask } from "../merger.js";
import { aiMergeTask, VerificationError } from "../merger.js";
type MockTask = {
id: string;
@@ -690,6 +704,32 @@ describe("ProjectEngine merge error recovery", () => {
);
});
it("leaves task in-review without bounce when VerificationError is an unrecovered missing-workspace-entry environment fault", async () => {
const verificationError = new VerificationError("Deterministic test verification failed", {
allPassed: false,
failedCommand: "testCommand",
environmentFault: {
kind: "missing-workspace-entry",
packageName: "@fusion/dashboard",
recovered: false,
},
});
vi.mocked(aiMergeTask).mockRejectedValueOnce(verificationError);
const store = makeStore({
tasks: [makeTask({ verificationFailureCount: 2, status: "in-review" })],
});
const engine = createEngine(store);
await runMergeCycle(engine);
expect(store.moveTask).not.toHaveBeenCalledWith(TASK_ID, "in-progress");
expect(store.updateTask).not.toHaveBeenCalledWith(
TASK_ID,
expect.objectContaining({ verificationFailureCount: 3 }),
);
});
it("increments verificationFailureCount across consecutive verification bounces", async () => {
const verificationError = new Error("Deterministic test verification failed");
verificationError.name = "VerificationError";

View File

@@ -4576,6 +4576,180 @@ describe("aiMergeTask — deterministic merge verification", () => {
);
expect(testRuns.length).toBeGreaterThan(0);
});
it("runs ensure-test-artifacts preamble before verification and logs it", async () => {
setupHappyPathExecSync();
mockedCreateFnAgent.mockResolvedValue({ session: { prompt: vi.fn(), dispose: vi.fn() } } as any);
const store = createMockStore();
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({
...DEFAULT_SETTINGS,
testCommand: "vitest run",
verificationFixRetries: 0,
});
await aiMergeTask(store, "/tmp/root", "FN-050");
const nodeCallIndex = mockedExecSync.mock.calls.findIndex((call) =>
String(call[0]).includes("node scripts/ensure-test-artifacts.mjs"),
);
const testCallIndex = mockedExecSync.mock.calls.findIndex((call) => String(call[0]).includes("vitest run"));
expect(nodeCallIndex).toBeGreaterThan(-1);
expect(testCallIndex).toBeGreaterThan(nodeCallIndex);
expect((store.logEntry as ReturnType<typeof vi.fn>).mock.calls.some((call) => String(call[1]).includes("[verification:bootstrap]"))).toBe(true);
});
it("rebuilds missing workspace package and retries verification once", async () => {
let testAttempts = 0;
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
if (cmdStr.includes("rev-parse --verify")) return Buffer.from("abc123");
if (cmdStr === "git rev-parse HEAD" || cmdStr.startsWith("git rev-parse HEAD ")) return "mergedcommit123";
if (cmdStr.includes("git log")) return "- feat: something" as any;
if (cmdStr.includes("merge-base")) return Buffer.from("abc123");
if (cmdStr.includes("git diff") && cmdStr.includes("--stat")) return "1 file changed" as any;
if (cmdStr.includes("merge --squash")) return Buffer.from("");
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
if (cmdStr.includes("diff --cached")) return "0" as any;
if (cmdStr.includes("show --shortstat")) return "3 files changed, 10 insertions(+), 2 deletions(-)" as any;
if (cmdStr.includes("branch -d") || cmdStr.includes("branch -D")) return Buffer.from("");
if (cmdStr.includes("worktree remove")) return Buffer.from("");
if (cmdStr.includes("vitest run")) {
testAttempts += 1;
if (testAttempts === 1) {
const err = new Error("vite failure") as any;
err.status = 1;
err.stderr = 'Failed to resolve entry for package "@fusion/dashboard"';
throw err;
}
return Buffer.from("");
}
return Buffer.from("");
});
mockedCreateFnAgent.mockResolvedValue({ session: { prompt: vi.fn(), dispose: vi.fn() } } as any);
const store = createMockStore();
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ ...DEFAULT_SETTINGS, testCommand: "vitest run", verificationFixRetries: 0 });
const result = await aiMergeTask(store, "/tmp/root", "FN-050");
expect(result.merged).toBe(true);
expect(mockedExecSync.mock.calls.some((call) => String(call[0]).includes("pnpm --filter @fusion/dashboard build"))).toBe(true);
expect(testAttempts).toBe(2);
});
it("throws VerificationError with environmentFault.recovered=false when retry still fails the same way", async () => {
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
if (cmdStr.includes("rev-parse --verify")) return Buffer.from("abc123");
if (cmdStr === "git rev-parse HEAD" || cmdStr.startsWith("git rev-parse HEAD ")) return "mergedcommit123";
if (cmdStr.includes("git log")) return "- feat: something" as any;
if (cmdStr.includes("merge-base")) return Buffer.from("abc123");
if (cmdStr.includes("git diff") && cmdStr.includes("--stat")) return "1 file changed" as any;
if (cmdStr.includes("merge --squash")) return Buffer.from("");
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
if (cmdStr.includes("diff --cached")) return "0" as any;
if (cmdStr.includes("show --shortstat")) return "3 files changed, 10 insertions(+), 2 deletions(-)" as any;
if (cmdStr.includes("branch -d") || cmdStr.includes("branch -D")) return Buffer.from("");
if (cmdStr.includes("worktree remove")) return Buffer.from("");
if (cmdStr.includes("vitest run")) {
const err = new Error("vite failure") as any;
err.status = 1;
err.stderr = 'Failed to resolve entry for package "@fusion/dashboard"';
throw err;
}
return Buffer.from("");
});
mockedCreateFnAgent.mockResolvedValue({ session: { prompt: vi.fn(), dispose: vi.fn() } } as any);
const store = createMockStore();
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ ...DEFAULT_SETTINGS, testCommand: "vitest run", verificationFixRetries: 0 });
await expect(aiMergeTask(store, "/tmp/root", "FN-050")).rejects.toMatchObject({
name: "VerificationError",
verificationResult: {
environmentFault: {
kind: "missing-workspace-entry",
packageName: "@fusion/dashboard",
recovered: false,
},
},
});
});
it("throws normal VerificationError without environmentFault when retry fails differently", async () => {
let testAttempts = 0;
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
if (cmdStr.includes("rev-parse --verify")) return Buffer.from("abc123");
if (cmdStr === "git rev-parse HEAD" || cmdStr.startsWith("git rev-parse HEAD ")) return "mergedcommit123";
if (cmdStr.includes("git log")) return "- feat: something" as any;
if (cmdStr.includes("merge-base")) return Buffer.from("abc123");
if (cmdStr.includes("git diff") && cmdStr.includes("--stat")) return "1 file changed" as any;
if (cmdStr.includes("merge --squash")) return Buffer.from("");
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
if (cmdStr.includes("diff --cached")) return "0" as any;
if (cmdStr.includes("show --shortstat")) return "3 files changed, 10 insertions(+), 2 deletions(-)" as any;
if (cmdStr.includes("branch -d") || cmdStr.includes("branch -D")) return Buffer.from("");
if (cmdStr.includes("worktree remove")) return Buffer.from("");
if (cmdStr.includes("vitest run")) {
testAttempts += 1;
const err = new Error("verification failure") as any;
err.status = 1;
err.stderr = testAttempts === 1
? 'Failed to resolve entry for package "@fusion/dashboard"'
: "AssertionError: genuine test failure";
throw err;
}
return Buffer.from("");
});
mockedCreateFnAgent.mockResolvedValue({ session: { prompt: vi.fn(), dispose: vi.fn() } } as any);
const store = createMockStore();
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ ...DEFAULT_SETTINGS, testCommand: "vitest run", verificationFixRetries: 0 });
try {
await aiMergeTask(store, "/tmp/root", "FN-050");
throw new Error("expected verification error");
} catch (error) {
expect(error).toMatchObject({ name: "VerificationError" });
expect(error).not.toHaveProperty("verificationResult.environmentFault");
}
});
it("throws VerificationError when bootstrap preamble fails without environmentFault", async () => {
mockedExecSync.mockImplementation((cmd: any) => {
const cmdStr = String(cmd);
if (cmdStr.includes("node scripts/ensure-test-artifacts.mjs")) {
const err = new Error("bootstrap failed") as any;
err.status = 1;
err.stderr = "bootstrap failed";
throw err;
}
if (cmdStr.includes("rev-parse --verify")) return Buffer.from("abc123");
if (cmdStr === "git rev-parse HEAD" || cmdStr.startsWith("git rev-parse HEAD ")) return "mergedcommit123";
if (cmdStr.includes("git log")) return "- feat: something" as any;
if (cmdStr.includes("merge-base")) return Buffer.from("abc123");
if (cmdStr.includes("git diff") && cmdStr.includes("--stat")) return "1 file changed" as any;
if (cmdStr.includes("merge --squash")) return Buffer.from("");
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
if (cmdStr.includes("diff --cached")) return "0" as any;
if (cmdStr.includes("show --shortstat")) return "3 files changed, 10 insertions(+), 2 deletions(-)" as any;
if (cmdStr.includes("branch -d") || cmdStr.includes("branch -D")) return Buffer.from("");
if (cmdStr.includes("worktree remove")) return Buffer.from("");
return Buffer.from("");
});
mockedCreateFnAgent.mockResolvedValue({ session: { prompt: vi.fn(), dispose: vi.fn() } } as any);
const store = createMockStore();
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValue({ ...DEFAULT_SETTINGS, testCommand: "vitest run", verificationFixRetries: 0 });
try {
await aiMergeTask(store, "/tmp/root", "FN-050");
throw new Error("expected verification error");
} catch (error) {
expect(error).toMatchObject({ name: "VerificationError" });
expect(error).not.toHaveProperty("verificationResult.environmentFault");
}
});
});
describe("shouldSyncDependenciesForMerge", () => {

View File

@@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { execWithProcessGroup } from "../verification-utils.js";
import { detectMissingWorkspaceEntry, execWithProcessGroup } from "../verification-utils.js";
const onPosix = process.platform !== "win32";
const itPosix = onPosix ? it : it.skip;
@@ -79,3 +79,31 @@ setInterval(() => {}, 1000);
await expect(access(markerPath)).rejects.toThrow();
});
});
describe("detectMissingWorkspaceEntry", () => {
it("matches @fusion package names from stderr", () => {
expect(
detectMissingWorkspaceEntry('Error: Failed to resolve entry for package "@fusion/dashboard" from vite'),
).toEqual({ packageName: "@fusion/dashboard" });
});
it("matches @fusion-plugin-examples package names from stderr", () => {
expect(
detectMissingWorkspaceEntry('Failed to resolve entry for package "@fusion-plugin-examples/hermes-runtime"'),
).toEqual({ packageName: "@fusion-plugin-examples/hermes-runtime" });
});
it("returns null for unrelated stderr", () => {
expect(detectMissingWorkspaceEntry("Some different failure output")).toBeNull();
});
it("returns null for truncated messages without closing quote", () => {
expect(detectMissingWorkspaceEntry('Failed to resolve entry for package "@fusion/da')).toBeNull();
});
it("finds matches in stdout when stderr does not contain one", () => {
expect(
detectMissingWorkspaceEntry("no error in stderr", 'Failed to resolve entry for package "@fusion/core" in stdout'),
).toEqual({ packageName: "@fusion/core" });
});
});

View File

@@ -5,6 +5,7 @@ import { promisify } from "node:util";
const execAsync = promisify(exec);
const execFileAsync = promisify(execFile);
import {
detectMissingWorkspaceEntry,
runVerificationCommand as runVerificationCommandShared,
summarizeVerificationOutput,
truncateWithEllipsis,
@@ -659,10 +660,136 @@ async function runDeterministicVerification(
await store.logEntry(taskId, deterministicVerificationMessage);
await store.appendAgentLog(taskId, deterministicVerificationMessage, "text", undefined, "merger");
const bootstrapScriptPath = join(rootDir, "scripts/ensure-test-artifacts.mjs");
if (hasTestCommand || hasBuildCommand) {
if (!existsSync(bootstrapScriptPath)) {
const bootstrapMissingMessage = `${taskId}: [verification:bootstrap] script missing at scripts/ensure-test-artifacts.mjs — skipping preamble`;
mergerLog.warn(bootstrapMissingMessage);
await store.logEntry(taskId, bootstrapMissingMessage);
await store.appendAgentLog(taskId, bootstrapMissingMessage, "text", undefined, "merger");
} else {
const bootstrapCommand = "node scripts/ensure-test-artifacts.mjs";
await store.logEntry(taskId, `[verification:bootstrap] running: ${bootstrapCommand}`);
await store.appendAgentLog(taskId, "[verification:bootstrap] running bootstrap preamble", "tool", bootstrapCommand, "merger");
try {
throwIfAborted(signal, taskId);
await execAsync(bootstrapCommand, {
cwd: rootDir,
timeout: 300_000,
maxBuffer: 10 * 1024 * 1024,
signal,
});
throwIfAborted(signal, taskId);
await store.logEntry(taskId, "[verification:bootstrap] bootstrap preamble succeeded");
await store.appendAgentLog(taskId, "[verification:bootstrap] bootstrap preamble succeeded", "tool_result", undefined, "merger");
} catch (error) {
throwIfAborted(signal, taskId);
const err = error as { stdout?: string | Buffer; stderr?: string | Buffer; status?: number; code?: number | string; message?: string };
const bootstrapStdout = err?.stdout?.toString?.() || "";
const bootstrapStderr = err?.stderr?.toString?.() || "";
const bootstrapOutput = bootstrapStderr || bootstrapStdout || err?.message || "Unknown bootstrap failure";
const bootstrapExitCode = typeof err?.status === "number"
? err.status
: (typeof err?.code === "number" ? err.code : null);
result.allPassed = false;
result.failedCommand = "bootstrap";
await store.logEntry(
taskId,
`[verification:bootstrap] bootstrap preamble failed (exit ${bootstrapExitCode ?? "unknown"}): ${truncateWithEllipsis(bootstrapOutput, VERIFICATION_LOG_MAX_CHARS)}`,
"VerificationError",
);
await store.appendAgentLog(
taskId,
"[verification:bootstrap] bootstrap preamble failed",
"tool_error",
`exit ${bootstrapExitCode ?? "unknown"}`,
"merger",
);
throw new VerificationError(
`Verification bootstrap preamble failed for ${taskId}`,
result,
);
}
}
}
let missingEntryRetryAttempted = false;
const executeVerificationWithRetry = async (
command: string,
type: "test" | "build",
failedCommandLabel: "testCommand" | "buildCommand",
): Promise<VerificationCommandResult> => {
const firstAttempt = await runVerificationCommand(
store, rootDir, taskId, command, type, signal,
);
if (firstAttempt.success) {
return firstAttempt;
}
const missingWorkspaceEntry = detectMissingWorkspaceEntry(firstAttempt.stderr, firstAttempt.stdout);
if (!missingWorkspaceEntry || missingEntryRetryAttempted) {
return firstAttempt;
}
missingEntryRetryAttempted = true;
const packageName = missingWorkspaceEntry.packageName;
const rebuildCommand = `pnpm --filter ${packageName} build`;
await store.logEntry(taskId, `[verification:retry] bootstrap-built: detected missing workspace entry for ${packageName}; running ${rebuildCommand}`);
await store.appendAgentLog(taskId, "[verification:retry] bootstrap-built", "tool", rebuildCommand, "merger");
try {
throwIfAborted(signal, taskId);
await execAsync(rebuildCommand, {
cwd: rootDir,
timeout: 300_000,
maxBuffer: 10 * 1024 * 1024,
signal,
});
throwIfAborted(signal, taskId);
} catch (_error) {
throwIfAborted(signal, taskId);
await store.logEntry(taskId, `[verification:retry] retry-different-failure: workspace rebuild failed for ${packageName}`);
await store.appendAgentLog(taskId, "[verification:retry] retry-different-failure", "tool_error", packageName, "merger");
return firstAttempt;
}
const retryAttempt = await runVerificationCommand(
store, rootDir, taskId, command, type, signal,
);
if (retryAttempt.success) {
result.environmentFault = {
kind: "missing-workspace-entry",
packageName,
recovered: true,
};
await store.logEntry(taskId, `[verification:retry] retry-success: rebuilt ${packageName} and ${failedCommandLabel} now passes`);
await store.appendAgentLog(taskId, "[verification:retry] retry-success", "tool_result", packageName, "merger");
return retryAttempt;
}
const retryMissingWorkspaceEntry = detectMissingWorkspaceEntry(retryAttempt.stderr, retryAttempt.stdout);
if (retryMissingWorkspaceEntry?.packageName === packageName) {
result.environmentFault = {
kind: "missing-workspace-entry",
packageName,
recovered: false,
};
await store.logEntry(taskId, `[verification:retry] retry-still-missing: ${packageName} still missing after rebuild`);
await store.appendAgentLog(taskId, "[verification:retry] retry-still-missing", "tool_error", packageName, "merger");
return retryAttempt;
}
await store.logEntry(taskId, `[verification:retry] retry-different-failure: rebuild fixed entry point but ${failedCommandLabel} still failed`);
await store.appendAgentLog(taskId, "[verification:retry] retry-different-failure", "tool_error", packageName, "merger");
return retryAttempt;
};
// Run test command first if configured
if (hasTestCommand) {
const testResult = await runVerificationCommand(
store, rootDir, taskId, normalizedTestCommand!, "test", signal,
const testResult = await executeVerificationWithRetry(
normalizedTestCommand!, "test", "testCommand",
);
result.testResult = testResult;
@@ -690,8 +817,8 @@ async function runDeterministicVerification(
// Run build command second if configured
if (hasBuildCommand) {
const buildResult = await runVerificationCommand(
store, rootDir, taskId, normalizedBuildCommand!, "build", signal,
const buildResult = await executeVerificationWithRetry(
normalizedBuildCommand!, "build", "buildCommand",
);
result.buildResult = buildResult;

View File

@@ -22,7 +22,7 @@ import { NotificationService } from "./notification/index.js";
import { GridlockDetector } from "./gridlock-detector.js";
import { CronRunner, createAiPromptExecutor } from "./cron-runner.js";
import type { RoutineRunner } from "./routine-runner.js";
import { aiMergeTask, sweepStaleAutostashes } from "./merger.js";
import { aiMergeTask, sweepStaleAutostashes, VerificationError } from "./merger.js";
import { PRIORITY_MERGE } from "./concurrency.js";
import { runtimeLog } from "./logger.js";
import type { HeartbeatTriggerScheduler } from "./agent-heartbeat.js";
@@ -1468,6 +1468,18 @@ export class ProjectEngine {
errorMsg.includes("Deterministic build verification failed");
if (taskOnErr && isVerificationError) {
if (
err instanceof VerificationError
&& err.verificationResult?.environmentFault?.kind === "missing-workspace-entry"
&& err.verificationResult.environmentFault.recovered === false
) {
const packageName = err.verificationResult.environmentFault.packageName;
const message = `${taskId}: verification failed with environment fault (missing-workspace-entry: ${packageName}) — leaving in-review for next sweep, not incrementing verificationFailureCount`;
await store.logEntry(taskId, message, "VerificationError").catch(() => undefined);
runtimeLog.log(`Auto-merge: ${message}`);
continue;
}
const failedKind = errorMsg.includes("build verification") ? "build" : "test";
const previousBounces = taskOnErr.verificationFailureCount ?? 0;
const nextBounces = previousBounces + 1;

View File

@@ -30,6 +30,11 @@ export interface VerificationResult {
buildResult?: VerificationCommandResult;
allPassed: boolean;
failedCommand?: string;
environmentFault?: {
kind: "missing-workspace-entry";
packageName: string;
recovered: boolean;
};
}
// ── Process group exec ─────────────────────────────────────────────────
@@ -173,6 +178,23 @@ export function truncateWithEllipsis(text: string, maxChars: number): string {
return `${text.slice(0, maxChars)}\n... (truncated)`;
}
export function detectMissingWorkspaceEntry(stderr: string, stdout?: string): { packageName: string } | null {
const pattern = /Failed to resolve entry for package\s+"(@fusion\/[a-z0-9-]+|@fusion-plugin-examples\/[a-z0-9-]+)"/;
const stderrMatch = stderr.match(pattern);
if (stderrMatch) {
return { packageName: stderrMatch[1] };
}
if (stdout) {
const stdoutMatch = stdout.match(pattern);
if (stdoutMatch) {
return { packageName: stdoutMatch[1] };
}
}
return null;
}
function truncateOutput(output: string): string {
if (output.length <= VERIFICATION_LOG_MAX_CHARS) return output;
return `... output truncated to last ${VERIFICATION_LOG_MAX_CHARS} characters ...\n${output.slice(-VERIFICATION_LOG_MAX_CHARS)}`;