fix(engine): cap verification-failure bounces, reap unregistered worktrees, dedupe activity log
Three fixes for the worktree-overflow / stuck-task incident: 1. Cap deterministic-verification-failure bounces (fix #2) Auto-merge previously bounced an in-review task back to in-progress on every verification failure with no upper bound. A single flaky test could keep a task ping-ponging in-review→in-progress forever, holding its worktree and consuming agent slots. Adds verificationFailureCount on Task (DB migration v48), increments on each bounce, and after 3 failures marks the task failed and creates a follow-up triage task so a fresh agent can investigate the underlying flake instead of re-running the same fix loop. 2. Reap unregistered orphan worktree dirs even when recycle is on (fix #3) cleanupOrphans previously bailed out entirely when recycleWorktrees was true, leaving stale dirs (clear-hawk-broken, *-bak, leftover crash debris) on disk forever. New reapUnregisteredOrphans pass removes only directories that aren't registered git worktrees, so the recycle pool keeps its warm worktrees but the trash gets cleared. 3. Idempotence guard on activity-log listener wiring (fix #6) setupActivityLogListeners() was registering handlers on every call. When init() ran twice, every task:created / task:moved event wrote N rows to activityLog, producing the duplicate entries visible in the DB. Added activityListenersWired flag so repeated calls no-op. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
@@ -30,10 +30,12 @@ import { aiMergeTask } from "../merger.js";
|
||||
|
||||
type MockTask = {
|
||||
id: string;
|
||||
title?: string;
|
||||
column: "in-review";
|
||||
mergeRetries: number;
|
||||
status: string | null;
|
||||
error: string | null;
|
||||
verificationFailureCount?: number;
|
||||
updatedAt: string;
|
||||
log: Array<{ action?: string }>;
|
||||
};
|
||||
@@ -46,6 +48,7 @@ type MockTaskStore = {
|
||||
moveTask: ReturnType<typeof vi.fn>;
|
||||
logEntry: ReturnType<typeof vi.fn>;
|
||||
getActiveMergingTask: ReturnType<typeof vi.fn>;
|
||||
createTask: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
const TASK_ID = "FN-2084";
|
||||
@@ -94,6 +97,10 @@ function makeStore({
|
||||
moveTask: vi.fn(async () => undefined),
|
||||
logEntry: vi.fn(async () => undefined),
|
||||
getActiveMergingTask: vi.fn(() => null),
|
||||
createTask: vi.fn(async (input: { description: string }) => ({
|
||||
id: "FN-9999",
|
||||
description: input.description,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -253,21 +260,59 @@ describe("ProjectEngine merge error recovery", () => {
|
||||
|
||||
expect(store.addTaskComment).toHaveBeenCalledWith(
|
||||
TASK_ID,
|
||||
expect.stringContaining("Deterministic test verification failed during merge."),
|
||||
expect.stringContaining("Deterministic test verification failed during merge"),
|
||||
"agent",
|
||||
);
|
||||
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, {
|
||||
status: null,
|
||||
mergeRetries: 0,
|
||||
error: null,
|
||||
verificationFailureCount: 1,
|
||||
});
|
||||
expect(store.moveTask).toHaveBeenCalledWith(TASK_ID, "in-progress");
|
||||
expect(store.logEntry).toHaveBeenCalledWith(
|
||||
TASK_ID,
|
||||
"Deterministic test verification failed — moved back to in-progress for remediation",
|
||||
"Deterministic test verification failed (1/3) — moved back to in-progress for remediation",
|
||||
);
|
||||
expect(logSpy).toHaveBeenCalledWith(
|
||||
`Auto-merge: ${TASK_ID} deterministic test verification failed — moved to in-progress`,
|
||||
`Auto-merge: ${TASK_ID} deterministic test verification failed (1/3) — moved to in-progress`,
|
||||
);
|
||||
});
|
||||
|
||||
it("caps verification-failure bounces and creates a follow-up task", async () => {
|
||||
const verificationError = new Error("Deterministic test verification failed");
|
||||
verificationError.name = "VerificationError";
|
||||
vi.mocked(aiMergeTask).mockRejectedValueOnce(verificationError);
|
||||
|
||||
// Task already bounced 2 times — this attempt would push it to 3 (the cap)
|
||||
const store = makeStore({
|
||||
tasks: [
|
||||
makeTask({ verificationFailureCount: 2, title: "do the thing" }),
|
||||
],
|
||||
});
|
||||
const engine = createEngine(store);
|
||||
|
||||
await runMergeCycle(engine);
|
||||
|
||||
// Original task is failed (not bounced back)
|
||||
expect(store.moveTask).not.toHaveBeenCalledWith(TASK_ID, "in-progress");
|
||||
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, expect.objectContaining({
|
||||
status: "failed",
|
||||
verificationFailureCount: 3,
|
||||
}));
|
||||
|
||||
// Follow-up triage task created with context
|
||||
expect(store.createTask).toHaveBeenCalledWith(expect.objectContaining({
|
||||
column: "triage",
|
||||
priority: "high",
|
||||
description: expect.stringContaining(TASK_ID),
|
||||
}));
|
||||
|
||||
// Comment links the follow-up
|
||||
expect(store.addTaskComment).toHaveBeenCalledWith(
|
||||
TASK_ID,
|
||||
expect.stringContaining("FN-9999"),
|
||||
"agent",
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -142,6 +142,12 @@ export class ProjectEngine {
|
||||
private shuttingDown = false;
|
||||
|
||||
private static readonly MAX_AUTO_MERGE_RETRIES = 3;
|
||||
/** Cap on outer in-review→in-progress bounces caused by deterministic
|
||||
* verification failures during auto-merge. After this many failed merges
|
||||
* for the same task, we stop bouncing it back, mark it failed, and create
|
||||
* a follow-up triage task so a fresh agent (or human) can investigate
|
||||
* the underlying flake/regression instead of looping forever. */
|
||||
private static readonly MAX_VERIFICATION_FAILURE_BOUNCES = 3;
|
||||
/** 30-minute cooldown before a retry-exhausted task gets another sweep attempt */
|
||||
private static readonly AUTO_MERGE_COOLDOWN_MS = 30 * 60 * 1000;
|
||||
|
||||
@@ -1051,22 +1057,75 @@ export class ProjectEngine {
|
||||
|
||||
if (taskOnErr && isVerificationError) {
|
||||
const failedKind = errorMsg.includes("build verification") ? "build" : "test";
|
||||
const previousBounces = taskOnErr.verificationFailureCount ?? 0;
|
||||
const nextBounces = previousBounces + 1;
|
||||
const cap = ProjectEngine.MAX_VERIFICATION_FAILURE_BOUNCES;
|
||||
|
||||
if (nextBounces >= cap) {
|
||||
// Cap reached — stop bouncing the task and create a follow-up.
|
||||
// The original task stays in in-review with status=failed so a
|
||||
// human can inspect; the follow-up captures the failure context
|
||||
// so a fresh agent can investigate (often a flaky test or an
|
||||
// unrelated regression that won't be fixed by re-running this
|
||||
// task's branch).
|
||||
try {
|
||||
await store.updateTask(taskId, {
|
||||
status: "failed",
|
||||
verificationFailureCount: nextBounces,
|
||||
error: `Deterministic ${failedKind} verification failed ${nextBounces}× — auto-merge giving up to avoid infinite retry loop. See follow-up task for investigation.`,
|
||||
});
|
||||
const followUpDescription =
|
||||
`Investigate repeated ${failedKind} verification failure on ${taskId} (${taskOnErr.title || "untitled"}). ` +
|
||||
`Auto-merge attempted to fix and re-verify ${nextBounces} times without success — likely a flaky test or unrelated regression rather than a fix this task can produce on its own. ` +
|
||||
`Look at the most recent [verification] log entries on ${taskId} for the failing command and output, then either fix the underlying issue or quarantine the flake.`;
|
||||
const followUp = await store.createTask({
|
||||
description: followUpDescription,
|
||||
column: "triage",
|
||||
priority: "high",
|
||||
});
|
||||
await store.addTaskComment(
|
||||
taskId,
|
||||
`Auto-merge giving up after ${nextBounces} verification-failure bounces. Created follow-up ${followUp.id} to investigate.`,
|
||||
"agent",
|
||||
);
|
||||
await store.logEntry(
|
||||
taskId,
|
||||
`Auto-merge gave up after ${nextBounces} verification-failure bounces — created follow-up ${followUp.id}`,
|
||||
"VerificationError",
|
||||
);
|
||||
runtimeLog.warn(
|
||||
`Auto-merge: ${taskId} hit verification-failure cap (${nextBounces}/${cap}) — failed task and created follow-up ${followUp.id}`,
|
||||
);
|
||||
} catch (followUpErr) {
|
||||
runtimeLog.error(
|
||||
`Auto-merge: failed to fail-and-followup ${taskId} after verification cap: ${followUpErr instanceof Error ? followUpErr.message : String(followUpErr)}`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Under cap — bounce back as before, but record the increment.
|
||||
try {
|
||||
await store.addTaskComment(
|
||||
taskId,
|
||||
`Deterministic ${failedKind} verification failed during merge. ` +
|
||||
`Deterministic ${failedKind} verification failed during merge (attempt ${nextBounces}/${cap}). ` +
|
||||
`See the prior [verification] log entry for the truncated command output. ` +
|
||||
`Please fix the failing ${failedKind} and push the update so the merge can retry.`,
|
||||
"agent",
|
||||
);
|
||||
await store.updateTask(taskId, { status: null, mergeRetries: 0, error: null });
|
||||
await store.updateTask(taskId, {
|
||||
status: null,
|
||||
mergeRetries: 0,
|
||||
error: null,
|
||||
verificationFailureCount: nextBounces,
|
||||
});
|
||||
await store.moveTask(taskId, "in-progress");
|
||||
await store.logEntry(
|
||||
taskId,
|
||||
`Deterministic ${failedKind} verification failed — moved back to in-progress for remediation`,
|
||||
`Deterministic ${failedKind} verification failed (${nextBounces}/${cap}) — moved back to in-progress for remediation`,
|
||||
);
|
||||
runtimeLog.log(
|
||||
`Auto-merge: ${taskId} deterministic ${failedKind} verification failed — moved to in-progress`,
|
||||
`Auto-merge: ${taskId} deterministic ${failedKind} verification failed (${nextBounces}/${cap}) — moved to in-progress`,
|
||||
);
|
||||
} catch {
|
||||
runtimeLog.error(
|
||||
|
||||
@@ -15,11 +15,11 @@
|
||||
|
||||
import { exec } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { existsSync, readdirSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { existsSync, readdirSync, rmSync, statSync } from "node:fs";
|
||||
import { isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { getTaskMergeBlocker, type TaskStore, type Settings, type Task, type MergeDetails } from "@fusion/core";
|
||||
import { createLogger } from "./logger.js";
|
||||
import { scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||||
import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
|
||||
|
||||
const log = createLogger("self-healing");
|
||||
const execAsync = promisify(exec);
|
||||
@@ -1463,18 +1463,30 @@ export class SelfHealingManager {
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove orphaned worktrees not assigned to any active task. */
|
||||
/**
|
||||
* Remove orphaned worktrees not assigned to any active task.
|
||||
*
|
||||
* When `recycleWorktrees` is OFF: removes registered idle worktrees too —
|
||||
* they would otherwise pile up since the pool isn't keeping them.
|
||||
*
|
||||
* When `recycleWorktrees` is ON: leaves registered idle worktrees alone
|
||||
* (the pool wants them for reuse) but still reaps unregistered stale dirs
|
||||
* left behind by killed runs (e.g., `clear-hawk-broken`, `*-bak`). Those
|
||||
* dirs can never be recycled — they aren't git worktrees — so they only
|
||||
* waste disk.
|
||||
*/
|
||||
private async cleanupOrphans(): Promise<number> {
|
||||
try {
|
||||
const settings = await this.store.getSettings();
|
||||
|
||||
if (settings.recycleWorktrees) {
|
||||
// Recycle on: only sweep unregistered stale dirs.
|
||||
return await this.reapUnregisteredOrphans();
|
||||
}
|
||||
|
||||
const orphaned = await scanIdleWorktrees(this.options.rootDir, this.store);
|
||||
if (orphaned.length === 0) return 0;
|
||||
|
||||
// Only clean up if recycling is disabled — otherwise they belong in the pool
|
||||
const settings = await this.store.getSettings();
|
||||
if (settings.recycleWorktrees) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let cleaned = 0;
|
||||
for (const worktreePath of orphaned) {
|
||||
try {
|
||||
@@ -1500,6 +1512,52 @@ export class SelfHealingManager {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sweep unregistered stale directories under `<rootDir>/.worktrees/` —
|
||||
* directories that exist on disk but are NOT registered git worktrees.
|
||||
* Safe to run alongside `recycleWorktrees: true` because the pool only
|
||||
* tracks registered idle worktrees, never these orphans.
|
||||
*/
|
||||
private async reapUnregisteredOrphans(): Promise<number> {
|
||||
const worktreesDir = join(this.options.rootDir, ".worktrees");
|
||||
if (!existsSync(worktreesDir)) return 0;
|
||||
|
||||
let dirs: string[];
|
||||
try {
|
||||
dirs = readdirSync(worktreesDir, { withFileTypes: true })
|
||||
.filter((e) => e.isDirectory())
|
||||
.map((e) => join(worktreesDir, e.name));
|
||||
} catch (err: unknown) {
|
||||
log.warn(`Failed to read .worktrees/ for unregistered orphan reap: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return 0;
|
||||
}
|
||||
if (dirs.length === 0) return 0;
|
||||
|
||||
const registered = await getRegisteredWorktreePaths(this.options.rootDir);
|
||||
const unregistered = dirs.filter((d) => !registered.has(resolve(d)));
|
||||
|
||||
let cleaned = 0;
|
||||
for (const path of unregistered) {
|
||||
const rel = relative(worktreesDir, path);
|
||||
if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) {
|
||||
log.warn(`Refusing to remove path outside .worktrees: ${path}`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
rmSync(path, { recursive: true, force: true });
|
||||
log.log(`Cleaned unregistered worktree dir: ${path}`);
|
||||
cleaned++;
|
||||
} catch (err: unknown) {
|
||||
log.warn(`Failed to remove unregistered worktree dir ${path}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (cleaned > 0) {
|
||||
log.log(`Cleaned ${cleaned} unregistered worktree dir(s) (recycle mode preserves registered idle worktrees)`);
|
||||
}
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove orphaned `fusion/*` branches that are not associated with any
|
||||
* active (non-archived, non-merger-managed) task.
|
||||
|
||||
Reference in New Issue
Block a user