root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(engine): cap verification-failure bounces, reap unregistered worktrees, dedupe activity log

Browse Source 
Three fixes for the worktree-overflow / stuck-task incident:

1. Cap deterministic-verification-failure bounces (fix #2)
   Auto-merge previously bounced an in-review task back to in-progress
   on every verification failure with no upper bound. A single flaky test
   could keep a task ping-ponging in-review→in-progress forever, holding
   its worktree and consuming agent slots. Adds verificationFailureCount
   on Task (DB migration v48), increments on each bounce, and after 3
   failures marks the task failed and creates a follow-up triage task
   so a fresh agent can investigate the underlying flake instead of
   re-running the same fix loop.

2. Reap unregistered orphan worktree dirs even when recycle is on (fix #3)
   cleanupOrphans previously bailed out entirely when recycleWorktrees
   was true, leaving stale dirs (clear-hawk-broken, *-bak, leftover
   crash debris) on disk forever. New reapUnregisteredOrphans pass
   removes only directories that aren't registered git worktrees, so
   the recycle pool keeps its warm worktrees but the trash gets cleared.

3. Idempotence guard on activity-log listener wiring (fix #6)
   setupActivityLogListeners() was registering handlers on every call.
   When init() ran twice, every task:created / task:moved event wrote
   N rows to activityLog, producing the duplicate entries visible in
   the DB. Added activityListenersWired flag so repeated calls no-op.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-26 21:09:28 -07:00
parent 07e86e696d
commit 53decd1bd2
12 changed files with 237 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
packages/core/src/__tests__/db.test.ts
View File

@@ -131,7 +131,7 @@ describe("Database", () => {
});
it("seeds schema version", () => {
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
});
it("seeds lastModified", () => {
@@ -154,7 +154,7 @@ describe("Database", () => {
it("is idempotent - calling init() twice does not fail", () => {
expect(() => db.init()).not.toThrow();
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
});
it("does not overwrite existing config on re-init", () => {
@@ -761,7 +761,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 29 (includes v1→v2 through v26→v29)
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -786,11 +786,11 @@ describe("schema migrations", () => {
const db = new Database(fusionDir);
db.init();
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
// Re-init should not fail
db.init();
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
db.close();
});
@@ -825,7 +825,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
expect(cols.map((col) => col.name)).toContain("priority");
@@ -866,7 +866,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
const colNames = cols.map((col) => col.name);
@@ -935,7 +935,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
const colNames = cols.map((col) => col.name);
@@ -976,7 +976,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>;
expect(tables).toEqual([{ name: "agentRatings" }]);
@@ -1000,7 +1000,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>;
expect(tables).toEqual([{ name: "mission_events" }]);
@@ -1104,7 +1104,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 29
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -1473,7 +1473,7 @@ describe("createDatabase factory", () => {
const db = createDatabase(fusionDir);
db.init();
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
expect(db.getLastModified()).toBeGreaterThan(0);
db.close();

8
packages/core/src/__tests__/insight-store.test.ts
View File

@@ -779,7 +779,7 @@ describe("Migration: pre-33 DB upgrade", () => {
// Step 1: Create a fresh database at v33 (runs all migrations up to 33)
const db1 = createDatabase(legacyDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(47);
expect(db1.getSchemaVersion()).toBe(48);
db1.close();
// Step 2: Manually downgrade to version 32 and drop insight tables
@@ -814,7 +814,7 @@ describe("Migration: pre-33 DB upgrade", () => {
expect(tableNamesBefore).not.toContain("project_insight_runs");
// Now run init — this triggers the v32→v33 migration
db3.init();
expect(db3.getSchemaVersion()).toBe(47);
expect(db3.getSchemaVersion()).toBe(48);
// Step 4: Verify insight tables exist after migration
const tablesAfter = db3.prepare(
@@ -845,12 +845,12 @@ describe("Migration: pre-33 DB upgrade", () => {
try {
const db1 = createDatabase(testDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(47);
expect(db1.getSchemaVersion()).toBe(48);
db1.close();
const db2 = createDatabase(testDir);
expect(() => db2.init()).not.toThrow();
expect(db2.getSchemaVersion()).toBe(47);
expect(db2.getSchemaVersion()).toBe(48);
db2.close();
} finally {
rmSync(testDir, { recursive: true, force: true });

2
packages/core/src/__tests__/mission-store.test.ts
View File

@@ -2629,7 +2629,7 @@ describe("MissionStore", () => {
describe("Loop State & Validator Run Schema (v31)", () => {
it("schema version is 40 after migration", () => {
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
});
it("mission_features table has loop state columns", () => {

2
packages/core/src/__tests__/roadmap-store.test.ts
View File

@@ -742,7 +742,7 @@ describe("RoadmapStore", () => {
describe("schema version", () => {
it("schema version is 40 after init", () => {
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
});
});

2
packages/core/src/__tests__/run-audit.test.ts
View File

@@ -465,7 +465,7 @@ describe("Run Audit", () => {
});
it("schema version is bumped to 40", () => {
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
});
});
});

2
packages/core/src/__tests__/task-documents.test.ts
View File

@@ -51,7 +51,7 @@ describe("TaskStore task documents", () => {
expect(tableNames.has("task_documents")).toBe(true);
expect(tableNames.has("task_document_revisions")).toBe(true);
expect(db.getSchemaVersion()).toBe(47);
expect(db.getSchemaVersion()).toBe(48);
const index = db
.prepare(

9
packages/core/src/db.ts
View File

@@ -1836,6 +1836,15 @@ export class Database {
});
}
// Outer verification-failure bounce counter — counts in-review→in-progress
// returns triggered by VerificationError. Capped to prevent infinite
// re-merge loops on flaky tests (see project-engine.ts auto-merge handler).
if (version < 48) {
this.applyMigration(48, () => {
this.addColumnIfMissing("tasks", "verificationFailureCount", "INTEGER DEFAULT 0");
});
}
}
/**

27
packages/core/src/store.ts
View File

@@ -52,6 +52,7 @@ interface TaskRow {
postReviewFixCount: number | null;
recoveryRetryCount: number | null;
taskDoneRetryCount: number | null;
verificationFailureCount: number | null;
nextRecoveryAt: string | null;
error: string | null;
summary: string | null;
@@ -350,6 +351,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
private configPath: string;
/** SQLite database for structured data storage */
private _db: Database | null = null;
private activityListenersWired = false;
/** Separate SQLite database for compact archived task snapshots. */
private _archiveDb: ArchiveDatabase | null = null;
@@ -530,6 +532,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
postReviewFixCount: row.postReviewFixCount ?? undefined,
recoveryRetryCount: row.recoveryRetryCount ?? undefined,
taskDoneRetryCount: row.taskDoneRetryCount ?? undefined,
verificationFailureCount: row.verificationFailureCount ?? undefined,
nextRecoveryAt: row.nextRecoveryAt || undefined,
error: row.error || undefined,
summary: row.summary || undefined,
@@ -823,7 +826,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
"modelPresetId", "modelProvider", "modelId",
"validatorModelProvider", "validatorModelId",
"planningModelProvider", "planningModelId",
"mergeRetries", "workflowStepRetries", "stuckKillCount", "postReviewFixCount", "recoveryRetryCount", "taskDoneRetryCount", "nextRecoveryAt",
"mergeRetries", "workflowStepRetries", "stuckKillCount", "postReviewFixCount", "recoveryRetryCount", "taskDoneRetryCount", "verificationFailureCount", "nextRecoveryAt",
"error", "summary", "thinkingLevel", "executionMode",
"tokenUsageInputTokens", "tokenUsageOutputTokens", "tokenUsageCachedTokens", "tokenUsageTotalTokens", "tokenUsageFirstUsedAt", "tokenUsageLastUsedAt",
"createdAt", "updatedAt", "columnMovedAt",
@@ -842,7 +845,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
"modelPresetId", "modelProvider", "modelId",
"validatorModelProvider", "validatorModelId",
"planningModelProvider", "planningModelId",
"mergeRetries", "workflowStepRetries", "stuckKillCount", "postReviewFixCount", "recoveryRetryCount", "taskDoneRetryCount", "nextRecoveryAt",
"mergeRetries", "workflowStepRetries", "stuckKillCount", "postReviewFixCount", "recoveryRetryCount", "taskDoneRetryCount", "verificationFailureCount", "nextRecoveryAt",
"error", "summary", "thinkingLevel", "executionMode",
"tokenUsageInputTokens", "tokenUsageOutputTokens", "tokenUsageCachedTokens", "tokenUsageTotalTokens", "tokenUsageFirstUsedAt", "tokenUsageLastUsedAt",
"createdAt", "updatedAt", "columnMovedAt",
@@ -884,7 +887,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
id, title, description, priority, "column", status, size, reviewLevel, currentStep,
worktree, blockedBy, paused, baseBranch, branch, baseCommitSha, modelPresetId, modelProvider,
modelId, validatorModelProvider, validatorModelId, planningModelProvider, planningModelId, mergeRetries,
workflowStepRetries, stuckKillCount, postReviewFixCount, recoveryRetryCount, taskDoneRetryCount, nextRecoveryAt, error,
workflowStepRetries, stuckKillCount, postReviewFixCount, recoveryRetryCount, taskDoneRetryCount, verificationFailureCount, nextRecoveryAt, error,
summary, thinkingLevel, executionMode, tokenUsageInputTokens, tokenUsageOutputTokens, tokenUsageCachedTokens,
tokenUsageTotalTokens, tokenUsageFirstUsedAt, tokenUsageLastUsedAt, createdAt, updatedAt, columnMovedAt,
dependencies, steps, log, attachments, steeringComments,
@@ -892,7 +895,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
sourceIssueProvider, sourceIssueRepository, sourceIssueExternalIssueId, sourceIssueNumber, sourceIssueUrl,
mergeDetails, breakIntoSubtasks, enabledWorkflowSteps, modifiedFiles, missionId, sliceId, assignedAgentId, assigneeUserId, checkedOutBy, checkedOutAt
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
ON CONFLICT(id) DO UPDATE SET
title = excluded.title,
@@ -922,6 +925,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
postReviewFixCount = excluded.postReviewFixCount,
recoveryRetryCount = excluded.recoveryRetryCount,
taskDoneRetryCount = excluded.taskDoneRetryCount,
verificationFailureCount = excluded.verificationFailureCount,
nextRecoveryAt = excluded.nextRecoveryAt,
error = excluded.error,
summary = excluded.summary,
@@ -989,6 +993,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
task.postReviewFixCount ?? 0,
task.recoveryRetryCount ?? null,
task.taskDoneRetryCount ?? 0,
task.verificationFailureCount ?? 0,
task.nextRecoveryAt ?? null,
task.error ?? null,
task.summary ?? null,
@@ -1073,8 +1078,15 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
/**
* Set up event listeners for activity logging.
* Call after init() to record task lifecycle events.
*
* Idempotent — repeated calls are no-ops. Without this guard, each duplicate
* call double-registers handlers, causing the activity log to record every
* `task:created` / `task:moved` event N times where N = number of init() calls.
*/
private setupActivityLogListeners(): void {
if (this.activityListenersWired) return;
this.activityListenersWired = true;
// Task created
this.on("task:created", (task) => {
this.recordActivityFromListener(
@@ -2553,7 +2565,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
async updateTask(
id: string,
updates: { title?: string; description?: string; priority?: TaskPriority | null; prompt?: string; worktree?: string | null; status?: string | null; dependencies?: string[]; steps?: import("./types.js").TaskStep[]; currentStep?: number; blockedBy?: string | null; assignedAgentId?: string | null; assigneeUserId?: string | null; checkedOutBy?: string | null; checkedOutAt?: string | null; paused?: boolean; baseBranch?: string | null; branch?: string | null; baseCommitSha?: string | null; size?: "S" | "M" | "L"; reviewLevel?: number; executionMode?: import("./types.js").ExecutionMode | null; mergeRetries?: number; workflowStepRetries?: number; stuckKillCount?: number | null; postReviewFixCount?: number | null; recoveryRetryCount?: number | null; taskDoneRetryCount?: number | null; nextRecoveryAt?: string | null; enabledWorkflowSteps?: string[]; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; planningModelProvider?: string | null; planningModelId?: string | null; thinkingLevel?: string | null; error?: string | null; summary?: string | null; sessionFile?: string | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; mergeDetails?: import("./types.js").MergeDetails | null; sourceIssue?: import("./types.js").TaskSourceIssue | null; tokenUsage?: import("./types.js").TaskTokenUsage | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
updates: { title?: string; description?: string; priority?: TaskPriority | null; prompt?: string; worktree?: string | null; status?: string | null; dependencies?: string[]; steps?: import("./types.js").TaskStep[]; currentStep?: number; blockedBy?: string | null; assignedAgentId?: string | null; assigneeUserId?: string | null; checkedOutBy?: string | null; checkedOutAt?: string | null; paused?: boolean; baseBranch?: string | null; branch?: string | null; baseCommitSha?: string | null; size?: "S" | "M" | "L"; reviewLevel?: number; executionMode?: import("./types.js").ExecutionMode | null; mergeRetries?: number; workflowStepRetries?: number; stuckKillCount?: number | null; postReviewFixCount?: number | null; recoveryRetryCount?: number | null; taskDoneRetryCount?: number | null; verificationFailureCount?: number | null; nextRecoveryAt?: string | null; enabledWorkflowSteps?: string[]; modelProvider?: string | null; modelId?: string | null; validatorModelProvider?: string | null; validatorModelId?: string | null; planningModelProvider?: string | null; planningModelId?: string | null; thinkingLevel?: string | null; error?: string | null; summary?: string | null; sessionFile?: string | null; workflowStepResults?: import("./types.js").WorkflowStepResult[] | null; mergeDetails?: import("./types.js").MergeDetails | null; sourceIssue?: import("./types.js").TaskSourceIssue | null; tokenUsage?: import("./types.js").TaskTokenUsage | null; modifiedFiles?: string[] | null; missionId?: string | null; sliceId?: string | null },
runContext?: RunMutationContext,
): Promise<Task> {
return this.withTaskLock(id, async () => {
@@ -2674,6 +2686,11 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
} else if (updates.taskDoneRetryCount !== undefined) {
task.taskDoneRetryCount = updates.taskDoneRetryCount;
}
if (updates.verificationFailureCount === null) {
task.verificationFailureCount = undefined;
} else if (updates.verificationFailureCount !== undefined) {
task.verificationFailureCount = updates.verificationFailureCount;
}
if (updates.nextRecoveryAt === null) {
task.nextRecoveryAt = undefined;
} else if (updates.nextRecoveryAt !== undefined) {

7
packages/core/src/types.ts
View File

@@ -815,6 +815,13 @@ export interface Task {
* failures. Capped by `MAX_TASK_DONE_RETRIES`; when exhausted the task stays
* in `in-review` for human inspection. Cleared on successful completion. */
taskDoneRetryCount?: number;
/** Number of times this task has bounced from `in-review` back to `in-progress`
* due to a deterministic verification failure during auto-merge. Incremented
* by the auto-merge error handler (project-engine.ts). When this reaches
* `MAX_VERIFICATION_FAILURE_BOUNCES`, the task is marked failed and a
* follow-up triage task is created so a human / fresh agent can investigate
* rather than endlessly re-attempting the same fix. */
verificationFailureCount?: number;
/** ISO-8601 timestamp indicating when the task becomes eligible for the next
* recovery retry. Scheduler and triage processor skip tasks whose
* `nextRecoveryAt` is still in the future. Cleared alongside `recoveryRetryCount`. */

51
packages/engine/src/__tests__/merge-error-recovery.test.ts
View File

@@ -30,10 +30,12 @@ import { aiMergeTask } from "../merger.js";
type MockTask = {
id: string;
title?: string;
column: "in-review";
mergeRetries: number;
status: string | null;
error: string | null;
verificationFailureCount?: number;
updatedAt: string;
log: Array<{ action?: string }>;
};
@@ -46,6 +48,7 @@ type MockTaskStore = {
moveTask: ReturnType<typeof vi.fn>;
logEntry: ReturnType<typeof vi.fn>;
getActiveMergingTask: ReturnType<typeof vi.fn>;
createTask: ReturnType<typeof vi.fn>;
};
const TASK_ID = "FN-2084";
@@ -94,6 +97,10 @@ function makeStore({
moveTask: vi.fn(async () => undefined),
logEntry: vi.fn(async () => undefined),
getActiveMergingTask: vi.fn(() => null),
createTask: vi.fn(async (input: { description: string }) => ({
id: "FN-9999",
description: input.description,
})),
};
}
@@ -253,21 +260,59 @@ describe("ProjectEngine merge error recovery", () => {
expect(store.addTaskComment).toHaveBeenCalledWith(
TASK_ID,
expect.stringContaining("Deterministic test verification failed during merge."),
expect.stringContaining("Deterministic test verification failed during merge"),
"agent",
);
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, {
status: null,
mergeRetries: 0,
error: null,
verificationFailureCount: 1,
});
expect(store.moveTask).toHaveBeenCalledWith(TASK_ID, "in-progress");
expect(store.logEntry).toHaveBeenCalledWith(
TASK_ID,
"Deterministic test verification failed — moved back to in-progress for remediation",
"Deterministic test verification failed (1/3) — moved back to in-progress for remediation",
);
expect(logSpy).toHaveBeenCalledWith(
`Auto-merge: ${TASK_ID} deterministic test verification failed — moved to in-progress`,
`Auto-merge: ${TASK_ID} deterministic test verification failed (1/3) — moved to in-progress`,
);
});
it("caps verification-failure bounces and creates a follow-up task", async () => {
const verificationError = new Error("Deterministic test verification failed");
verificationError.name = "VerificationError";
vi.mocked(aiMergeTask).mockRejectedValueOnce(verificationError);
// Task already bounced 2 times — this attempt would push it to 3 (the cap)
const store = makeStore({
tasks: [
makeTask({ verificationFailureCount: 2, title: "do the thing" }),
],
});
const engine = createEngine(store);
await runMergeCycle(engine);
// Original task is failed (not bounced back)
expect(store.moveTask).not.toHaveBeenCalledWith(TASK_ID, "in-progress");
expect(store.updateTask).toHaveBeenCalledWith(TASK_ID, expect.objectContaining({
status: "failed",
verificationFailureCount: 3,
}));
// Follow-up triage task created with context
expect(store.createTask).toHaveBeenCalledWith(expect.objectContaining({
column: "triage",
priority: "high",
description: expect.stringContaining(TASK_ID),
}));
// Comment links the follow-up
expect(store.addTaskComment).toHaveBeenCalledWith(
TASK_ID,
expect.stringContaining("FN-9999"),
"agent",
);
});

67
packages/engine/src/project-engine.ts
View File

@@ -142,6 +142,12 @@ export class ProjectEngine {
private shuttingDown = false;
private static readonly MAX_AUTO_MERGE_RETRIES = 3;
/** Cap on outer in-review→in-progress bounces caused by deterministic
* verification failures during auto-merge. After this many failed merges
* for the same task, we stop bouncing it back, mark it failed, and create
* a follow-up triage task so a fresh agent (or human) can investigate
* the underlying flake/regression instead of looping forever. */
private static readonly MAX_VERIFICATION_FAILURE_BOUNCES = 3;
/** 30-minute cooldown before a retry-exhausted task gets another sweep attempt */
private static readonly AUTO_MERGE_COOLDOWN_MS = 30 * 60 * 1000;
@@ -1051,22 +1057,75 @@ export class ProjectEngine {
if (taskOnErr && isVerificationError) {
const failedKind = errorMsg.includes("build verification") ? "build" : "test";
const previousBounces = taskOnErr.verificationFailureCount ?? 0;
const nextBounces = previousBounces + 1;
const cap = ProjectEngine.MAX_VERIFICATION_FAILURE_BOUNCES;
if (nextBounces >= cap) {
// Cap reached — stop bouncing the task and create a follow-up.
// The original task stays in in-review with status=failed so a
// human can inspect; the follow-up captures the failure context
// so a fresh agent can investigate (often a flaky test or an
// unrelated regression that won't be fixed by re-running this
// task's branch).
try {
await store.updateTask(taskId, {
status: "failed",
verificationFailureCount: nextBounces,
error: `Deterministic ${failedKind} verification failed ${nextBounces}× — auto-merge giving up to avoid infinite retry loop. See follow-up task for investigation.`,
});
const followUpDescription =
`Investigate repeated ${failedKind} verification failure on ${taskId} (${taskOnErr.title || "untitled"}). ` +
`Auto-merge attempted to fix and re-verify ${nextBounces} times without success — likely a flaky test or unrelated regression rather than a fix this task can produce on its own. ` +
`Look at the most recent [verification] log entries on ${taskId} for the failing command and output, then either fix the underlying issue or quarantine the flake.`;
const followUp = await store.createTask({
description: followUpDescription,
column: "triage",
priority: "high",
});
await store.addTaskComment(
taskId,
`Auto-merge giving up after ${nextBounces} verification-failure bounces. Created follow-up ${followUp.id} to investigate.`,
"agent",
);
await store.logEntry(
taskId,
`Auto-merge gave up after ${nextBounces} verification-failure bounces — created follow-up ${followUp.id}`,
"VerificationError",
);
runtimeLog.warn(
`Auto-merge: ${taskId} hit verification-failure cap (${nextBounces}/${cap}) — failed task and created follow-up ${followUp.id}`,
);
} catch (followUpErr) {
runtimeLog.error(
`Auto-merge: failed to fail-and-followup ${taskId} after verification cap: ${followUpErr instanceof Error ? followUpErr.message : String(followUpErr)}`,
);
}
continue;
}
// Under cap — bounce back as before, but record the increment.
try {
await store.addTaskComment(
taskId,
`Deterministic ${failedKind} verification failed during merge. ` +
`Deterministic ${failedKind} verification failed during merge (attempt ${nextBounces}/${cap}). ` +
`See the prior [verification] log entry for the truncated command output. ` +
`Please fix the failing ${failedKind} and push the update so the merge can retry.`,
"agent",
);
await store.updateTask(taskId, { status: null, mergeRetries: 0, error: null });
await store.updateTask(taskId, {
status: null,
mergeRetries: 0,
error: null,
verificationFailureCount: nextBounces,
});
await store.moveTask(taskId, "in-progress");
await store.logEntry(
taskId,
`Deterministic ${failedKind} verification failed — moved back to in-progress for remediation`,
`Deterministic ${failedKind} verification failed (${nextBounces}/${cap}) — moved back to in-progress for remediation`,
);
runtimeLog.log(
`Auto-merge: ${taskId} deterministic ${failedKind} verification failed — moved to in-progress`,
`Auto-merge: ${taskId} deterministic ${failedKind} verification failed (${nextBounces}/${cap}) — moved to in-progress`,
);
} catch {
runtimeLog.error(

78
packages/engine/src/self-healing.ts
View File

@@ -15,11 +15,11 @@
import { exec } from "node:child_process";
import { promisify } from "node:util";
import { existsSync, readdirSync, statSync } from "node:fs";
import { join } from "node:path";
import { existsSync, readdirSync, rmSync, statSync } from "node:fs";
import { isAbsolute, join, relative, resolve } from "node:path";
import { getTaskMergeBlocker, type TaskStore, type Settings, type Task, type MergeDetails } from "@fusion/core";
import { createLogger } from "./logger.js";
import { scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
import { getRegisteredWorktreePaths, scanIdleWorktrees, scanOrphanedBranches } from "./worktree-pool.js";
const log = createLogger("self-healing");
const execAsync = promisify(exec);
@@ -1463,18 +1463,30 @@ export class SelfHealingManager {
}
}
/** Remove orphaned worktrees not assigned to any active task. */
/**
* Remove orphaned worktrees not assigned to any active task.
*
* When `recycleWorktrees` is OFF: removes registered idle worktrees too —
* they would otherwise pile up since the pool isn't keeping them.
*
* When `recycleWorktrees` is ON: leaves registered idle worktrees alone
* (the pool wants them for reuse) but still reaps unregistered stale dirs
* left behind by killed runs (e.g., `clear-hawk-broken`, `*-bak`). Those
* dirs can never be recycled — they aren't git worktrees — so they only
* waste disk.
*/
private async cleanupOrphans(): Promise<number> {
try {
const settings = await this.store.getSettings();
if (settings.recycleWorktrees) {
// Recycle on: only sweep unregistered stale dirs.
return await this.reapUnregisteredOrphans();
}
const orphaned = await scanIdleWorktrees(this.options.rootDir, this.store);
if (orphaned.length === 0) return 0;
// Only clean up if recycling is disabled — otherwise they belong in the pool
const settings = await this.store.getSettings();
if (settings.recycleWorktrees) {
return 0;
}
let cleaned = 0;
for (const worktreePath of orphaned) {
try {
@@ -1500,6 +1512,52 @@ export class SelfHealingManager {
}
}
/**
* Sweep unregistered stale directories under `<rootDir>/.worktrees/` —
* directories that exist on disk but are NOT registered git worktrees.
* Safe to run alongside `recycleWorktrees: true` because the pool only
* tracks registered idle worktrees, never these orphans.
*/
private async reapUnregisteredOrphans(): Promise<number> {
const worktreesDir = join(this.options.rootDir, ".worktrees");
if (!existsSync(worktreesDir)) return 0;
let dirs: string[];
try {
dirs = readdirSync(worktreesDir, { withFileTypes: true })
.filter((e) => e.isDirectory())
.map((e) => join(worktreesDir, e.name));
} catch (err: unknown) {
log.warn(`Failed to read .worktrees/ for unregistered orphan reap: ${err instanceof Error ? err.message : String(err)}`);
return 0;
}
if (dirs.length === 0) return 0;
const registered = await getRegisteredWorktreePaths(this.options.rootDir);
const unregistered = dirs.filter((d) => !registered.has(resolve(d)));
let cleaned = 0;
for (const path of unregistered) {
const rel = relative(worktreesDir, path);
if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) {
log.warn(`Refusing to remove path outside .worktrees: ${path}`);
continue;
}
try {
rmSync(path, { recursive: true, force: true });
log.log(`Cleaned unregistered worktree dir: ${path}`);
cleaned++;
} catch (err: unknown) {
log.warn(`Failed to remove unregistered worktree dir ${path}: ${err instanceof Error ? err.message : String(err)}`);
}
}
if (cleaned > 0) {
log.log(`Cleaned ${cleaned} unregistered worktree dir(s) (recycle mode preserves registered idle worktrees)`);
}
return cleaned;
}
/**
* Remove orphaned `fusion/*` branches that are not associated with any
* active (non-archived, non-merger-managed) task.