FN-6188: harden AI merge temp worktree cleanup
Improve AI merge temp worktree cleanup and add recovery coverage. - audit AI merge clean-room cleanup success and failure paths, including stderr/code details - sweep stale fusion-ai-merge temp directories from tmpdir() during self-healing with active-session safeguards - document the native temp-dir sweep behavior and add merge cleanup/self-healing regression tests Files changed: docs/architecture.md | 1 + .../engine/src/__tests__/merger-ai-cleanup.test.ts | 169 +++++++++++++++++ .../__tests__/self-healing-tempdir-sweep.test.ts | 200 +++++++++++++++++++++ packages/engine/src/merger-ai.ts | 54 +++++- packages/engine/src/run-audit.ts | 26 +++ packages/engine/src/self-healing.ts | 105 +++++++++++ 6 files changed, 551 insertions(+), 4 deletions(-) Fusion-Task-Id: FN-6188 Fusion-Task-Lineage: eae9bcdc-9765-4104-85dc-81bb50445e99
This commit is contained in:
gsxdsm
@@ -670,6 +670,7 @@ Runtime action-gate flow (v1):
|
||||
- `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification
|
||||
- `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions
|
||||
- Batch 1 maintenance now includes one `fts-maintenance` step for both search indexes. The live `tasks_fts` branch still runs `merge` every tick, `optimize` every 4th tick, and `rebuild` above `32 MiB` or `1 MiB × live task count`. The archive `archived_tasks_fts` branch is lighter because archive writes are mostly append-only: `merge` every 8th tick, `optimize` every 24th tick, and `rebuild` above `64 MiB` or `512 KiB × archived row count`. Each branch is independently guarded by `fts5Available` and emits `task:fts-maintenance` run-audit telemetry with distinct `target` values (`tasks_fts` vs `archived_tasks_fts`).
|
||||
- Batch 1 also sweeps stale AI merge clean-room worktrees under `tmpdir()` whose names start with `fusion-ai-merge-`. The sweep only considers directories older than 2 hours, canonicalizes paths before checking `activeSessionRegistry`, attempts `git worktree remove --force <path>` before filesystem removal, and emits `worktree:tempdir-sweep` run-audit telemetry for removal attempts and failures. It is intentionally native even when `worktrunk.enabled` because these temp-dir worktrees are outside the worktrunk-managed project layout. Fresh directories, active-session paths, and individual removal failures are skipped/logged without aborting the maintenance cycle.
|
||||
- `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`.
|
||||
- Mission validation has a dedicated stale-run reaper: startup recovery and Batch 2 maintenance call `reapStaleMissionValidatorRuns()` when wired by the runtime, using `VALIDATOR_RUN_STALE_MAX_AGE_MS` (currently 6 hours). The sweep terminates ownerless `mission_validator_runs.status='running'` rows as `error`, writes the reap reason into `summary`, leaves `lastValidatorRunId` pointing at the now-terminal run, and emits run-audit telemetry with `mutationType: "mission:validator-run-reaped"` plus `runId`/`featureId`/`missionId`/`triggerType`/`elapsedMs` metadata. Active mission features move to `loopState="needs_fix"` + `lastValidatorStatus="error"` unless their parent mission is already `complete`/`archived`.
|
||||
|
||||
|
||||
169
packages/engine/src/__tests__/merger-ai-cleanup.test.ts
Normal file
169
packages/engine/src/__tests__/merger-ai-cleanup.test.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { execSync } from "node:child_process";
|
||||
import { cleanupAiMergeWorktree, runAiMerge } from "../merger-ai.js";
|
||||
import type { RunAuditor } from "../run-audit.js";
|
||||
|
||||
const tracked = new Set<string>();
|
||||
const RM = { recursive: true, force: true, maxRetries: 5, retryDelay: 50 } as const;
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
for (const dir of tracked) {
|
||||
try { rmSync(dir, RM); } catch { /* best effort */ }
|
||||
}
|
||||
tracked.clear();
|
||||
});
|
||||
|
||||
function git(cwd: string, args: string): string {
|
||||
return execSync(`git ${args}`, { cwd, encoding: "utf-8" }).trim();
|
||||
}
|
||||
|
||||
function makeAudit() {
|
||||
const events: any[] = [];
|
||||
const audit: RunAuditor = {
|
||||
git: vi.fn(async (event: any) => { events.push(event); }),
|
||||
database: vi.fn(async () => undefined),
|
||||
filesystem: vi.fn(async () => undefined),
|
||||
sandbox: vi.fn(async () => undefined),
|
||||
};
|
||||
return { audit, events };
|
||||
}
|
||||
|
||||
async function cleanup(input: Partial<Parameters<typeof cleanupAiMergeWorktree>[0]> = {}) {
|
||||
const mergeRoot = input.mergeRoot ?? mkdtempSync(join(tmpdir(), "fusion-ai-merge-fn-1-cleanup-test-"));
|
||||
tracked.add(mergeRoot);
|
||||
const { audit, events } = makeAudit();
|
||||
const logs: string[] = [];
|
||||
await cleanupAiMergeWorktree({
|
||||
taskId: "FN-1",
|
||||
mergeRoot,
|
||||
projectRootDir: input.projectRootDir ?? process.cwd(),
|
||||
worktreeAdded: input.worktreeAdded ?? true,
|
||||
audit: input.audit ?? audit,
|
||||
log: input.log ?? vi.fn(async (message: string) => { logs.push(message); }),
|
||||
gitRunner: input.gitRunner ?? vi.fn(async () => ""),
|
||||
rmRunner: input.rmRunner ?? rm,
|
||||
});
|
||||
return { mergeRoot, events, logs };
|
||||
}
|
||||
|
||||
function initRepoWithBranch(): { dir: string } {
|
||||
const dir = mkdtempSync(join(tmpdir(), "fusion-ai-merge-cleanup-test-"));
|
||||
tracked.add(dir);
|
||||
git(dir, "init -q -b main");
|
||||
git(dir, "config user.email t@t.t");
|
||||
git(dir, "config user.name t");
|
||||
writeFileSync(join(dir, "base.txt"), "base\n");
|
||||
git(dir, "add -A");
|
||||
git(dir, "commit -q -m base");
|
||||
git(dir, "checkout -q -b fusion/fn-1");
|
||||
writeFileSync(join(dir, "feature.txt"), "feature work\n");
|
||||
git(dir, "add -A");
|
||||
git(dir, "commit -q -m 'feat: work'");
|
||||
git(dir, "checkout -q main");
|
||||
return { dir };
|
||||
}
|
||||
|
||||
function makeStore() {
|
||||
const task: any = {
|
||||
id: "FN-1",
|
||||
column: "in-review",
|
||||
status: null,
|
||||
branch: "fusion/fn-1",
|
||||
worktree: null,
|
||||
title: "do the thing",
|
||||
steps: [],
|
||||
};
|
||||
const audits: any[] = [];
|
||||
const logs: string[] = [];
|
||||
const store: any = {
|
||||
getTask: vi.fn(async () => task),
|
||||
getSettings: vi.fn(async () => ({ merger: { mode: "ai", maxReviewPasses: 1 } })),
|
||||
updateTask: vi.fn(async (_id: string, patch: Record<string, unknown>) => { Object.assign(task, patch); return task; }),
|
||||
moveTask: vi.fn(async (_id: string, column: string) => { task.column = column; return task; }),
|
||||
emit: vi.fn(),
|
||||
logEntry: vi.fn(async (_id: string, message: string) => { logs.push(message); }),
|
||||
appendAgentLog: vi.fn(async (_id: string, message: string) => { logs.push(message); }),
|
||||
recordRunAuditEvent: vi.fn(async (event: any) => { audits.push(event); }),
|
||||
};
|
||||
return { store, audits, logs };
|
||||
}
|
||||
|
||||
function realMergeAgent() {
|
||||
return vi.fn(async (cwd: string) => {
|
||||
execSync("git merge --squash fusion/fn-1", { cwd, stdio: "pipe" });
|
||||
execSync("git add -A", { cwd, stdio: "pipe" });
|
||||
execSync('git commit -q -m "squash: feature"', { cwd, stdio: "pipe" });
|
||||
});
|
||||
}
|
||||
|
||||
describe("AI merge temp worktree cleanup", () => {
|
||||
it("emits audit event and logs stderr on git worktree removal failure", async () => {
|
||||
const err = new Error("git remove failed") as Error & { stderr?: string; code?: string };
|
||||
err.stderr = "fatal: simulated worktree remove failure";
|
||||
err.code = "1";
|
||||
|
||||
const { mergeRoot, events, logs } = await cleanup({ gitRunner: vi.fn(async () => { throw err; }) });
|
||||
|
||||
expect(events).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "git-remove", success: false, error: expect.stringContaining("simulated worktree remove failure"), code: "1" }) }),
|
||||
expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "fs-rm", success: true }) }),
|
||||
]));
|
||||
expect(logs.join("\n")).toContain("simulated worktree remove failure");
|
||||
expect(existsSync(mergeRoot)).toBe(false);
|
||||
});
|
||||
|
||||
it("emits audit event and logs errno details on filesystem rm failure", async () => {
|
||||
const err = new Error("simulated filesystem cleanup denial") as NodeJS.ErrnoException;
|
||||
err.code = "EACCES";
|
||||
|
||||
const { events, logs } = await cleanup({ rmRunner: vi.fn(async () => { throw err; }) as typeof rm });
|
||||
|
||||
expect(events).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "git-remove", success: true }) }),
|
||||
expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "fs-rm", success: false, code: "EACCES", error: expect.stringContaining("simulated filesystem cleanup denial") }) }),
|
||||
]));
|
||||
expect(logs.join("\n")).toContain("EACCES");
|
||||
});
|
||||
|
||||
it("emits success audit events on happy-path cleanup", async () => {
|
||||
const { events } = await cleanup();
|
||||
|
||||
expect(events).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "git-remove", success: true }) }),
|
||||
expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "fs-rm", success: true }) }),
|
||||
]));
|
||||
});
|
||||
|
||||
it("skips git removal but still audits filesystem cleanup when worktree was not added", async () => {
|
||||
const gitRunner = vi.fn(async () => "");
|
||||
|
||||
const { events } = await cleanup({ worktreeAdded: false, gitRunner });
|
||||
|
||||
expect(gitRunner).not.toHaveBeenCalled();
|
||||
expect(events.some((event) => event.metadata.phase === "git-remove")).toBe(false);
|
||||
expect(events).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ type: "merge:ai-worktree-cleanup", metadata: expect.objectContaining({ phase: "fs-rm", success: true }) }),
|
||||
]));
|
||||
});
|
||||
|
||||
it("runAiMerge emits success cleanup audit events", async () => {
|
||||
const { dir } = initRepoWithBranch();
|
||||
const { store, audits } = makeStore();
|
||||
|
||||
await runAiMerge(store, dir, "FN-1", { manual: true }, {
|
||||
mergeAgent: realMergeAgent(),
|
||||
reviewAgent: vi.fn(async () => "REVIEW_VERDICT: approve"),
|
||||
});
|
||||
|
||||
const cleanupEvents = audits.filter((event) => event.mutationType === "merge:ai-worktree-cleanup");
|
||||
expect(cleanupEvents).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ metadata: expect.objectContaining({ phase: "git-remove", success: true }) }),
|
||||
expect.objectContaining({ metadata: expect.objectContaining({ phase: "fs-rm", success: true }) }),
|
||||
]));
|
||||
});
|
||||
});
|
||||
200
packages/engine/src/__tests__/self-healing-tempdir-sweep.test.ts
Normal file
200
packages/engine/src/__tests__/self-healing-tempdir-sweep.test.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, utimesSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const osState = vi.hoisted(() => ({ tempRoot: "" }));
|
||||
const fsState = vi.hoisted(() => ({ failRmPath: "", rmCalls: [] as string[] }));
|
||||
const childState = vi.hoisted(() => ({ execCalls: [] as string[] }));
|
||||
|
||||
vi.mock("node:os", async () => {
|
||||
const actual = await vi.importActual<typeof import("node:os")>("node:os");
|
||||
return { ...actual, tmpdir: vi.fn(() => osState.tempRoot || actual.tmpdir()) };
|
||||
});
|
||||
|
||||
vi.mock("node:fs", async () => {
|
||||
const actual = await vi.importActual<typeof import("node:fs")>("node:fs");
|
||||
return {
|
||||
...actual,
|
||||
rmSync: vi.fn((path: Parameters<typeof actual.rmSync>[0], options?: Parameters<typeof actual.rmSync>[1]) => {
|
||||
const pathString = String(path);
|
||||
fsState.rmCalls.push(pathString);
|
||||
if (fsState.failRmPath && pathString === fsState.failRmPath) {
|
||||
const err = new Error("simulated tempdir rm failure") as NodeJS.ErrnoException;
|
||||
err.code = "EACCES";
|
||||
throw err;
|
||||
}
|
||||
return actual.rmSync(path, options);
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("node:child_process", async () => {
|
||||
const actual = await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
return {
|
||||
...actual,
|
||||
exec: vi.fn((command: string, optionsOrCallback: unknown, maybeCallback?: unknown) => {
|
||||
childState.execCalls.push(command);
|
||||
const callback = typeof optionsOrCallback === "function" ? optionsOrCallback : maybeCallback;
|
||||
queueMicrotask(() => {
|
||||
if (typeof callback === "function") callback(null, "", "");
|
||||
});
|
||||
return {} as ReturnType<typeof actual.exec>;
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
import { activeSessionRegistry } from "../active-session-registry.js";
|
||||
import { SelfHealingManager } from "../self-healing.js";
|
||||
|
||||
const RM = { recursive: true, force: true, maxRetries: 5, retryDelay: 50 } as const;
|
||||
let sandboxRoot = "";
|
||||
let projectRoot = "";
|
||||
|
||||
beforeEach(() => {
|
||||
sandboxRoot = mkdtempSync(join(tmpdir(), "fusion-tempdir-sweep-sandbox-"));
|
||||
projectRoot = mkdtempSync(join(tmpdir(), "fusion-tempdir-sweep-project-"));
|
||||
osState.tempRoot = sandboxRoot;
|
||||
fsState.failRmPath = "";
|
||||
fsState.rmCalls = [];
|
||||
childState.execCalls = [];
|
||||
activeSessionRegistry.clear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
activeSessionRegistry.clear();
|
||||
osState.tempRoot = "";
|
||||
fsState.failRmPath = "";
|
||||
fsState.rmCalls = [];
|
||||
childState.execCalls = [];
|
||||
for (const dir of [sandboxRoot, projectRoot]) {
|
||||
try { rmSync(dir, RM); } catch { /* best effort */ }
|
||||
}
|
||||
});
|
||||
|
||||
function makeStore(settings: Record<string, unknown> = {}) {
|
||||
const audits: any[] = [];
|
||||
const store: any = {
|
||||
getSettings: vi.fn(async () => ({ ...settings })),
|
||||
recordRunAuditEvent: vi.fn(async (event: any) => { audits.push(event); }),
|
||||
};
|
||||
return { store, audits };
|
||||
}
|
||||
|
||||
function makeManager(settings: Record<string, unknown> = {}) {
|
||||
const { store, audits } = makeStore(settings);
|
||||
const manager = new SelfHealingManager(store, { rootDir: projectRoot });
|
||||
return { manager, audits };
|
||||
}
|
||||
|
||||
function tempMergeDir(name = `fusion-ai-merge-fn-1-${Math.random().toString(36).slice(2)}`): string {
|
||||
const dir = join(sandboxRoot, name);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
function makeStale(path: string): void {
|
||||
const old = new Date(Date.now() - 3 * 60 * 60 * 1000);
|
||||
utimesSync(path, old, old);
|
||||
}
|
||||
|
||||
async function sweep(manager: SelfHealingManager): Promise<number> {
|
||||
return await (manager as any).cleanupStaleTempMergeWorktrees();
|
||||
}
|
||||
|
||||
function sweepAudits(audits: any[]) {
|
||||
return audits.filter((event) => event.mutationType === "worktree:tempdir-sweep");
|
||||
}
|
||||
|
||||
describe("SelfHealingManager temp-dir AI merge worktree sweep", () => {
|
||||
it("removes stale fusion-ai-merge directories and emits success audits", async () => {
|
||||
const stale = tempMergeDir();
|
||||
makeStale(stale);
|
||||
const { manager, audits } = makeManager();
|
||||
|
||||
await expect(sweep(manager)).resolves.toBe(1);
|
||||
|
||||
expect(existsSync(stale)).toBe(false);
|
||||
expect(sweepAudits(audits)).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ mutationType: "worktree:tempdir-sweep", metadata: expect.objectContaining({ path: realpathSync(sandboxRoot) + "/" + stale.split("/").pop(), success: true, reason: "stale" }) }),
|
||||
]));
|
||||
});
|
||||
|
||||
it("skips directories younger than the staleness threshold", async () => {
|
||||
const fresh = tempMergeDir();
|
||||
const { manager } = makeManager();
|
||||
|
||||
await expect(sweep(manager)).resolves.toBe(0);
|
||||
|
||||
expect(existsSync(fresh)).toBe(true);
|
||||
});
|
||||
|
||||
it("skips active session paths and removes them after unregister", async () => {
|
||||
const stale = tempMergeDir();
|
||||
makeStale(stale);
|
||||
const canonical = realpathSync(stale);
|
||||
activeSessionRegistry.registerPath(canonical, { taskId: "FN-1", kind: "executor", ownerKey: "FN-1" });
|
||||
const { manager, audits } = makeManager();
|
||||
|
||||
await expect(sweep(manager)).resolves.toBe(0);
|
||||
expect(existsSync(stale)).toBe(true);
|
||||
expect(sweepAudits(audits)).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ metadata: expect.objectContaining({ path: canonical, success: false, reason: "active-session" }) }),
|
||||
]));
|
||||
|
||||
activeSessionRegistry.unregisterPath(canonical);
|
||||
await expect(sweep(manager)).resolves.toBe(1);
|
||||
expect(existsSync(stale)).toBe(false);
|
||||
});
|
||||
|
||||
it("leaves non-fusion-ai-merge directories untouched", async () => {
|
||||
const other = join(sandboxRoot, "other-temp-dir");
|
||||
mkdirSync(other, { recursive: true });
|
||||
makeStale(other);
|
||||
const { manager } = makeManager();
|
||||
|
||||
await expect(sweep(manager)).resolves.toBe(0);
|
||||
|
||||
expect(existsSync(other)).toBe(true);
|
||||
});
|
||||
|
||||
it("attempts git worktree removal before filesystem removal", async () => {
|
||||
const stale = tempMergeDir();
|
||||
makeStale(stale);
|
||||
const canonical = realpathSync(stale);
|
||||
const { manager } = makeManager();
|
||||
|
||||
await expect(sweep(manager)).resolves.toBe(1);
|
||||
|
||||
expect(childState.execCalls[0]).toContain(`git worktree remove --force '${canonical.replace(/'/g, `'"'"'`)}'`);
|
||||
expect(fsState.rmCalls[0]).toBe(canonical);
|
||||
});
|
||||
|
||||
it("continues when one stale directory fails filesystem removal", async () => {
|
||||
const failing = tempMergeDir("fusion-ai-merge-fn-1-failing");
|
||||
const succeeding = tempMergeDir("fusion-ai-merge-fn-1-succeeding");
|
||||
makeStale(failing);
|
||||
makeStale(succeeding);
|
||||
fsState.failRmPath = realpathSync(failing);
|
||||
const { manager, audits } = makeManager();
|
||||
|
||||
await expect(sweep(manager)).resolves.toBe(1);
|
||||
|
||||
expect(existsSync(failing)).toBe(true);
|
||||
expect(existsSync(succeeding)).toBe(false);
|
||||
expect(sweepAudits(audits)).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ metadata: expect.objectContaining({ path: realpathSync(failing), success: false, reason: "fs-rm-failed", error: expect.stringContaining("simulated tempdir rm failure") }) }),
|
||||
expect.objectContaining({ metadata: expect.objectContaining({ path: expect.stringContaining("succeeding"), success: true, reason: "stale" }) }),
|
||||
]));
|
||||
});
|
||||
|
||||
it("proceeds when worktrunk is enabled", async () => {
|
||||
const stale = tempMergeDir();
|
||||
makeStale(stale);
|
||||
const { manager } = makeManager({ worktrunk: { enabled: true } });
|
||||
|
||||
await expect(sweep(manager)).resolves.toBe(1);
|
||||
|
||||
expect(existsSync(stale)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -87,6 +87,55 @@ async function gitOk(args: string[], cwd: string): Promise<boolean> {
|
||||
}
|
||||
}
|
||||
|
||||
function getErrorMessage(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
|
||||
function getErrorStringProperty(err: unknown, key: "stderr" | "code"): string | undefined {
|
||||
if (!err || typeof err !== "object" || !(key in err)) return undefined;
|
||||
const value = (err as Record<string, unknown>)[key];
|
||||
return typeof value === "string" && value.trim() ? value : undefined;
|
||||
}
|
||||
|
||||
function describeCleanupError(err: unknown): string {
|
||||
const stderr = getErrorStringProperty(err, "stderr");
|
||||
const message = getErrorMessage(err);
|
||||
return stderr ? `${message}: ${stderr.trim()}` : message;
|
||||
}
|
||||
|
||||
export async function cleanupAiMergeWorktree(input: {
|
||||
taskId: string;
|
||||
mergeRoot: string;
|
||||
projectRootDir: string;
|
||||
worktreeAdded: boolean;
|
||||
audit: RunAuditor;
|
||||
log: (message: string) => Promise<void>;
|
||||
gitRunner?: typeof git;
|
||||
rmRunner?: typeof rm;
|
||||
}): Promise<void> {
|
||||
const { taskId, mergeRoot, projectRootDir, worktreeAdded, audit, log, gitRunner = git, rmRunner = rm } = input;
|
||||
if (worktreeAdded) {
|
||||
try {
|
||||
await gitRunner(["worktree", "remove", "--force", mergeRoot], projectRootDir);
|
||||
await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "git-remove", success: true } });
|
||||
} catch (err: unknown) {
|
||||
const error = describeCleanupError(err);
|
||||
const code = getErrorStringProperty(err, "code");
|
||||
await log(`AI merge cleanup: git worktree remove failed for ${mergeRoot}${code ? ` (${code})` : ""}: ${error}`);
|
||||
await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "git-remove", success: false, error, ...(code ? { code } : {}) } });
|
||||
}
|
||||
}
|
||||
try {
|
||||
await rmRunner(mergeRoot, { recursive: true, force: true });
|
||||
await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "fs-rm", success: true } });
|
||||
} catch (err: unknown) {
|
||||
const error = getErrorMessage(err);
|
||||
const code = getErrorStringProperty(err, "code");
|
||||
await log(`AI merge cleanup: filesystem rm failed for ${mergeRoot}${code ? ` (${code})` : ""}: ${error}`);
|
||||
await audit.git({ type: "merge:ai-worktree-cleanup", target: mergeRoot, metadata: { taskId, mergeRoot, phase: "fs-rm", success: false, error, ...(code ? { code } : {}) } });
|
||||
}
|
||||
}
|
||||
|
||||
const FUSION_TASK_ID_TRAILER_KEY = "Fusion-Task-Id";
|
||||
|
||||
/** Trailers that associate the squash commit with its board task: the
|
||||
@@ -836,10 +885,7 @@ export async function runAiMerge(
|
||||
await log(`AI merge: advanced ${integrationBranch} → ${short(squashSha)} (local checkout: ${landed.localSync})`);
|
||||
return await finalizeMerged(store, projectRootDir, taskId, task, branch, integrationBranch, squashSha, audit, log, { empty: false });
|
||||
} finally {
|
||||
if (worktreeAdded) {
|
||||
await gitOk(["worktree", "remove", "--force", mergeRoot], projectRootDir);
|
||||
}
|
||||
await rm(mergeRoot, { recursive: true, force: true }).catch(() => undefined);
|
||||
await cleanupAiMergeWorktree({ taskId, mergeRoot, projectRootDir, worktreeAdded, audit, log });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,6 +167,32 @@ export type GitMutationType =
|
||||
| "merge:ai-review-landed-with-concerns"
|
||||
| "merge:ai-local-sync"
|
||||
| "merge:ai-landed"
|
||||
/**
|
||||
* Metadata shape:
|
||||
* ```ts
|
||||
* {
|
||||
* taskId: string;
|
||||
* mergeRoot: string;
|
||||
* phase: "git-remove" | "fs-rm";
|
||||
* success: boolean;
|
||||
* error?: string;
|
||||
* code?: string;
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
| "merge:ai-worktree-cleanup"
|
||||
/**
|
||||
* Metadata shape:
|
||||
* ```ts
|
||||
* {
|
||||
* path: string;
|
||||
* success: boolean;
|
||||
* reason?: "stale" | "active-session" | "git-remove-failed" | "fs-rm-failed" | "not-directory" | "stat-failed";
|
||||
* error?: string;
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
| "worktree:tempdir-sweep"
|
||||
| "merge:reuse-handoff-acquired"
|
||||
| "merge:reuse-handoff-refused"
|
||||
| "merge:reuse-handoff-released"
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
* - `pruneWorktrees`: defer to backend prune
|
||||
* - `cleanupOrphans`: defer to backend prune/remove semantics
|
||||
* - `reapUnregisteredOrphans`: defer to backend prune/remove semantics
|
||||
* - `cleanupStaleTempMergeWorktrees`: remains native (temp-dir scope, outside worktrunk layout)
|
||||
* - `enforceWorktreeCap`: defer to backend prune/remove semantics
|
||||
* - `reclaimSelfOwnedBranchConflicts`: remains native (branch-level)
|
||||
* - `reclaimStaleActiveBranches`: remains native (branch-level)
|
||||
@@ -27,6 +28,7 @@ import { exec, execSync } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { setImmediate as setImmediateCb } from "node:timers";
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { isAbsolute, join, relative, resolve } from "node:path";
|
||||
import { IN_REVIEW_STALL_DEADLOCK_LOG_PREFIX, IN_REVIEW_STALL_LOG_PREFIX, IN_REVIEW_STALL_TERMINAL_LOG_PREFIX, allowsAutoMergeProcessing, countRecentIdenticalStallEntries, detectDependencyCycle, detectSelfDefeatingDependency, getInReviewStalledSignal, getInReviewStallReason, getPrimaryPrInfo, getStalePausedReviewSignal, getStalePausedTodoSignal, getTaskHardMergeBlocker, getTaskMergeBlocker, isEphemeralAgent, isMergeRequestContractShadowEnabled, isWorkflowColumnsEnabled, isSharedBranchGroupMemberIntegration, parseExplicitDuplicateMarker, type AgentStore, type ChatStore, type MessageStore, type TaskStore, type Settings, type Task, type MergeDetails, type TaskPriority, type MergeResult } from "@fusion/core";
|
||||
import type { MeshLeaseManager } from "./mesh-lease-manager.js";
|
||||
@@ -73,6 +75,7 @@ const BOARD_STALL_NOTIFICATION_COOLDOWN_MS = 60 * 60_000;
|
||||
const DB_CORRUPTION_NOTIFICATION_COOLDOWN_MS = 60 * 60 * 1000;
|
||||
const FTS_MAINTENANCE_MERGE_CADENCE_TICKS = 1;
|
||||
const FTS_MAINTENANCE_OPTIMIZE_CADENCE_TICKS = 4;
|
||||
const STALE_TEMP_MERGE_WORKTREE_MS = 2 * 60 * 60 * 1000;
|
||||
// Live pathology peaked around 775 KB/task (~96 MB for ~120 tasks), while a
|
||||
// rebuilt healthy index was ~0.1 MB. Keep the steady-state budget generous but
|
||||
// bounded so sustained text churn heals before segment growth becomes material.
|
||||
@@ -1808,6 +1811,16 @@ export class SelfHealingManager {
|
||||
const batch1Fns: Array<{ name: string; fn: () => Promise<unknown> }> = [
|
||||
{ name: "prune-worktrees", fn: () => this.pruneWorktrees() },
|
||||
{ name: "cleanup-orphans", fn: () => this.cleanupOrphans() },
|
||||
{
|
||||
name: "cleanup-stale-temp-merge-worktrees",
|
||||
fn: async () => {
|
||||
const cleaned = await this.cleanupStaleTempMergeWorktrees();
|
||||
if (cleaned > 0) {
|
||||
log.log(`Cleaned ${cleaned} stale AI merge temp worktree(s)`);
|
||||
}
|
||||
return cleaned;
|
||||
},
|
||||
},
|
||||
{ name: "cleanup-orphaned-branches", fn: () => this.cleanupOrphanedBranches() },
|
||||
{
|
||||
name: "cleanup-old-chats",
|
||||
@@ -8838,6 +8851,98 @@ export class SelfHealingManager {
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sweep stale AI merge clean-room worktrees from `tmpdir()`.
|
||||
*
|
||||
* These worktrees are intentionally outside the project/worktrunk-managed
|
||||
* `.worktrees/` layout, so this native sweep proceeds even when worktrunk is
|
||||
* enabled. Safety is bounded by a two-hour age gate plus active-session checks.
|
||||
*/
|
||||
private async cleanupStaleTempMergeWorktrees(): Promise<number> {
|
||||
try {
|
||||
const settings = await this.store.getSettings();
|
||||
if (settings.worktrunk?.enabled === true) {
|
||||
log.log("[self-healing] temp-dir sweep: worktrunk enabled — AI merge temp worktrees are outside worktrunk's managed layout, proceeding with native sweep");
|
||||
}
|
||||
|
||||
const tempRoot = tmpdir();
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = readdirSync(tempRoot).filter((entry) => entry.startsWith("fusion-ai-merge-"));
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`[self-healing] temp-dir sweep: failed to read ${tempRoot}: ${errorMessage}`);
|
||||
return 0;
|
||||
}
|
||||
if (entries.length === 0) return 0;
|
||||
|
||||
const auditor = createRunAuditor(this.store, {
|
||||
runId: generateSyntheticRunId("self-heal", "tempdir-sweep"),
|
||||
agentId: "self-healing",
|
||||
phase: "tempdir-sweep",
|
||||
});
|
||||
const now = Date.now();
|
||||
let cleaned = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
const path = join(tempRoot, entry);
|
||||
let canonicalPath = path;
|
||||
try {
|
||||
const stat = statSync(path);
|
||||
if (!stat.isDirectory()) {
|
||||
await auditor.git({ type: "worktree:tempdir-sweep", target: path, metadata: { path, success: false, reason: "not-directory" } });
|
||||
continue;
|
||||
}
|
||||
const ageMs = now - stat.mtimeMs;
|
||||
if (ageMs < STALE_TEMP_MERGE_WORKTREE_MS) continue;
|
||||
try {
|
||||
canonicalPath = realpathSync(path);
|
||||
} catch {
|
||||
canonicalPath = path;
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`[self-healing] temp-dir sweep: failed to stat ${path}: ${errorMessage}`);
|
||||
await auditor.git({ type: "worktree:tempdir-sweep", target: path, metadata: { path, success: false, reason: "stat-failed", error: errorMessage } });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (activeSessionRegistry.isPathActive(canonicalPath) || activeSessionRegistry.isPathActive(path)) {
|
||||
log.log(`[self-healing] temp-dir sweep: deferring ${canonicalPath}: active session present`);
|
||||
await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "active-session" } });
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
await execAsync(`git worktree remove --force ${shellQuote(canonicalPath)}`, {
|
||||
cwd: this.options.rootDir,
|
||||
timeout: 120_000,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`[self-healing] temp-dir sweep: git worktree remove failed for ${canonicalPath}: ${errorMessage} — falling back to filesystem removal`);
|
||||
await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "git-remove-failed", error: errorMessage } });
|
||||
}
|
||||
|
||||
try {
|
||||
rmSync(canonicalPath, { recursive: true, force: true });
|
||||
log.log(`[self-healing] temp-dir sweep: cleaned stale AI merge worktree ${canonicalPath}`);
|
||||
await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: true, reason: "stale" } });
|
||||
cleaned++;
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`[self-healing] temp-dir sweep: failed to remove ${canonicalPath}: ${errorMessage}`);
|
||||
await auditor.git({ type: "worktree:tempdir-sweep", target: canonicalPath, metadata: { path: canonicalPath, success: false, reason: "fs-rm-failed", error: errorMessage } });
|
||||
}
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
} catch (err: unknown) {
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
log.error(`[self-healing] temp-dir sweep failed: ${errorMessage}`);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve orphaned `fusion/*` branches.
|
||||
|
||||
Reference in New Issue
Block a user