feat(FN-3827): sweep stale autostashes after merge completion
Merger now properly cleans up autostash git refs that survive past their useful lifetime, fixing race conditions during restore/cleanup and adding scheduled stale-sweep runs in the engine. The feature includes docs for the autostash lifecycle, new test coverage for the cleanup and sweep paths, and a Fusion-Task-Id: FN-3827
This commit is contained in:
@@ -6,7 +6,7 @@ import { execSync } from "node:child_process";
|
||||
import type { TaskStore } from "@fusion/core";
|
||||
import { __test__ } from "../merger.js";
|
||||
|
||||
const { sweepAutostashOrphans, parseAutostashTaskId } = __test__;
|
||||
const { sweepAutostashOrphans, parseAutostashTaskId, sweepStaleAutostashes, dropAutostashHandle } = __test__;
|
||||
|
||||
function git(cwd: string, cmd: string): string {
|
||||
return execSync(cmd, { cwd, stdio: "pipe" }).toString("utf-8").trim();
|
||||
@@ -65,6 +65,82 @@ describe("parseAutostashTaskId", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("sweepStaleAutostashes", () => {
|
||||
let dir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), "fn-autostash-stale-"));
|
||||
initRepo(dir);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("drops stale autostashes older than maxAgeMs", async () => {
|
||||
const oldTs = Date.now() - 26 * 60 * 60 * 1000;
|
||||
const staleLabel = `fusion-merger-autostash:FN-5001:${oldTs}`;
|
||||
writeFileSync(join(dir, "file.txt"), "stale\n");
|
||||
git(dir, `git stash push -m ${JSON.stringify(staleLabel)} file.txt`);
|
||||
|
||||
const res = await sweepStaleAutostashes(dir, { maxAgeMs: 24 * 60 * 60 * 1000 });
|
||||
|
||||
expect(res.dropped).toBe(1);
|
||||
expect(stashList(dir)).not.toContain("fusion-merger-autostash:FN-5001");
|
||||
});
|
||||
|
||||
it("keeps recent autostashes within maxAgeMs", async () => {
|
||||
const freshLabel = `fusion-merger-autostash:FN-5002:${Date.now()}`;
|
||||
writeFileSync(join(dir, "file.txt"), "fresh\n");
|
||||
git(dir, `git stash push -m ${JSON.stringify(freshLabel)} file.txt`);
|
||||
|
||||
const res = await sweepStaleAutostashes(dir, { maxAgeMs: 24 * 60 * 60 * 1000 });
|
||||
|
||||
expect(res.dropped).toBe(0);
|
||||
expect(stashList(dir)).toContain("fusion-merger-autostash:FN-5002");
|
||||
});
|
||||
|
||||
it("ignores non-fusion stash labels", async () => {
|
||||
writeFileSync(join(dir, "file.txt"), "manual\n");
|
||||
git(dir, "git stash push -m \"manual\" file.txt");
|
||||
|
||||
const res = await sweepStaleAutostashes(dir, { maxAgeMs: 1 });
|
||||
|
||||
expect(res.dropped).toBe(0);
|
||||
expect(stashList(dir)).toContain("manual");
|
||||
});
|
||||
|
||||
it("tolerates malformed autostash labels", async () => {
|
||||
const malformed = "fusion-merger-autostash:FN-5003:not-a-timestamp";
|
||||
writeFileSync(join(dir, "file.txt"), "bad\n");
|
||||
git(dir, `git stash push -m ${JSON.stringify(malformed)} file.txt`);
|
||||
|
||||
await expect(sweepStaleAutostashes(dir, { maxAgeMs: 1 })).resolves.toEqual({ dropped: 0 });
|
||||
expect(stashList(dir)).toContain("not-a-timestamp");
|
||||
});
|
||||
|
||||
it("dropAutostashHandle drops primary and rescue shas", async () => {
|
||||
const pLabel = `fusion-merger-autostash:FN-5004:${Date.now() - 1000}`;
|
||||
writeFileSync(join(dir, "file.txt"), "primary\n");
|
||||
git(dir, `git stash push -m ${JSON.stringify(pLabel)} file.txt`);
|
||||
const primarySha = git(dir, 'git stash list --format="%H" -n 1');
|
||||
|
||||
const rLabel = `fusion-merger-autostash:FN-5004:race-rescue-0:${Date.now() - 500}`;
|
||||
writeFileSync(join(dir, "file.txt"), "rescue\n");
|
||||
git(dir, `git stash push -m ${JSON.stringify(rLabel)} file.txt`);
|
||||
const rescueSha = git(dir, 'git stash list --format="%H" -n 1');
|
||||
|
||||
const result = await dropAutostashHandle(dir, "FN-5004", {
|
||||
sha: primarySha,
|
||||
label: pLabel,
|
||||
rescueShas: [{ sha: rescueSha, label: rLabel }],
|
||||
}, { keepIfLive: false });
|
||||
|
||||
expect(result.dropped).toBe(2);
|
||||
expect(stashList(dir)).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("sweepAutostashOrphans", () => {
|
||||
let dir: string;
|
||||
|
||||
|
||||
@@ -486,6 +486,61 @@ describe("aiMergeTask abort handling", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("aiMergeTask autostash cleanup", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockedExistsSync.mockReturnValue(true);
|
||||
mockedCreateFnAgent.mockResolvedValue({
|
||||
session: {
|
||||
prompt: vi.fn().mockResolvedValue(undefined),
|
||||
dispose: vi.fn(),
|
||||
},
|
||||
} as any);
|
||||
});
|
||||
|
||||
it("drops task autostash after successful merge restore", async () => {
|
||||
const store = createMockStore({ id: "FN-050", worktree: "/tmp/root/.worktrees/KB-050" });
|
||||
const stashSha = "1111111111111111111111111111111111111111";
|
||||
let dropped = false;
|
||||
|
||||
mockedExecSync.mockImplementation((cmd: any) => {
|
||||
const cmdStr = String(cmd);
|
||||
if (cmdStr.includes('git stash list --format="%H %gd %s"')) {
|
||||
return dropped ? "" : `${stashSha} stash@{0} fusion-merger-autostash:FN-050:1`;
|
||||
}
|
||||
if (cmdStr.includes("git status -z --porcelain")) return " M file.txt\0" as any;
|
||||
if (cmdStr.includes("git stash create")) return stashSha as any;
|
||||
if (cmdStr.includes("git stash store")) return "" as any;
|
||||
if (cmdStr.includes('git stash list --format="%H %gd"')) return dropped ? "" : `${stashSha} stash@{0}`;
|
||||
if (cmdStr.includes("git rev-parse stash@{0}")) return stashSha as any;
|
||||
if (cmdStr.includes("git stash drop stash@{0}")) {
|
||||
dropped = true;
|
||||
return "" as any;
|
||||
}
|
||||
if (cmdStr.includes("git stash apply")) return "" as any;
|
||||
if (cmdStr.includes("rev-parse --verify")) return Buffer.from("abc123");
|
||||
if (cmdStr === "git rev-parse HEAD" || cmdStr.startsWith("git rev-parse HEAD ")) return "mergedcommit123";
|
||||
if (cmdStr.includes("git log")) return "- feat: something" as any;
|
||||
if (cmdStr.includes("merge-base")) return Buffer.from("abc123");
|
||||
if (cmdStr.includes("git diff") && cmdStr.includes("--stat")) return "1 file changed" as any;
|
||||
if (cmdStr.includes("merge --squash")) return Buffer.from("");
|
||||
if (cmdStr.includes("diff --cached --quiet")) return "1" as any;
|
||||
if (cmdStr.includes("diff --cached")) return "0" as any;
|
||||
if (cmdStr.includes("show --shortstat")) return "3 files changed, 10 insertions(+), 2 deletions(-)" as any;
|
||||
if (cmdStr.includes("branch -d") || cmdStr.includes("branch -D")) return Buffer.from("");
|
||||
if (cmdStr.includes("worktree remove")) return Buffer.from("");
|
||||
return Buffer.from("");
|
||||
});
|
||||
|
||||
await aiMergeTask(store, "/tmp/root", "FN-050");
|
||||
|
||||
expect(dropped).toBe(true);
|
||||
expect(
|
||||
mockedExecSync.mock.calls.some((call) => String(call[0]).includes("git stash drop stash@{0}")),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("aiMergeTask — conditional worktree cleanup", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
|
||||
@@ -1448,9 +1448,46 @@ async function sweepAutostashOrphans(
|
||||
}
|
||||
}
|
||||
|
||||
const AUTOSTASH_TIMESTAMP_RE = /^fusion-merger-autostash:[A-Za-z]+-\d+:(?:race-rescue-\d+:)?(\d+)$/;
|
||||
|
||||
export async function sweepStaleAutostashes(
|
||||
rootDir: string,
|
||||
options: { maxAgeMs: number; taskStore?: TaskStore },
|
||||
): Promise<{ dropped: number }> {
|
||||
try {
|
||||
void options.taskStore;
|
||||
const now = Date.now();
|
||||
const threshold = Math.max(0, Math.trunc(options.maxAgeMs));
|
||||
const entries = await listOrphanedAutostashes(rootDir);
|
||||
let dropped = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
const match = AUTOSTASH_TIMESTAMP_RE.exec(entry.label.trim());
|
||||
if (!match) continue;
|
||||
const ts = Number.parseInt(match[1] ?? "", 10);
|
||||
if (!Number.isFinite(ts)) continue;
|
||||
if (now - ts <= threshold) continue;
|
||||
const sourceTaskId = parseAutostashTaskId(entry.label) ?? "autostash-sweep";
|
||||
const result = await dropAutostashBySha(rootDir, sourceTaskId, entry.sha);
|
||||
if (result.dropped) dropped += 1;
|
||||
}
|
||||
|
||||
const hours = Math.max(1, Math.round(threshold / 3_600_000));
|
||||
mergerLog.log(`startup-sweep: dropped ${dropped} stale fusion-merger-autostash entries older than ${hours}h`);
|
||||
return { dropped };
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
mergerLog.warn(`startup-sweep: stale autostash sweep failed (${msg})`);
|
||||
return { dropped: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
export const __test__ = {
|
||||
sweepAutostashOrphans,
|
||||
parseAutostashTaskId,
|
||||
dropAutostashHandle,
|
||||
isAutostashLive,
|
||||
sweepStaleAutostashes,
|
||||
};
|
||||
|
||||
async function stashUnrelatedRootDirChanges(
|
||||
@@ -1652,6 +1689,73 @@ async function dropAutostashBySha(
|
||||
return { dropped: false, reason: "exhausted retry attempts" };
|
||||
}
|
||||
|
||||
async function isAutostashLive(rootDir: string, sha: string): Promise<boolean> {
|
||||
try {
|
||||
const stashFiles = await listStashChangedPaths(rootDir, sha);
|
||||
if (stashFiles.size === 0) return false;
|
||||
const pathsArg = [...stashFiles].map(quoteArg).join(" ");
|
||||
const { stdout: pathDiffOut } = await execAsync(
|
||||
`git diff --name-only HEAD ${quoteArg(sha)} -- ${pathsArg}`,
|
||||
{ cwd: rootDir, encoding: "utf-8" },
|
||||
);
|
||||
return pathDiffOut.trim().length > 0;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
async function dropAutostashHandle(
|
||||
rootDir: string,
|
||||
taskId: string,
|
||||
handle: AutostashHandle,
|
||||
options: {
|
||||
keepIfLive: boolean;
|
||||
store?: TaskStore;
|
||||
context?: string;
|
||||
},
|
||||
): Promise<{ dropped: number; keptLive: number; failed: number }> {
|
||||
const entries = [
|
||||
{ sha: handle.sha, label: handle.label, kind: "primary" as const },
|
||||
...(handle.rescueShas ?? []).map((r) => ({ sha: r.sha, label: r.label, kind: "race-rescue" as const })),
|
||||
];
|
||||
|
||||
let dropped = 0;
|
||||
let keptLive = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
if (options.keepIfLive) {
|
||||
const live = await isAutostashLive(rootDir, entry.sha);
|
||||
if (live) {
|
||||
keptLive += 1;
|
||||
mergerLog.warn(`${taskId}: preserving live ${entry.kind} autostash ${entry.sha.slice(0, 7)} (${entry.label})`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const dropResult = await dropAutostashBySha(rootDir, taskId, entry.sha);
|
||||
if (dropResult.dropped) {
|
||||
dropped += 1;
|
||||
mergerLog.log(`${taskId}: dropped ${entry.kind} autostash ${entry.sha.slice(0, 7)} (${entry.label})`);
|
||||
} else {
|
||||
failed += 1;
|
||||
mergerLog.warn(
|
||||
`${taskId}: failed to drop ${entry.kind} autostash ${entry.sha.slice(0, 7)} (${entry.label}) — ${dropResult.reason ?? "unknown"}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.store && options.context) {
|
||||
await options.store.logEntry(
|
||||
taskId,
|
||||
`${options.context}: autostash cleanup dropped ${dropped}, preserved ${keptLive} live, failed ${failed}`,
|
||||
entries.map((entry) => `${entry.kind} ${entry.sha.slice(0, 7)} (${entry.label})`).join("\n"),
|
||||
).catch(() => undefined);
|
||||
}
|
||||
|
||||
return { dropped, keptLive, failed };
|
||||
}
|
||||
|
||||
/**
|
||||
* AI fix-agent for autostash apply conflicts. Spawned only when applying
|
||||
* the stashed dev work hits a conflict — the merge has already committed
|
||||
@@ -2221,6 +2325,44 @@ ${fileList}
|
||||
* `git apply --3way` from the patch, fall through to AI patch-recovery
|
||||
* if needed. See `tryRecoverHardFailApply`.
|
||||
*/
|
||||
async function restoreRescueAutostashes(
|
||||
rootDir: string,
|
||||
taskId: string,
|
||||
handle: AutostashHandle,
|
||||
ctx: {
|
||||
store: TaskStore;
|
||||
},
|
||||
): Promise<{ unresolvedCount: number }> {
|
||||
const rescueShas = handle.rescueShas ?? [];
|
||||
if (rescueShas.length === 0) return { unresolvedCount: 0 };
|
||||
|
||||
let unresolvedCount = 0;
|
||||
for (const rescue of rescueShas) {
|
||||
try {
|
||||
await execAsync(`git stash apply ${rescue.sha}`, { cwd: rootDir });
|
||||
const dropResult = await dropAutostashBySha(rootDir, taskId, rescue.sha);
|
||||
if (dropResult.dropped) {
|
||||
mergerLog.log(`${taskId}: restored and dropped race-rescue autostash ${rescue.sha.slice(0, 7)} (${rescue.label})`);
|
||||
} else {
|
||||
unresolvedCount += 1;
|
||||
mergerLog.warn(`${taskId}: restored race-rescue autostash ${rescue.sha.slice(0, 7)} but drop failed (${dropResult.reason ?? "unknown"})`);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
unresolvedCount += 1;
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
mergerLog.warn(`${taskId}: race-rescue autostash apply failed for ${rescue.sha.slice(0, 7)} (${msg}); preserving stash for manual recovery`);
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.store.logEntry(
|
||||
taskId,
|
||||
`Race-rescue autostash restore attempted: ${rescueShas.length - unresolvedCount} restored, ${unresolvedCount} preserved`,
|
||||
rescueShas.map((r) => `${r.sha.slice(0, 7)} (${r.label})`).join("\n"),
|
||||
).catch(() => undefined);
|
||||
|
||||
return { unresolvedCount };
|
||||
}
|
||||
|
||||
async function restoreUnrelatedRootDirChanges(
|
||||
rootDir: string,
|
||||
taskId: string,
|
||||
@@ -5727,11 +5869,28 @@ export async function aiMergeTask(
|
||||
if (resultForFinally) {
|
||||
resultForFinally.autostash = outcome;
|
||||
}
|
||||
|
||||
const rescueRestore = outcome.status === "restored" || outcome.status === "ai-resolved"
|
||||
? await restoreRescueAutostashes(rootDir, taskId, autostashHandle, { store })
|
||||
: { unresolvedCount: 0 };
|
||||
const keepIfLive = outcome.status === "failed"
|
||||
|| outcome.status === "conflict-needs-manual"
|
||||
|| rescueRestore.unresolvedCount > 0;
|
||||
await dropAutostashHandle(rootDir, taskId, autostashHandle, {
|
||||
keepIfLive,
|
||||
store,
|
||||
context: "Post-restore autostash cleanup",
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
// Any throw from restore should never propagate out of the merger
|
||||
// — the merge result has already been recorded. Log and swallow.
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
mergerLog.warn(`${taskId}: autostash restore threw unexpectedly (${msg}) — stash may be left in place; check git stash list`);
|
||||
mergerLog.warn(`${taskId}: autostash restore threw unexpectedly (${msg}) — running keep-if-live cleanup sweep`);
|
||||
await dropAutostashHandle(rootDir, taskId, autostashHandle, {
|
||||
keepIfLive: true,
|
||||
store,
|
||||
context: "Autostash restore exception cleanup",
|
||||
});
|
||||
if (resultForFinally) {
|
||||
resultForFinally.autostash = {
|
||||
status: "failed",
|
||||
|
||||
@@ -21,7 +21,7 @@ import { NotificationService } from "./notification/index.js";
|
||||
import { GridlockDetector } from "./gridlock-detector.js";
|
||||
import { CronRunner, createAiPromptExecutor } from "./cron-runner.js";
|
||||
import type { RoutineRunner } from "./routine-runner.js";
|
||||
import { aiMergeTask } from "./merger.js";
|
||||
import { aiMergeTask, sweepStaleAutostashes } from "./merger.js";
|
||||
import { PRIORITY_MERGE } from "./concurrency.js";
|
||||
import { runtimeLog } from "./logger.js";
|
||||
import type { HeartbeatTriggerScheduler } from "./agent-heartbeat.js";
|
||||
@@ -170,6 +170,7 @@ export class ProjectEngine {
|
||||
private activeMergeTaskId: string | null = null;
|
||||
private mergeAbortController: AbortController | null = null;
|
||||
private mergeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private autostashSweepTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
/**
|
||||
* Pending manual merge resolvers — keyed by taskId.
|
||||
@@ -444,6 +445,10 @@ export class ProjectEngine {
|
||||
// 8. Start periodic merge retry sweep
|
||||
this.scheduleMergeRetry(store);
|
||||
|
||||
// 9. Startup + periodic stale autostash sweeps (independent of autoMerge)
|
||||
void this.runStaleAutostashSweep(store, "startup");
|
||||
this.scheduleStaleAutostashSweep(store);
|
||||
|
||||
this.started = true;
|
||||
runtimeLog.log(`ProjectEngine started for ${this.config.projectId}`);
|
||||
}
|
||||
@@ -467,6 +472,10 @@ export class ProjectEngine {
|
||||
clearTimeout(this.mergeRetryTimer);
|
||||
this.mergeRetryTimer = null;
|
||||
}
|
||||
if (this.autostashSweepTimer) {
|
||||
clearTimeout(this.autostashSweepTimer);
|
||||
this.autostashSweepTimer = null;
|
||||
}
|
||||
|
||||
// Abort active/pending merge work before tearing down sessions.
|
||||
this.mergeAbortController?.abort();
|
||||
@@ -1920,6 +1929,44 @@ export class ProjectEngine {
|
||||
}
|
||||
}
|
||||
|
||||
private resolveAutostashMaxAgeMs(settings: Settings): number {
|
||||
const hours = Math.max(1, Math.trunc(settings.mergerAutostashMaxAgeHours ?? 24));
|
||||
return hours * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
private async runStaleAutostashSweep(store: TaskStore, reason: "startup" | "periodic"): Promise<void> {
|
||||
try {
|
||||
const settings = await store.getSettings();
|
||||
if (settings.globalPause || settings.enginePaused) return;
|
||||
const maxAgeMs = this.resolveAutostashMaxAgeMs(settings);
|
||||
const result = await sweepStaleAutostashes(this.config.workingDirectory, {
|
||||
maxAgeMs,
|
||||
taskStore: store,
|
||||
});
|
||||
if (result.dropped > 0) {
|
||||
runtimeLog.log(`${reason === "startup" ? "Startup" : "Periodic"} stale autostash sweep dropped ${result.dropped} stash(es)`);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
runtimeLog.warn(`Stale autostash ${reason} sweep failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleStaleAutostashSweep(store: TaskStore): void {
|
||||
if (this.shuttingDown) return;
|
||||
const schedule = async () => {
|
||||
if (this.shuttingDown) return;
|
||||
try {
|
||||
await this.runStaleAutostashSweep(store, "periodic");
|
||||
} finally {
|
||||
if (!this.shuttingDown) {
|
||||
this.autostashSweepTimer = setTimeout(() => void schedule(), 60 * 60 * 1000);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
this.autostashSweepTimer = setTimeout(() => void schedule(), 60 * 60 * 1000);
|
||||
}
|
||||
|
||||
private scheduleMergeRetry(store: TaskStore): void {
|
||||
if (this.shuttingDown) return;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user