feat(KB-154): persist worktree pool across engine restarts

- Add rehydrate() method and scanIdleWorktrees() to restore idle worktrees from disk on startup
- Add cleanupOrphanedWorktrees() to remove stale worktrees when recycling is disabled
- Wire startup rehydrate/cleanup into dashboard command based on recycleWorktrees setting
- Export new functions from @kb/engine and update dashboard imports
- Add unit tests for rehydrate/scan/cleanup and integration tests for restart resilience
This commit is contained in:
Dustin Byrne
2026-03-28 02:45:47 -04:00
parent 5e3cafe05b
commit a2a12f94eb
6 changed files with 566 additions and 6 deletions

View File

@@ -6,6 +6,6 @@ export { Scheduler, type SchedulerOptions } from "./scheduler.js";
export { aiMergeTask, type MergerOptions } from "./merger.js";
export { reviewStep, type ReviewType, type ReviewVerdict, type ReviewResult, type ReviewOptions } from "./reviewer.js";
export { createKbAgent, type AgentOptions, type AgentResult } from "./pi.js";
export { WorktreePool } from "./worktree-pool.js";
export { WorktreePool, scanIdleWorktrees, cleanupOrphanedWorktrees } from "./worktree-pool.js";
export { createLogger, type Logger } from "./logger.js";
export { isUsageLimitError, UsageLimitPauser } from "./usage-limit-detector.js";

View File

@@ -25,20 +25,23 @@ vi.mock("node:child_process", () => ({
}));
vi.mock("node:fs", () => ({
existsSync: vi.fn().mockReturnValue(true),
readdirSync: vi.fn().mockReturnValue([]),
}));
import { TaskExecutor } from "./executor.js";
import { TriageProcessor } from "./triage.js";
import { Scheduler } from "./scheduler.js";
import { aiMergeTask } from "./merger.js";
import { WorktreePool, scanIdleWorktrees, cleanupOrphanedWorktrees } from "./worktree-pool.js";
import { createKbAgent } from "./pi.js";
import { execSync } from "node:child_process";
import { existsSync } from "node:fs";
import { existsSync, readdirSync } from "node:fs";
import type { Task, TaskDetail, TaskStep, Column, Settings, StepStatus } from "@kb/core";
const mockedCreateHaiAgent = vi.mocked(createKbAgent);
const mockedExecSync = vi.mocked(execSync);
const mockedExistsSync = vi.mocked(existsSync);
const mockedReaddirSync = vi.mocked(readdirSync);
// ── Mock helpers ──────────────────────────────────────────────────────────
@@ -701,3 +704,209 @@ describe("Crash scenario edge cases", () => {
expect(sem.activeCount).toBe(0);
});
});
// ── Worktree pool restart resilience tests ────────────────────────────────
function makeDirEntry(name: string) {
return { name, isDirectory: () => true } as any;
}
describe("Worktree pool restart with recycleWorktrees=true", () => {
it("pool is rehydrated with idle worktrees from disk", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("swift-falcon"),
makeDirEntry("calm-river"),
makeDirEntry("bold-eagle"),
] as any);
mockedExistsSync.mockReturnValue(true);
const store = createMockStore();
store.listTasks.mockResolvedValue([
makeTask("KB-100", "in-progress", { worktree: "/root/.worktrees/swift-falcon" }),
makeTask("KB-101", "done", { worktree: "/root/.worktrees/calm-river" }),
]);
// Simulate startup rehydration
const pool = new WorktreePool();
const idlePaths = await scanIdleWorktrees("/root", store);
pool.rehydrate(idlePaths);
// swift-falcon → in-progress, not idle
// calm-river → done, idle
// bold-eagle → unassigned, idle
expect(pool.size).toBe(2);
expect(pool.has("/root/.worktrees/calm-river")).toBe(true);
expect(pool.has("/root/.worktrees/bold-eagle")).toBe(true);
expect(pool.has("/root/.worktrees/swift-falcon")).toBe(false);
});
it("executor acquires from rehydrated pool instead of creating new worktrees", async () => {
// Setup: rehydrate pool with one idle worktree
mockedReaddirSync.mockReturnValue([
makeDirEntry("idle-wt"),
] as any);
const store = createMockStore();
store.listTasks.mockResolvedValue([]);
store.getSettings.mockResolvedValue({
...DEFAULT_SETTINGS,
recycleWorktrees: true,
});
store.getTask.mockResolvedValue(makeTaskDetail("KB-110", "in-progress"));
const pool = new WorktreePool();
const idlePaths = await scanIdleWorktrees("/root", store);
pool.rehydrate(idlePaths);
expect(pool.size).toBe(1);
// Now simulate executor acquiring from pool
// The pool path exists on disk, but the task's default path does not
mockedExistsSync.mockImplementation(
(p) => p === "/root/.worktrees/idle-wt",
);
mockAgentSuccess();
const executor = new TaskExecutor(store, "/root", { pool });
await executor.execute(makeTask("KB-110", "in-progress"));
await new Promise((r) => setTimeout(r, 50));
// Pool should be empty (worktree acquired)
expect(pool.size).toBe(0);
// No git worktree add calls (reused from pool)
const worktreeAddCalls = mockedExecSync.mock.calls.filter(
(c) => typeof c[0] === "string" && (c[0] as string).includes("worktree add"),
);
expect(worktreeAddCalls).toHaveLength(0);
// Should log pool acquisition
expect(store.logEntry).toHaveBeenCalledWith(
"KB-110",
expect.stringContaining("Acquired worktree from pool"),
);
});
it("worktrees assigned to in-progress tasks are preserved (not in pool)", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("active-wt"),
makeDirEntry("idle-wt"),
] as any);
mockedExistsSync.mockReturnValue(true);
const store = createMockStore();
store.listTasks.mockResolvedValue([
makeTask("KB-120", "in-progress", { worktree: "/root/.worktrees/active-wt" }),
]);
const pool = new WorktreePool();
const idlePaths = await scanIdleWorktrees("/root", store);
pool.rehydrate(idlePaths);
// Only idle-wt should be in pool (active-wt is assigned to in-progress task)
expect(pool.size).toBe(1);
expect(pool.has("/root/.worktrees/idle-wt")).toBe(true);
expect(pool.has("/root/.worktrees/active-wt")).toBe(false);
});
it("worktrees assigned to in-review tasks are preserved (not in pool)", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("review-wt"),
] as any);
mockedExistsSync.mockReturnValue(true);
const store = createMockStore();
store.listTasks.mockResolvedValue([
makeTask("KB-121", "in-review", { worktree: "/root/.worktrees/review-wt" }),
]);
const pool = new WorktreePool();
const idlePaths = await scanIdleWorktrees("/root", store);
pool.rehydrate(idlePaths);
// review-wt is assigned to in-review task — NOT idle
expect(pool.size).toBe(0);
});
});
describe("Worktree cleanup on restart with recycleWorktrees=false", () => {
it("orphaned worktrees are cleaned up via git worktree remove", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("orphan-1"),
makeDirEntry("orphan-2"),
] as any);
mockedExistsSync.mockReturnValue(true);
mockedExecSync.mockReturnValue(Buffer.from(""));
const store = createMockStore();
store.listTasks.mockResolvedValue([]);
const cleaned = await cleanupOrphanedWorktrees("/root", store);
expect(cleaned).toBe(2);
const removeCalls = mockedExecSync.mock.calls.filter(
(c) => typeof c[0] === "string" && (c[0] as string).includes("worktree remove"),
);
expect(removeCalls).toHaveLength(2);
});
it("worktrees assigned to in-progress tasks are preserved during cleanup", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("active-wt"),
makeDirEntry("orphan-wt"),
] as any);
mockedExistsSync.mockReturnValue(true);
mockedExecSync.mockReturnValue(Buffer.from(""));
const store = createMockStore();
store.listTasks.mockResolvedValue([
makeTask("KB-130", "in-progress", { worktree: "/root/.worktrees/active-wt" }),
]);
const cleaned = await cleanupOrphanedWorktrees("/root", store);
expect(cleaned).toBe(1);
const removeCalls = mockedExecSync.mock.calls.filter(
(c) => typeof c[0] === "string" && (c[0] as string).includes("worktree remove"),
);
expect(removeCalls).toHaveLength(1);
expect(removeCalls[0][0]).toContain("orphan-wt");
expect(removeCalls[0][0]).not.toContain("active-wt");
});
it("worktrees assigned to in-review tasks are preserved during cleanup", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("review-wt"),
] as any);
mockedExistsSync.mockReturnValue(true);
const store = createMockStore();
store.listTasks.mockResolvedValue([
makeTask("KB-131", "in-review", { worktree: "/root/.worktrees/review-wt" }),
]);
const cleaned = await cleanupOrphanedWorktrees("/root", store);
// review-wt is assigned to in-review task — should NOT be removed
expect(cleaned).toBe(0);
});
});
describe("Edge case: worktree deleted between scan and acquire", () => {
it("acquire returns null when rehydrated worktree was deleted from disk", async () => {
const pool = new WorktreePool();
// Rehydrate succeeds (path exists at scan time)
mockedExistsSync.mockReturnValue(true);
pool.rehydrate(["/root/.worktrees/vanished-wt"]);
expect(pool.size).toBe(1);
// Between rehydrate and acquire, the directory is deleted
mockedExistsSync.mockReturnValue(false);
// acquire() checks existsSync and prunes the stale entry
const result = pool.acquire();
expect(result).toBeNull();
expect(pool.size).toBe(0);
});
});

View File

@@ -6,14 +6,17 @@ vi.mock("node:child_process", () => ({
vi.mock("node:fs", () => ({
existsSync: vi.fn().mockReturnValue(true),
readdirSync: vi.fn().mockReturnValue([]),
}));
import { WorktreePool } from "./worktree-pool.js";
import { WorktreePool, scanIdleWorktrees, cleanupOrphanedWorktrees } from "./worktree-pool.js";
import { execSync } from "node:child_process";
import { existsSync } from "node:fs";
import { existsSync, readdirSync } from "node:fs";
import type { Task, Column } from "@kb/core";
const mockedExecSync = vi.mocked(execSync);
const mockedExistsSync = vi.mocked(existsSync);
const mockedReaddirSync = vi.mocked(readdirSync);
describe("WorktreePool", () => {
let pool: WorktreePool;
@@ -161,4 +164,217 @@ describe("WorktreePool", () => {
expect(calls).toContain('git checkout -B "kb/kb-001" main');
});
});
describe("rehydrate", () => {
it("loads paths into the idle set", () => {
mockedExistsSync.mockReturnValue(true);
pool.rehydrate(["/tmp/wt-1", "/tmp/wt-2", "/tmp/wt-3"]);
expect(pool.size).toBe(3);
expect(pool.has("/tmp/wt-1")).toBe(true);
expect(pool.has("/tmp/wt-2")).toBe(true);
expect(pool.has("/tmp/wt-3")).toBe(true);
});
it("skips paths that don't exist on disk", () => {
mockedExistsSync.mockImplementation((p) => p === "/tmp/good-wt");
pool.rehydrate(["/tmp/good-wt", "/tmp/gone-wt"]);
expect(pool.size).toBe(1);
expect(pool.has("/tmp/good-wt")).toBe(true);
expect(pool.has("/tmp/gone-wt")).toBe(false);
});
it("handles empty array", () => {
pool.rehydrate([]);
expect(pool.size).toBe(0);
});
it("does not duplicate entries already in the pool", () => {
mockedExistsSync.mockReturnValue(true);
pool.release("/tmp/existing");
pool.rehydrate(["/tmp/existing", "/tmp/new"]);
expect(pool.size).toBe(2);
});
});
});
// ── Helper for mock store ─────────────────────────────────────────────
function makeTask(id: string, column: Column, worktree?: string): Task {
return {
id,
title: `Task ${id}`,
description: `Description for ${id}`,
column,
dependencies: [],
worktree,
steps: [],
currentStep: 0,
log: [],
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
};
}
function createMockStore(tasks: Task[] = []) {
return {
listTasks: vi.fn().mockResolvedValue(tasks),
} as any;
}
function makeDirEntry(name: string) {
return { name, isDirectory: () => true } as any;
}
// ── scanIdleWorktrees tests ───────────────────────────────────────────
describe("scanIdleWorktrees", () => {
beforeEach(() => {
vi.clearAllMocks();
mockedExistsSync.mockReturnValue(true);
});
it("correctly identifies idle vs active worktrees", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("swift-falcon"),
makeDirEntry("calm-river"),
makeDirEntry("bold-eagle"),
] as any);
const store = createMockStore([
makeTask("KB-001", "in-progress", "/root/.worktrees/swift-falcon"),
makeTask("KB-002", "done", "/root/.worktrees/calm-river"),
]);
const idle = await scanIdleWorktrees("/root", store);
// swift-falcon is assigned to in-progress task → NOT idle
// calm-river is assigned to done task → idle (done tasks don't count)
// bold-eagle is not assigned at all → idle
expect(idle).toContain("/root/.worktrees/calm-river");
expect(idle).toContain("/root/.worktrees/bold-eagle");
expect(idle).not.toContain("/root/.worktrees/swift-falcon");
});
it("handles empty .worktrees/ directory", async () => {
mockedReaddirSync.mockReturnValue([] as any);
const store = createMockStore([]);
const idle = await scanIdleWorktrees("/root", store);
expect(idle).toEqual([]);
});
it("handles missing .worktrees/ directory", async () => {
mockedExistsSync.mockReturnValue(false);
const store = createMockStore([]);
const idle = await scanIdleWorktrees("/root", store);
expect(idle).toEqual([]);
});
it("treats in-review tasks as active (worktree preserved)", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("review-wt"),
] as any);
const store = createMockStore([
makeTask("KB-010", "in-review", "/root/.worktrees/review-wt"),
]);
const idle = await scanIdleWorktrees("/root", store);
expect(idle).not.toContain("/root/.worktrees/review-wt");
});
it("returns all worktrees when no tasks exist", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("wt-1"),
makeDirEntry("wt-2"),
] as any);
const store = createMockStore([]);
const idle = await scanIdleWorktrees("/root", store);
expect(idle).toHaveLength(2);
expect(idle).toContain("/root/.worktrees/wt-1");
expect(idle).toContain("/root/.worktrees/wt-2");
});
});
// ── cleanupOrphanedWorktrees tests ────────────────────────────────────
describe("cleanupOrphanedWorktrees", () => {
beforeEach(() => {
vi.clearAllMocks();
mockedExistsSync.mockReturnValue(true);
mockedExecSync.mockReturnValue(Buffer.from(""));
});
it("removes worktrees not assigned to any active task", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("orphan-1"),
makeDirEntry("orphan-2"),
] as any);
const store = createMockStore([]);
const cleaned = await cleanupOrphanedWorktrees("/root", store);
expect(cleaned).toBe(2);
const removeCalls = mockedExecSync.mock.calls.filter(
(c) => typeof c[0] === "string" && (c[0] as string).includes("worktree remove"),
);
expect(removeCalls).toHaveLength(2);
expect(removeCalls[0][0]).toContain("/root/.worktrees/orphan-1");
expect(removeCalls[1][0]).toContain("/root/.worktrees/orphan-2");
});
it("preserves worktrees assigned to in-progress/in-review tasks", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("active-wt"),
makeDirEntry("orphan-wt"),
] as any);
const store = createMockStore([
makeTask("KB-001", "in-progress", "/root/.worktrees/active-wt"),
]);
const cleaned = await cleanupOrphanedWorktrees("/root", store);
expect(cleaned).toBe(1);
const removeCalls = mockedExecSync.mock.calls.filter(
(c) => typeof c[0] === "string" && (c[0] as string).includes("worktree remove"),
);
expect(removeCalls).toHaveLength(1);
expect(removeCalls[0][0]).toContain("orphan-wt");
expect(removeCalls[0][0]).not.toContain("active-wt");
});
it("handles git worktree remove failures gracefully (non-fatal)", async () => {
mockedReaddirSync.mockReturnValue([
makeDirEntry("fail-wt"),
makeDirEntry("ok-wt"),
] as any);
mockedExecSync.mockImplementation((cmd: any) => {
if (typeof cmd === "string" && cmd.includes("fail-wt")) {
throw new Error("worktree locked");
}
return Buffer.from("");
});
const store = createMockStore([]);
const cleaned = await cleanupOrphanedWorktrees("/root", store);
// Only 1 cleaned (the other failed), but no throw
expect(cleaned).toBe(1);
});
it("no-ops when .worktrees/ doesn't exist", async () => {
mockedExistsSync.mockReturnValue(false);
const store = createMockStore([]);
const cleaned = await cleanupOrphanedWorktrees("/root", store);
expect(cleaned).toBe(0);
expect(mockedExecSync).not.toHaveBeenCalled();
});
});

View File

@@ -1,5 +1,7 @@
import { execSync } from "node:child_process";
import { existsSync } from "node:fs";
import { existsSync, readdirSync } from "node:fs";
import { join } from "node:path";
import type { TaskStore } from "@kb/core";
import { worktreePoolLog } from "./logger.js";
/**
@@ -12,6 +14,14 @@ import { worktreePoolLog } from "./logger.js";
* The pool only tracks *idle* worktrees — those not currently assigned to
* any active task. The scheduler's `maxWorktrees` setting still governs
* the total number of worktrees (active + idle).
*
* **Lifecycle across restarts:** The pool is in-memory only, but on engine
* startup it can be rehydrated from disk state via {@link rehydrate} and
* {@link scanIdleWorktrees}. When `recycleWorktrees` is true, the startup
* sequence scans the `.worktrees/` directory, identifies idle worktrees
* (those not assigned to any active task), and bulk-loads them into the
* pool. When `recycleWorktrees` is false, orphaned worktrees are cleaned
* up via {@link cleanupOrphanedWorktrees}.
*/
export class WorktreePool {
private idle = new Set<string>();
@@ -68,6 +78,24 @@ export class WorktreePool {
return paths;
}
/**
* Bulk-load known idle worktree paths into the pool.
*
* Called at engine startup to restore the pool from disk state.
* Paths that no longer exist on disk are silently skipped.
*
* @param idlePaths — Absolute paths to idle worktree directories
*/
rehydrate(idlePaths: string[]): void {
for (const path of idlePaths) {
if (existsSync(path)) {
this.idle.add(path);
} else {
worktreePoolLog.log(`Rehydrate skipped (not on disk): ${path}`);
}
}
}
/**
* Prepare a recycled worktree for a new task.
*
@@ -105,3 +133,83 @@ export class WorktreePool {
});
}
}
/**
* Scan the `.worktrees/` directory to find idle worktrees that can be
* loaded into the pool on startup.
*
* A worktree is considered "idle" if it exists on disk under
* `<rootDir>/.worktrees/` but is NOT assigned (via `task.worktree`) to
* any non-done task.
*
* @param rootDir — Project root directory (parent of `.worktrees/`)
* @param store — Task store for listing tasks and their worktree assignments
* @returns Absolute paths of idle worktree directories
*/
export async function scanIdleWorktrees(rootDir: string, store: TaskStore): Promise<string[]> {
const worktreesDir = join(rootDir, ".worktrees");
if (!existsSync(worktreesDir)) {
return [];
}
// List all subdirectories under .worktrees/
let dirs: string[];
try {
const entries = readdirSync(worktreesDir, { withFileTypes: true });
dirs = entries
.filter((e) => e.isDirectory())
.map((e) => join(worktreesDir, e.name));
} catch {
return [];
}
if (dirs.length === 0) {
return [];
}
// Find worktree paths assigned to non-done tasks (active worktrees)
const tasks = await store.listTasks();
const activeWorktrees = new Set<string>();
for (const task of tasks) {
if (task.worktree && task.column !== "done") {
activeWorktrees.add(task.worktree);
}
}
// Return worktrees on disk that are NOT active
return dirs.filter((dir) => !activeWorktrees.has(dir));
}
/**
* Clean up orphaned worktrees left behind from previous engine runs.
*
* Removes worktree directories under `<rootDir>/.worktrees/` that are NOT
* assigned to any non-done task. Used on startup when `recycleWorktrees`
* is false to avoid disk waste.
*
* Failures on individual worktree removals are logged but not fatal.
*
* @param rootDir — Project root directory (parent of `.worktrees/`)
* @param store — Task store for listing tasks and their worktree assignments
* @returns Number of worktrees cleaned up
*/
export async function cleanupOrphanedWorktrees(rootDir: string, store: TaskStore): Promise<number> {
const orphaned = await scanIdleWorktrees(rootDir, store);
let cleaned = 0;
for (const worktreePath of orphaned) {
try {
execSync(`git worktree remove "${worktreePath}" --force`, {
cwd: rootDir,
stdio: "pipe",
});
worktreePoolLog.log(`Cleaned up orphaned worktree: ${worktreePath}`);
cleaned++;
} catch (err: any) {
worktreePoolLog.log(`Failed to remove orphaned worktree ${worktreePath}: ${err.message}`);
}
}
return cleaned;
}