feat(FN-4257): run durable heartbeat task sessions in worktree
Fusion-Task-Id: FN-4257 Fusion-Task-Lineage: db48d51e-9e7d-471d-b26b-2a8763fb41d1
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import type { Agent, AgentHeartbeatRun } from "@fusion/core";
|
||||
import { HeartbeatMonitor } from "../agent-heartbeat.js";
|
||||
import * as worktreeAcquisition from "../worktree-acquisition.js";
|
||||
import * as piModule from "../pi.js";
|
||||
|
||||
describe("heartbeat worktree cwd", () => {
|
||||
let store: any;
|
||||
let taskStore: any;
|
||||
const agent: Agent = { id: "a1", name: "A", role: "executor", state: "active", taskId: "FN-1", createdAt: "", updatedAt: "", metadata: {} } as any;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.spyOn(piModule, "createFnAgent").mockResolvedValue({ session: { prompt: vi.fn(), dispose: vi.fn() } } as any);
|
||||
vi.spyOn(worktreeAcquisition, "acquireTaskWorktree").mockResolvedValue({ worktreePath: "/tmp/wt", branch: "fusion/fn-1", source: "existing", hydrated: false, isResume: true });
|
||||
|
||||
const run: AgentHeartbeatRun = { id: "r1", agentId: "a1", status: "active", startedAt: new Date().toISOString(), endedAt: null } as any;
|
||||
store = {
|
||||
startHeartbeatRun: vi.fn().mockResolvedValue(run),
|
||||
saveRun: vi.fn(),
|
||||
getRunDetail: vi.fn().mockResolvedValue(run),
|
||||
getAgent: vi.fn().mockResolvedValue(agent),
|
||||
updateAgentState: vi.fn(),
|
||||
updateAgent: vi.fn(),
|
||||
endHeartbeatRun: vi.fn(),
|
||||
assignTask: vi.fn(),
|
||||
getBudgetStatus: vi.fn().mockResolvedValue({ isOverBudget: false, isOverThreshold: false, usagePercent: 0 }),
|
||||
getCachedAgent: vi.fn().mockReturnValue(null),
|
||||
getLastBlockedState: vi.fn().mockResolvedValue(null),
|
||||
setLastBlockedState: vi.fn(),
|
||||
clearLastBlockedState: vi.fn(),
|
||||
appendRunLog: vi.fn(),
|
||||
getAgentsByReportsTo: vi.fn().mockResolvedValue([]),
|
||||
recordHeartbeat: vi.fn(),
|
||||
};
|
||||
taskStore = {
|
||||
getSettings: vi.fn().mockResolvedValue({}),
|
||||
getTask: vi.fn().mockResolvedValue({ id: "FN-1", title: "t", description: "d", column: "todo", dependencies: [], steps: [], log: [] }),
|
||||
moveTask: vi.fn(),
|
||||
appendAgentLog: vi.fn(),
|
||||
listTasks: vi.fn().mockResolvedValue([]),
|
||||
selectNextTaskForAgent: vi.fn().mockResolvedValue(null),
|
||||
};
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("uses acquired worktree cwd for task-scoped runs", async () => {
|
||||
const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: "/repo" });
|
||||
await monitor.executeHeartbeat({ agentId: "a1", source: "on_demand" });
|
||||
expect(worktreeAcquisition.acquireTaskWorktree).toHaveBeenCalled();
|
||||
expect(piModule.createFnAgent).toHaveBeenCalledWith(expect.objectContaining({ cwd: "/tmp/wt" }));
|
||||
});
|
||||
|
||||
it("uses rootDir for no-task runs", async () => {
|
||||
store.getAgent.mockResolvedValue({ ...agent, taskId: undefined, soul: "x" });
|
||||
const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: "/repo" });
|
||||
await monitor.executeHeartbeat({ agentId: "a1", source: "on_demand" });
|
||||
expect(worktreeAcquisition.acquireTaskWorktree).not.toHaveBeenCalled();
|
||||
expect(piModule.createFnAgent).toHaveBeenCalledWith(expect.objectContaining({ cwd: "/repo" }));
|
||||
});
|
||||
|
||||
it("completes with worktree_acquisition_failed when helper throws", async () => {
|
||||
vi.spyOn(worktreeAcquisition, "acquireTaskWorktree").mockRejectedValueOnce(new Error("nope"));
|
||||
const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: "/repo" });
|
||||
await monitor.executeHeartbeat({ agentId: "a1", source: "on_demand" });
|
||||
expect(piModule.createFnAgent).not.toHaveBeenCalled();
|
||||
expect(taskStore.moveTask).toHaveBeenCalledWith("FN-1", "todo", { preserveProgress: true });
|
||||
});
|
||||
});
|
||||
@@ -27,7 +27,13 @@ vi.mock("../pi.js", () => ({
|
||||
}),
|
||||
}));
|
||||
import { createFnAgent } from "../pi.js";
|
||||
import { acquireTaskWorktree } from "../worktree-acquisition.js";
|
||||
const mockedCreateFnAgent = vi.mocked(createFnAgent);
|
||||
const mockedAcquireTaskWorktree = vi.mocked(acquireTaskWorktree);
|
||||
|
||||
vi.mock("../worktree-acquisition.js", () => ({
|
||||
acquireTaskWorktree: vi.fn(),
|
||||
}));
|
||||
|
||||
describe("executeHeartbeat", () => {
|
||||
let mockTaskStore: TaskStore;
|
||||
@@ -57,6 +63,8 @@ describe("executeHeartbeat", () => {
|
||||
prompt: "# Test PROMPT.md\nSome content",
|
||||
steps: [],
|
||||
column: "todo",
|
||||
worktree: "/tmp/worktree-fn-001",
|
||||
branch: "fusion/fn-001",
|
||||
dependencies: [],
|
||||
log: [],
|
||||
attachments: [],
|
||||
@@ -164,10 +172,17 @@ describe("executeHeartbeat", () => {
|
||||
beforeEach(() => {
|
||||
mockTaskStore = createMockTaskStore();
|
||||
vi.clearAllMocks();
|
||||
mockedAcquireTaskWorktree.mockResolvedValue({
|
||||
worktreePath: "/tmp/worktree-fn-001",
|
||||
branch: "fusion/fn-001",
|
||||
source: "existing",
|
||||
hydrated: false,
|
||||
isResume: true,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("reports health check", () => {
|
||||
|
||||
@@ -204,10 +204,14 @@ vi.mock("node:child_process", () => {
|
||||
|
||||
return { execSync: execSyncFn, exec: execFn, execFile: execFileFn, spawn: spawnFn };
|
||||
});
|
||||
vi.mock("node:fs", () => ({
|
||||
existsSync: vi.fn().mockReturnValue(true),
|
||||
readdirSync: vi.fn().mockReturnValue([]),
|
||||
}));
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:fs")>();
|
||||
return {
|
||||
...actual,
|
||||
existsSync: vi.fn().mockReturnValue(true),
|
||||
readdirSync: vi.fn().mockReturnValue([]),
|
||||
};
|
||||
});
|
||||
vi.mock("node:fs/promises", () => ({
|
||||
readFile: vi.fn().mockResolvedValue("# Task prompt content"),
|
||||
}));
|
||||
@@ -592,7 +596,7 @@ describe("In-progress task resume after restart", () => {
|
||||
const executor = new TaskExecutor(store, "/tmp/test");
|
||||
await executor.resumeOrphaned();
|
||||
await waitForAsyncExpectation(() => {
|
||||
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(1);
|
||||
expect(mockedCreateFnAgent).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// Must NOT mark the step done — the reset invalidated the prior approval.
|
||||
@@ -621,7 +625,7 @@ describe("In-progress task resume after restart", () => {
|
||||
const executor = new TaskExecutor(store, "/tmp/test");
|
||||
await executor.resumeOrphaned();
|
||||
await waitForAsyncExpectation(() => {
|
||||
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(1);
|
||||
expect(mockedCreateFnAgent).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
const updateStepDoneCalls = store.updateStep.mock.calls.filter(
|
||||
|
||||
88
packages/engine/src/__tests__/worktree-acquisition.test.ts
Normal file
88
packages/engine/src/__tests__/worktree-acquisition.test.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { acquireTaskWorktree } from "../worktree-acquisition.js";
|
||||
|
||||
vi.mock("../worktree-pool.js", async () => {
|
||||
const actual = await vi.importActual<any>("../worktree-pool.js");
|
||||
return { ...actual, isUsableTaskWorktree: vi.fn().mockResolvedValue(true) };
|
||||
});
|
||||
|
||||
vi.mock("../worktree-db-hydrate.js", () => ({
|
||||
hydrateWorktreeDb: vi.fn().mockResolvedValue({ degraded: false, tasksCopied: 1, documentsCopied: 1 }),
|
||||
}));
|
||||
|
||||
describe("acquireTaskWorktree", () => {
|
||||
const task = {
|
||||
id: "FN-1",
|
||||
title: "Task",
|
||||
description: "Desc",
|
||||
branch: null,
|
||||
worktree: null,
|
||||
} as any;
|
||||
|
||||
let store: any;
|
||||
beforeEach(() => {
|
||||
store = {
|
||||
updateTask: vi.fn().mockResolvedValue(undefined),
|
||||
logEntry: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
});
|
||||
|
||||
it("reuses existing usable worktree", async () => {
|
||||
const result = await acquireTaskWorktree({
|
||||
task: { ...task, worktree: process.cwd(), branch: "fusion/fn-1" },
|
||||
rootDir: process.cwd(),
|
||||
store,
|
||||
settings: {},
|
||||
createWorktree: vi.fn(),
|
||||
});
|
||||
expect(result.source).toBe("existing");
|
||||
expect(result.worktreePath).toBe(process.cwd());
|
||||
});
|
||||
|
||||
it("acquires from pool when enabled", async () => {
|
||||
const prepareForTask = vi.fn().mockResolvedValue("fusion/fn-1");
|
||||
const result = await acquireTaskWorktree({
|
||||
task,
|
||||
rootDir: process.cwd(),
|
||||
store,
|
||||
settings: { recycleWorktrees: true } as any,
|
||||
pool: {
|
||||
acquire: () => "/tmp/pooled",
|
||||
prepareForTask,
|
||||
release: vi.fn(),
|
||||
} as any,
|
||||
createWorktree: vi.fn(),
|
||||
});
|
||||
expect(result.source).toBe("pool");
|
||||
expect(prepareForTask).toHaveBeenCalled();
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-1", { worktree: "/tmp/pooled", branch: "fusion/fn-1" });
|
||||
});
|
||||
|
||||
it("creates fresh when pool disabled", async () => {
|
||||
const createWorktree = vi.fn().mockResolvedValue({ path: "/tmp/new", branch: "fusion/fn-1" });
|
||||
const result = await acquireTaskWorktree({
|
||||
task,
|
||||
rootDir: process.cwd(),
|
||||
store,
|
||||
settings: {},
|
||||
createWorktree,
|
||||
});
|
||||
expect(result.source).toBe("fresh");
|
||||
expect(createWorktree).toHaveBeenCalled();
|
||||
expect(store.updateTask).toHaveBeenCalledWith("FN-1", { worktree: "/tmp/new", branch: "fusion/fn-1" });
|
||||
});
|
||||
|
||||
it("skips init command when runInitCommand false", async () => {
|
||||
const runConfiguredCommand = vi.fn();
|
||||
await acquireTaskWorktree({
|
||||
task,
|
||||
rootDir: process.cwd(),
|
||||
store,
|
||||
settings: { worktreeInitCommand: "pnpm i" } as any,
|
||||
createWorktree: vi.fn().mockResolvedValue({ path: "/tmp/new", branch: "fusion/fn-1" }),
|
||||
runConfiguredCommand,
|
||||
runInitCommand: false,
|
||||
});
|
||||
expect(runConfiguredCommand).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
} from "./agent-instructions.js";
|
||||
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
|
||||
import { heartbeatLog, formatError } from "./logger.js";
|
||||
import { acquireTaskWorktree } from "./worktree-acquisition.js";
|
||||
import { createRunAuditor, type EngineRunContext } from "./run-audit.js";
|
||||
import { promptWithFallback } from "./pi.js";
|
||||
import { createResolvedAgentSession, extractRuntimeHint, resolveHeartbeatSessionModels } from "./agent-session-helpers.js";
|
||||
@@ -2103,6 +2104,36 @@ export class HeartbeatMonitor {
|
||||
heartbeatLog.warn(`Failed to read heartbeat model settings for ${agentId}: ${settingsErr instanceof Error ? settingsErr.message : String(settingsErr)}`);
|
||||
}
|
||||
|
||||
let sessionCwd = rootDir;
|
||||
if (!isNoTaskRun && taskDetail && heartbeatModelSettings) {
|
||||
try {
|
||||
const acquisition = await acquireTaskWorktree({
|
||||
task: taskDetail,
|
||||
rootDir,
|
||||
store: taskStore,
|
||||
settings: heartbeatModelSettings,
|
||||
logger: heartbeatLog,
|
||||
audit,
|
||||
runContext,
|
||||
runInitCommand: false,
|
||||
});
|
||||
sessionCwd = acquisition.worktreePath;
|
||||
} catch (worktreeErr) {
|
||||
const detail = worktreeErr instanceof Error ? worktreeErr.message : String(worktreeErr);
|
||||
heartbeatLog.warn(`Heartbeat worktree acquisition failed for ${agentId}: ${detail}`);
|
||||
if (taskDetail.column !== "done" && taskDetail.column !== "archived") {
|
||||
await taskStore.moveTask(taskDetail.id, "todo", { preserveProgress: true });
|
||||
}
|
||||
await this.completeRun(agentId, run.id, {
|
||||
status: "completed",
|
||||
resultJson: { reason: "worktree_acquisition_failed", detail },
|
||||
stderrExcerpt: detail,
|
||||
skipStateTransition: true,
|
||||
});
|
||||
return (await this.store.getRunDetail(agentId, run.id))!;
|
||||
}
|
||||
}
|
||||
|
||||
const heartbeatSessionModels = resolveHeartbeatSessionModels(heartbeatModelSettings, agent.runtimeConfig);
|
||||
|
||||
// Create agent session
|
||||
@@ -2110,7 +2141,7 @@ export class HeartbeatMonitor {
|
||||
sessionPurpose: "heartbeat",
|
||||
runtimeHint: extractRuntimeHint(agent.runtimeConfig),
|
||||
pluginRunner: this.pluginRunner,
|
||||
cwd: rootDir,
|
||||
cwd: sessionCwd,
|
||||
systemPrompt: systemPromptFinal,
|
||||
systemPromptLayers: heartbeatLayers,
|
||||
tools: "coding",
|
||||
|
||||
@@ -23,7 +23,7 @@ import {
|
||||
VERIFICATION_LOG_MAX_CHARS,
|
||||
type VerificationResult,
|
||||
} from "./verification-utils.js";
|
||||
import { generateWorktreeName, slugify } from "./worktree-names.js";
|
||||
import { generateWorktreeName } from "./worktree-names.js";
|
||||
import { Type, type Static } from "@mariozechner/pi-ai";
|
||||
import { describeModel, promptWithFallback, compactSessionContext } from "./pi.js";
|
||||
import { accumulateSessionTokenUsage } from "./session-token-usage.js";
|
||||
@@ -36,7 +36,7 @@ import { buildSessionSkillContext } from "./session-skill-context.js";
|
||||
import { reviewStep, type ReviewVerdict } from "./reviewer.js";
|
||||
import { ModelRegistry, SessionManager, type ToolDefinition, type AgentSession } from "@mariozechner/pi-coding-agent";
|
||||
import { PRIORITY_EXECUTE, type AgentSemaphore } from "./concurrency.js";
|
||||
import { getRegisteredWorktreePaths, isGitRepository, isRegisteredGitWorktree, isUsableTaskWorktree, type WorktreePool } from "./worktree-pool.js";
|
||||
import { getRegisteredWorktreePaths, isGitRepository, isRegisteredGitWorktree, type WorktreePool } from "./worktree-pool.js";
|
||||
import { BranchConflictError, isBranchConflictError, inspectBranchConflict } from "./branch-conflicts.js";
|
||||
import { AgentLogger } from "./agent-logger.js";
|
||||
import { executorLog, reviewerLog, formatError } from "./logger.js";
|
||||
@@ -49,7 +49,7 @@ import type { StuckTaskDetector, StuckTaskEvent } from "./stuck-task-detector.js
|
||||
import type { PluginRunner } from "./plugin-runner.js";
|
||||
import { isContextLimitError } from "./context-limit-detector.js";
|
||||
import { StepSessionExecutor } from "./step-session-executor.js";
|
||||
import { hydrateWorktreeDb } from "./worktree-db-hydrate.js";
|
||||
import { acquireTaskWorktree } from "./worktree-acquisition.js";
|
||||
import {
|
||||
resolveAgentInstructions,
|
||||
buildSystemPromptWithInstructions,
|
||||
@@ -2416,28 +2416,7 @@ export class TaskExecutor {
|
||||
}
|
||||
|
||||
// Hoist worktreePath so it's accessible in the catch block for dep-abort cleanup
|
||||
// Determine worktree name based on settings
|
||||
let worktreePath: string;
|
||||
if (task.worktree) {
|
||||
worktreePath = task.worktree;
|
||||
} else {
|
||||
const naming = settings.worktreeNaming || "random";
|
||||
let worktreeName: string;
|
||||
|
||||
switch (naming) {
|
||||
case "task-id":
|
||||
worktreeName = task.id.toLowerCase();
|
||||
break;
|
||||
case "task-title":
|
||||
worktreeName = slugify(task.title || task.description.slice(0, 60));
|
||||
break;
|
||||
case "random":
|
||||
default:
|
||||
worktreeName = generateWorktreeName(this.rootDir);
|
||||
break;
|
||||
}
|
||||
worktreePath = join(this.rootDir, ".worktrees", worktreeName);
|
||||
}
|
||||
let worktreePath = task.worktree ?? "";
|
||||
|
||||
// Set by stuck-abort handlers; the actual moveTask("todo") is deferred to
|
||||
// the finally block so this.executing is cleared first (prevents re-dispatch race).
|
||||
@@ -2472,255 +2451,48 @@ export class TaskExecutor {
|
||||
);
|
||||
}
|
||||
|
||||
// Create or reuse worktree — try pool first when recycling is enabled
|
||||
// Prefer the persisted branch from a prior run so the agent resumes on the
|
||||
// same branch instead of creating a fresh fusion/fn-XXXX-2, -3, etc.
|
||||
const branchName = task.branch || `fusion/${task.id.toLowerCase()}`;
|
||||
// Use generateWorktreeName for human-friendly directory names (adjective-noun pattern)
|
||||
// instead of task.id, so worktrees are named like ".worktrees/swift-falcon"
|
||||
let isResume = existsSync(worktreePath);
|
||||
let acquiredFromPool = false;
|
||||
const acquisition = await acquireTaskWorktree({
|
||||
task,
|
||||
rootDir: this.rootDir,
|
||||
store: this.store,
|
||||
settings,
|
||||
pool: this.options.pool,
|
||||
logger: executorLog,
|
||||
audit,
|
||||
runContext: this.currentRunContext,
|
||||
runInitCommand: true,
|
||||
createWorktree: this.createWorktree.bind(this),
|
||||
runConfiguredCommand,
|
||||
taskEnv,
|
||||
});
|
||||
worktreePath = acquisition.worktreePath;
|
||||
|
||||
// Resolve the base branch — set by the scheduler when a dep is in-review
|
||||
const baseBranch = task.executionStartBranch || null;
|
||||
const allowSiblingBranchRename = settings.executorAllowSiblingBranchRename === true;
|
||||
|
||||
if (task.worktree && isResume && !await isUsableTaskWorktree(this.rootDir, worktreePath)) {
|
||||
const invalidWorktreePath = worktreePath;
|
||||
executorLog.log(`${task.id}: assigned worktree is not usable; creating a fresh worktree instead: ${invalidWorktreePath}`);
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Assigned worktree is not a registered, usable git worktree; creating a fresh worktree instead`,
|
||||
invalidWorktreePath,
|
||||
this.currentRunContext,
|
||||
);
|
||||
await this.store.updateTask(task.id, { worktree: null, branch: null });
|
||||
worktreePath = join(this.rootDir, ".worktrees", generateWorktreeName(this.rootDir));
|
||||
isResume = existsSync(worktreePath);
|
||||
}
|
||||
|
||||
if (!isResume) {
|
||||
|
||||
// Try acquiring a warm worktree from the pool
|
||||
if (this.options.pool && settings.recycleWorktrees) {
|
||||
const pooled = this.options.pool.acquire();
|
||||
if (pooled) {
|
||||
try {
|
||||
const actualBranch = await this.options.pool.prepareForTask(
|
||||
pooled,
|
||||
branchName,
|
||||
baseBranch ?? undefined,
|
||||
{ allowSiblingBranchRename, repoDir: this.rootDir },
|
||||
);
|
||||
worktreePath = pooled;
|
||||
acquiredFromPool = true;
|
||||
executorLog.log(`Acquired worktree from pool: ${pooled}`);
|
||||
await this.store.updateTask(task.id, { worktree: worktreePath, branch: actualBranch });
|
||||
// Audit trail: record worktree reuse (FN-1404)
|
||||
await audit.git({ type: "worktree:reuse", target: worktreePath, metadata: { branch: actualBranch } });
|
||||
if (actualBranch !== branchName) {
|
||||
executorLog.log(`Branch conflict resolved: using ${actualBranch} instead of ${branchName}`);
|
||||
await this.store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath} (branch conflict: using ${actualBranch})`, undefined, this.currentRunContext);
|
||||
} else {
|
||||
await this.store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath}`, undefined, this.currentRunContext);
|
||||
}
|
||||
|
||||
if (this.rootDir !== worktreePath) {
|
||||
try {
|
||||
const hydration = await hydrateWorktreeDb({
|
||||
rootDir: this.rootDir,
|
||||
worktreePath,
|
||||
taskId: task.id,
|
||||
store: this.store,
|
||||
logger: executorLog,
|
||||
});
|
||||
if (hydration.degraded) {
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Worktree DB hydration degraded: ${hydration.reason ?? "unknown"}`,
|
||||
undefined,
|
||||
this.currentRunContext,
|
||||
);
|
||||
} else {
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Hydrated worktree DB: ${hydration.tasksCopied} tasks, ${hydration.documentsCopied} task_documents`,
|
||||
undefined,
|
||||
this.currentRunContext,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
executorLog.warn(`${task.id}: worktree DB hydration failed: ${formatError(error)}`);
|
||||
}
|
||||
}
|
||||
} catch (poolErr: unknown) {
|
||||
this.options.pool.release(pooled);
|
||||
if (isBranchConflictError(poolErr)) {
|
||||
throw poolErr;
|
||||
}
|
||||
const poolErrMessage = poolErr instanceof Error ? poolErr.message : String(poolErr);
|
||||
executorLog.log(`Pool prepareForTask failed, falling through to fresh worktree: ${poolErrMessage}`);
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Pool worktree preparation failed (${poolErrMessage}), creating fresh worktree`,
|
||||
undefined,
|
||||
this.currentRunContext,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall through to fresh worktree creation if pool had nothing
|
||||
if (!acquiredFromPool) {
|
||||
const created = await this.createWorktree(branchName, worktreePath, task.id, baseBranch ?? undefined, allowSiblingBranchRename);
|
||||
worktreePath = created.path;
|
||||
await this.store.updateTask(task.id, { worktree: created.path, branch: created.branch });
|
||||
// Audit trail: record worktree creation and branch creation (FN-1404)
|
||||
await audit.git({ type: "worktree:create", target: created.path, metadata: { branch: created.branch } });
|
||||
await audit.git({ type: "branch:create", target: created.branch });
|
||||
if (created.branch !== branchName) {
|
||||
executorLog.log(`Branch conflict resolved: using ${created.branch} instead of ${branchName}`);
|
||||
await this.store.logEntry(task.id, `Worktree created at ${worktreePath} (branch conflict: using ${created.branch})`, undefined, this.currentRunContext);
|
||||
} else if (baseBranch) {
|
||||
await this.store.logEntry(task.id, `Worktree created at ${worktreePath} (based on ${baseBranch})`, undefined, this.currentRunContext);
|
||||
} else {
|
||||
await this.store.logEntry(task.id, `Worktree created at ${worktreePath}`, undefined, this.currentRunContext);
|
||||
}
|
||||
|
||||
// Run worktree init command for fresh worktrees (skip for pooled — caches are warm).
|
||||
// The init command should deterministically install the full dependency
|
||||
// graph required by test/typecheck/build commands (for pnpm workspaces,
|
||||
// prefer `pnpm install --frozen-lockfile`) so transitive modules and
|
||||
// declarations like @vitest/runner, loupe, debug, @types/express, and
|
||||
// node-pty are present after bootstrap.
|
||||
//
|
||||
// NOTE: This is distinct from the separate workspace-export failure
|
||||
// class where internal packages fail to resolve because exports point
|
||||
// to missing dist/* outputs (e.g. "@fusion/core entry not found").
|
||||
// 5-minute timeout accommodates larger dependency installs.
|
||||
if (settings.worktreeInitCommand) {
|
||||
const initStartedAt = Date.now();
|
||||
try {
|
||||
const initResult = await runConfiguredCommand(settings.worktreeInitCommand, worktreePath, 300_000, taskEnv);
|
||||
if (initResult.spawnError || initResult.timedOut || initResult.exitCode !== 0) {
|
||||
throw new Error(configuredCommandErrorMessage(initResult));
|
||||
}
|
||||
await this.store.logEntry(task.id, `[timing] Worktree init command completed in ${Date.now() - initStartedAt}ms`, settings.worktreeInitCommand, this.currentRunContext);
|
||||
} catch (err: unknown) {
|
||||
await this.store.logEntry(task.id, `[timing] Worktree init command failed after ${Date.now() - initStartedAt}ms`, undefined, this.currentRunContext);
|
||||
const execError = err instanceof Error ? err : new Error(String(err));
|
||||
const message = "stderr" in execError && typeof (execError as Record<string, unknown>).stderr === "string"
|
||||
? String((execError as Record<string, unknown>).stderr)
|
||||
: execError.message;
|
||||
executorLog.error(`${task.id}: worktree init command failed — first test run will likely fail: ${message}`);
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Worktree init command failed (first test run will likely fail): ${message}`,
|
||||
undefined,
|
||||
this.currentRunContext,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Run setup script for fresh worktrees (after worktreeInitCommand)
|
||||
if (settings.setupScript) {
|
||||
const scriptCommand = settings.scripts?.[settings.setupScript];
|
||||
if (scriptCommand) {
|
||||
const setupStartedAt = Date.now();
|
||||
try {
|
||||
const setupResult = await runConfiguredCommand(scriptCommand, worktreePath, 120_000, taskEnv);
|
||||
if (setupResult.spawnError || setupResult.timedOut || setupResult.exitCode !== 0) {
|
||||
throw new Error(configuredCommandErrorMessage(setupResult));
|
||||
}
|
||||
await this.store.logEntry(task.id, `[timing] Setup script '${settings.setupScript}' completed in ${Date.now() - setupStartedAt}ms`, scriptCommand, this.currentRunContext);
|
||||
} catch (err: unknown) {
|
||||
const execError = err instanceof Error ? err : new Error(String(err));
|
||||
const message = "stderr" in execError && typeof (execError as Record<string, unknown>).stderr === "string"
|
||||
? String((execError as Record<string, unknown>).stderr)
|
||||
: execError.message;
|
||||
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' failed: ${message}`, undefined, this.currentRunContext);
|
||||
}
|
||||
} else {
|
||||
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' not found in scripts map — skipping`, undefined, this.currentRunContext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquiredFromPool) {
|
||||
if (this.rootDir !== worktreePath) {
|
||||
try {
|
||||
const hydration = await hydrateWorktreeDb({
|
||||
rootDir: this.rootDir,
|
||||
worktreePath,
|
||||
taskId: task.id,
|
||||
store: this.store,
|
||||
logger: executorLog,
|
||||
});
|
||||
if (hydration.degraded) {
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Worktree DB hydration degraded: ${hydration.reason ?? "unknown"}`,
|
||||
undefined,
|
||||
this.currentRunContext,
|
||||
);
|
||||
} else {
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Hydrated worktree DB: ${hydration.tasksCopied} tasks, ${hydration.documentsCopied} task_documents`,
|
||||
undefined,
|
||||
this.currentRunContext,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
executorLog.warn(`${task.id}: worktree DB hydration failed: ${formatError(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (task.worktree) {
|
||||
// Task already had a worktree assigned and it exists on disk — reuse it
|
||||
executorLog.log(`Reusing existing worktree: ${worktreePath}`);
|
||||
if (this.rootDir !== worktreePath) {
|
||||
if (!acquisition.isResume && acquisition.source === "fresh" && settings.setupScript) {
|
||||
const scriptCommand = settings.scripts?.[settings.setupScript];
|
||||
if (scriptCommand) {
|
||||
const setupStartedAt = Date.now();
|
||||
try {
|
||||
const hydration = await hydrateWorktreeDb({
|
||||
rootDir: this.rootDir,
|
||||
worktreePath,
|
||||
taskId: task.id,
|
||||
store: this.store,
|
||||
logger: executorLog,
|
||||
});
|
||||
if (hydration.degraded) {
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Worktree DB hydration degraded: ${hydration.reason ?? "unknown"}`,
|
||||
undefined,
|
||||
this.currentRunContext,
|
||||
);
|
||||
} else {
|
||||
await this.store.logEntry(
|
||||
task.id,
|
||||
`Hydrated worktree DB: ${hydration.tasksCopied} tasks, ${hydration.documentsCopied} task_documents`,
|
||||
undefined,
|
||||
this.currentRunContext,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
executorLog.warn(`${task.id}: worktree DB hydration failed: ${formatError(error)}`);
|
||||
const setupResult = await runConfiguredCommand(scriptCommand, worktreePath, 120_000, taskEnv);
|
||||
if (setupResult.spawnError || setupResult.timedOut || setupResult.exitCode !== 0) {
|
||||
throw new Error(configuredCommandErrorMessage(setupResult));
|
||||
}
|
||||
await this.store.logEntry(task.id, `[timing] Setup script '${settings.setupScript}' completed in ${Date.now() - setupStartedAt}ms`, scriptCommand, this.currentRunContext);
|
||||
} catch (err: unknown) {
|
||||
const execError = err instanceof Error ? err : new Error(String(err));
|
||||
const message = "stderr" in execError && typeof (execError as Record<string, unknown>).stderr === "string"
|
||||
? String((execError as Record<string, unknown>).stderr)
|
||||
: execError.message;
|
||||
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' failed: ${message}`, undefined, this.currentRunContext);
|
||||
}
|
||||
} else {
|
||||
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' not found in scripts map — skipping`, undefined, this.currentRunContext);
|
||||
}
|
||||
} else {
|
||||
// Directory exists at generated path but task has no worktree — create via normal flow
|
||||
const created = await this.createWorktree(branchName, worktreePath, task.id, undefined, allowSiblingBranchRename);
|
||||
worktreePath = created.path;
|
||||
await this.store.updateTask(task.id, { worktree: created.path, branch: created.branch });
|
||||
// Audit trail: record worktree creation and branch creation (FN-1404)
|
||||
await audit.git({ type: "worktree:create", target: created.path, metadata: { branch: created.branch } });
|
||||
await audit.git({ type: "branch:create", target: created.branch });
|
||||
}
|
||||
|
||||
// Capture the base commit SHA for diff computation whenever a task
|
||||
// starts with a newly assigned worktree. Recycled worktrees must
|
||||
// overwrite any prior task baseline instead of inheriting it.
|
||||
if (!isResume) {
|
||||
if (!acquisition.isResume) {
|
||||
try {
|
||||
const { stdout } = await execAsync("git rev-parse HEAD", {
|
||||
cwd: worktreePath,
|
||||
@@ -2776,7 +2548,7 @@ export class TaskExecutor {
|
||||
|
||||
// On resume (task.branch already set from a prior run), reconcile step
|
||||
// statuses from git history so the agent doesn't redo already-committed work.
|
||||
if (isResume && task.branch && detail.steps.length > 0) {
|
||||
if (acquisition.isResume && task.branch && detail.steps.length > 0) {
|
||||
await this.reconcileStepsFromGitHistory(task.id, detail, worktreePath);
|
||||
}
|
||||
|
||||
|
||||
207
packages/engine/src/worktree-acquisition.ts
Normal file
207
packages/engine/src/worktree-acquisition.ts
Normal file
@@ -0,0 +1,207 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { exec } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { join } from "node:path";
|
||||
import type { RunMutationContext, Settings, Task, TaskStore } from "@fusion/core";
|
||||
import { generateWorktreeName, slugify } from "./worktree-names.js";
|
||||
import { hydrateWorktreeDb } from "./worktree-db-hydrate.js";
|
||||
import { formatError } from "./logger.js";
|
||||
import { isBranchConflictError } from "./branch-conflicts.js";
|
||||
import { type WorktreePool, isUsableTaskWorktree } from "./worktree-pool.js";
|
||||
import type { RunAuditor } from "./run-audit.js";
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
/**
|
||||
* Worktree acquisition contract:
|
||||
* - `runInitCommand=true` runs the init command only for newly-created worktrees (fresh, not pool/existing).
|
||||
* - Heartbeat task runs should pass `runInitCommand=false`.
|
||||
* - Executor may pass `runInitCommand=true`; if heartbeat created the worktree earlier, executor reuses it and init is skipped.
|
||||
*/
|
||||
export interface AcquireTaskWorktreeOptions {
|
||||
task: Task;
|
||||
rootDir: string;
|
||||
store: TaskStore;
|
||||
settings: Partial<Settings>;
|
||||
pool?: WorktreePool;
|
||||
logger?: { log: (m: string) => void; warn: (m: string) => void; error?: (m: string) => void };
|
||||
audit?: Pick<RunAuditor, "git">;
|
||||
runContext?: RunMutationContext;
|
||||
runInitCommand?: boolean;
|
||||
createWorktree?: (
|
||||
branch: string,
|
||||
path: string,
|
||||
taskId: string,
|
||||
startPoint?: string,
|
||||
allowSiblingBranchRename?: boolean,
|
||||
) => Promise<{ path: string; branch: string }>;
|
||||
runConfiguredCommand?: (command: string, cwd: string, timeoutMs: number, env?: NodeJS.ProcessEnv) => Promise<{ spawnError?: string | Error; timedOut?: boolean; exitCode?: number | null }>;
|
||||
taskEnv?: NodeJS.ProcessEnv;
|
||||
}
|
||||
|
||||
export interface AcquireTaskWorktreeResult {
|
||||
worktreePath: string;
|
||||
branch: string;
|
||||
source: "existing" | "pool" | "fresh";
|
||||
hydrated: boolean;
|
||||
isResume: boolean;
|
||||
}
|
||||
|
||||
function configuredCommandErrorMessage(result: { spawnError?: string | Error; timedOut?: boolean; exitCode?: number | null }): string {
|
||||
if (result.spawnError) return `Failed to start command: ${result.spawnError}`;
|
||||
if (result.timedOut) return "Command timed out";
|
||||
return `Command exited with code ${result.exitCode ?? "unknown"}`;
|
||||
}
|
||||
|
||||
async function createWorktreeFallback(
|
||||
rootDir: string,
|
||||
branch: string,
|
||||
path: string,
|
||||
startPoint?: string,
|
||||
allowSiblingBranchRename = false,
|
||||
): Promise<{ path: string; branch: string }> {
|
||||
const escapedPath = JSON.stringify(path);
|
||||
const escapedBranch = JSON.stringify(branch);
|
||||
const escapedStart = startPoint ? JSON.stringify(startPoint) : undefined;
|
||||
const startArg = escapedStart ? ` ${escapedStart}` : "";
|
||||
|
||||
try {
|
||||
await execAsync(`git worktree add -b ${escapedBranch} ${escapedPath}${startArg}`, { cwd: rootDir });
|
||||
return { path, branch };
|
||||
} catch (error) {
|
||||
if (!allowSiblingBranchRename) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
for (let suffix = 2; suffix <= 50; suffix += 1) {
|
||||
const candidate = `${branch}-${suffix}`;
|
||||
const escapedCandidate = JSON.stringify(candidate);
|
||||
try {
|
||||
await execAsync(`git worktree add -b ${escapedCandidate} ${escapedPath}${startArg}`, { cwd: rootDir });
|
||||
return { path, branch: candidate };
|
||||
} catch {
|
||||
// try next suffix
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function acquireTaskWorktree(opts: AcquireTaskWorktreeOptions): Promise<AcquireTaskWorktreeResult> {
|
||||
const { task, rootDir, store, settings, pool, logger, audit, runContext, createWorktree, runConfiguredCommand, runInitCommand, taskEnv } = opts;
|
||||
const branchName = task.branch || `fusion/${task.id.toLowerCase()}`;
|
||||
const naming = settings.worktreeNaming || "random";
|
||||
const allowSiblingBranchRename = settings.executorAllowSiblingBranchRename === true;
|
||||
const baseBranch = task.executionStartBranch || null;
|
||||
|
||||
let worktreePath = task.worktree;
|
||||
if (!worktreePath) {
|
||||
const worktreeName = naming === "task-id"
|
||||
? task.id.toLowerCase()
|
||||
: naming === "task-title"
|
||||
? slugify(task.title || task.description.slice(0, 60))
|
||||
: generateWorktreeName(rootDir);
|
||||
worktreePath = join(rootDir, ".worktrees", worktreeName);
|
||||
}
|
||||
|
||||
let isResume = Boolean(task.worktree && existsSync(worktreePath));
|
||||
if (task.worktree && isResume && !await isUsableTaskWorktree(rootDir, worktreePath)) {
|
||||
logger?.log(`${task.id}: assigned worktree is not usable; creating a fresh worktree instead: ${worktreePath}`);
|
||||
await store.logEntry(task.id, "Assigned worktree is not a registered, usable git worktree; creating a fresh worktree instead", worktreePath, runContext);
|
||||
await store.updateTask(task.id, { worktree: null, branch: null });
|
||||
worktreePath = join(rootDir, ".worktrees", generateWorktreeName(rootDir));
|
||||
isResume = false;
|
||||
}
|
||||
|
||||
const hydrate = async (path: string): Promise<boolean> => {
|
||||
if (rootDir === path) return false;
|
||||
try {
|
||||
const hydration = await hydrateWorktreeDb({ rootDir, worktreePath: path, taskId: task.id, store, logger: logger ?? { warn: () => {} } });
|
||||
if (hydration.degraded) {
|
||||
await store.logEntry(task.id, `Worktree DB hydration degraded: ${hydration.reason ?? "unknown"}`, undefined, runContext);
|
||||
} else {
|
||||
await store.logEntry(task.id, `Hydrated worktree DB: ${hydration.tasksCopied} tasks, ${hydration.documentsCopied} task_documents`, undefined, runContext);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger?.warn(`${task.id}: worktree DB hydration failed: ${formatError(error)}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
if (task.worktree && isResume) {
|
||||
logger?.log(`Reusing existing worktree: ${worktreePath}`);
|
||||
const hydrated = await hydrate(worktreePath);
|
||||
return { worktreePath, branch: task.branch ?? branchName, source: "existing", hydrated, isResume: true };
|
||||
}
|
||||
|
||||
let acquiredFromPool = false;
|
||||
let branch = branchName;
|
||||
|
||||
if (!isResume && pool && settings.recycleWorktrees) {
|
||||
const pooled = pool.acquire();
|
||||
if (pooled) {
|
||||
try {
|
||||
const actualBranch = await pool.prepareForTask(pooled, branchName, baseBranch ?? undefined, { allowSiblingBranchRename, repoDir: rootDir });
|
||||
worktreePath = pooled;
|
||||
branch = actualBranch;
|
||||
acquiredFromPool = true;
|
||||
logger?.log(`Acquired worktree from pool: ${pooled}`);
|
||||
await store.updateTask(task.id, { worktree: worktreePath, branch: actualBranch });
|
||||
await audit?.git({ type: "worktree:reuse", target: worktreePath, metadata: { branch: actualBranch } });
|
||||
if (actualBranch !== branchName) {
|
||||
logger?.log(`Branch conflict resolved: using ${actualBranch} instead of ${branchName}`);
|
||||
await store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath} (branch conflict: using ${actualBranch})`, undefined, runContext);
|
||||
} else {
|
||||
await store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath}`, undefined, runContext);
|
||||
}
|
||||
const hydrated = await hydrate(worktreePath);
|
||||
return { worktreePath, branch, source: "pool", hydrated, isResume: false };
|
||||
} catch (poolErr) {
|
||||
pool.release(pooled);
|
||||
if (isBranchConflictError(poolErr)) throw poolErr;
|
||||
const poolErrMessage = poolErr instanceof Error ? poolErr.message : String(poolErr);
|
||||
logger?.log(`Pool prepareForTask failed, falling through to fresh worktree: ${poolErrMessage}`);
|
||||
await store.logEntry(task.id, `Pool worktree preparation failed (${poolErrMessage}), creating fresh worktree`, undefined, runContext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const createWorktreeImpl = createWorktree
|
||||
? createWorktree
|
||||
: (branch: string, path: string, _taskId: string, startPoint?: string, allowRename?: boolean) => createWorktreeFallback(rootDir, branch, path, startPoint, allowRename);
|
||||
|
||||
const created = await createWorktreeImpl(branchName, worktreePath, task.id, baseBranch ?? undefined, allowSiblingBranchRename);
|
||||
worktreePath = created.path;
|
||||
branch = created.branch;
|
||||
await store.updateTask(task.id, { worktree: created.path, branch: created.branch });
|
||||
await audit?.git({ type: "worktree:create", target: created.path, metadata: { branch: created.branch } });
|
||||
await audit?.git({ type: "branch:create", target: created.branch });
|
||||
if (created.branch !== branchName) {
|
||||
logger?.log(`Branch conflict resolved: using ${created.branch} instead of ${branchName}`);
|
||||
await store.logEntry(task.id, `Worktree created at ${worktreePath} (branch conflict: using ${created.branch})`, undefined, runContext);
|
||||
} else if (baseBranch) {
|
||||
await store.logEntry(task.id, `Worktree created at ${worktreePath} (based on ${baseBranch})`, undefined, runContext);
|
||||
} else {
|
||||
await store.logEntry(task.id, `Worktree created at ${worktreePath}`, undefined, runContext);
|
||||
}
|
||||
|
||||
if (runInitCommand && settings.worktreeInitCommand && runConfiguredCommand) {
|
||||
const initStartedAt = Date.now();
|
||||
try {
|
||||
const initResult = await runConfiguredCommand(settings.worktreeInitCommand, worktreePath, 300_000, taskEnv);
|
||||
if (initResult.spawnError || initResult.timedOut || initResult.exitCode !== 0) {
|
||||
throw new Error(configuredCommandErrorMessage(initResult));
|
||||
}
|
||||
await store.logEntry(task.id, `[timing] Worktree init command completed in ${Date.now() - initStartedAt}ms`, settings.worktreeInitCommand, runContext);
|
||||
} catch (err) {
|
||||
await store.logEntry(task.id, `[timing] Worktree init command failed after ${Date.now() - initStartedAt}ms`, undefined, runContext);
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
logger?.error?.(`${task.id}: worktree init command failed — first test run will likely fail: ${message}`);
|
||||
await store.logEntry(task.id, `Worktree init command failed (first test run will likely fail): ${message}`, undefined, runContext);
|
||||
}
|
||||
}
|
||||
|
||||
const hydrated = await hydrate(worktreePath);
|
||||
return { worktreePath, branch, source: acquiredFromPool ? "pool" : "fresh", hydrated, isResume: false };
|
||||
}
|
||||
Reference in New Issue
Block a user