feat(FN-4257): run durable heartbeat task sessions in worktree
Fusion-Task-Id: FN-4257 Fusion-Task-Lineage: db48d51e-9e7d-471d-b26b-2a8763fb41d1
This commit is contained in:
5
.changeset/fn-4257-permanent-agent-worktree.md
Normal file
5
.changeset/fn-4257-permanent-agent-worktree.md
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
Run task-scoped heartbeat sessions for permanent (durable) agents inside the task's git worktree, matching ephemeral-agent task execution. No-task heartbeats continue to use the project root.
|
||||||
@@ -234,6 +234,8 @@ Heartbeat sessions for durable agents resolve models with heartbeat-specific fal
|
|||||||
|
|
||||||
When the runtime model is present and differs from execution-lane settings, heartbeat passes the execution-lane model as a fallback pair for session creation.
|
When the runtime model is present and differs from execution-lane settings, heartbeat passes the execution-lane model as a fallback pair for session creation.
|
||||||
|
|
||||||
|
Task-scoped heartbeat runs for durable agents execute inside the task's git worktree (same as ephemeral task execution), while no-task heartbeat runs continue to execute from the project root.
|
||||||
|
|
||||||
If a heartbeat cannot create/run a session due to unavailable provider credentials or missing provider registration, Fusion records `resultJson.reason = "heartbeat_model_unavailable"` with actionable diagnostics in `resultJson.detail`/`stderrExcerpt`.
|
If a heartbeat cannot create/run a session due to unavailable provider credentials or missing provider registration, Fusion records `resultJson.reason = "heartbeat_model_unavailable"` with actionable diagnostics in `resultJson.detail`/`stderrExcerpt`.
|
||||||
|
|
||||||
### Durable-agent transient error auto-recovery
|
### Durable-agent transient error auto-recovery
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||||
|
import type { Agent, AgentHeartbeatRun } from "@fusion/core";
|
||||||
|
import { HeartbeatMonitor } from "../agent-heartbeat.js";
|
||||||
|
import * as worktreeAcquisition from "../worktree-acquisition.js";
|
||||||
|
import * as piModule from "../pi.js";
|
||||||
|
|
||||||
|
describe("heartbeat worktree cwd", () => {
|
||||||
|
let store: any;
|
||||||
|
let taskStore: any;
|
||||||
|
const agent: Agent = { id: "a1", name: "A", role: "executor", state: "active", taskId: "FN-1", createdAt: "", updatedAt: "", metadata: {} } as any;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
vi.spyOn(piModule, "createFnAgent").mockResolvedValue({ session: { prompt: vi.fn(), dispose: vi.fn() } } as any);
|
||||||
|
vi.spyOn(worktreeAcquisition, "acquireTaskWorktree").mockResolvedValue({ worktreePath: "/tmp/wt", branch: "fusion/fn-1", source: "existing", hydrated: false, isResume: true });
|
||||||
|
|
||||||
|
const run: AgentHeartbeatRun = { id: "r1", agentId: "a1", status: "active", startedAt: new Date().toISOString(), endedAt: null } as any;
|
||||||
|
store = {
|
||||||
|
startHeartbeatRun: vi.fn().mockResolvedValue(run),
|
||||||
|
saveRun: vi.fn(),
|
||||||
|
getRunDetail: vi.fn().mockResolvedValue(run),
|
||||||
|
getAgent: vi.fn().mockResolvedValue(agent),
|
||||||
|
updateAgentState: vi.fn(),
|
||||||
|
updateAgent: vi.fn(),
|
||||||
|
endHeartbeatRun: vi.fn(),
|
||||||
|
assignTask: vi.fn(),
|
||||||
|
getBudgetStatus: vi.fn().mockResolvedValue({ isOverBudget: false, isOverThreshold: false, usagePercent: 0 }),
|
||||||
|
getCachedAgent: vi.fn().mockReturnValue(null),
|
||||||
|
getLastBlockedState: vi.fn().mockResolvedValue(null),
|
||||||
|
setLastBlockedState: vi.fn(),
|
||||||
|
clearLastBlockedState: vi.fn(),
|
||||||
|
appendRunLog: vi.fn(),
|
||||||
|
getAgentsByReportsTo: vi.fn().mockResolvedValue([]),
|
||||||
|
recordHeartbeat: vi.fn(),
|
||||||
|
};
|
||||||
|
taskStore = {
|
||||||
|
getSettings: vi.fn().mockResolvedValue({}),
|
||||||
|
getTask: vi.fn().mockResolvedValue({ id: "FN-1", title: "t", description: "d", column: "todo", dependencies: [], steps: [], log: [] }),
|
||||||
|
moveTask: vi.fn(),
|
||||||
|
appendAgentLog: vi.fn(),
|
||||||
|
listTasks: vi.fn().mockResolvedValue([]),
|
||||||
|
selectNextTaskForAgent: vi.fn().mockResolvedValue(null),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses acquired worktree cwd for task-scoped runs", async () => {
|
||||||
|
const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: "/repo" });
|
||||||
|
await monitor.executeHeartbeat({ agentId: "a1", source: "on_demand" });
|
||||||
|
expect(worktreeAcquisition.acquireTaskWorktree).toHaveBeenCalled();
|
||||||
|
expect(piModule.createFnAgent).toHaveBeenCalledWith(expect.objectContaining({ cwd: "/tmp/wt" }));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses rootDir for no-task runs", async () => {
|
||||||
|
store.getAgent.mockResolvedValue({ ...agent, taskId: undefined, soul: "x" });
|
||||||
|
const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: "/repo" });
|
||||||
|
await monitor.executeHeartbeat({ agentId: "a1", source: "on_demand" });
|
||||||
|
expect(worktreeAcquisition.acquireTaskWorktree).not.toHaveBeenCalled();
|
||||||
|
expect(piModule.createFnAgent).toHaveBeenCalledWith(expect.objectContaining({ cwd: "/repo" }));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("completes with worktree_acquisition_failed when helper throws", async () => {
|
||||||
|
vi.spyOn(worktreeAcquisition, "acquireTaskWorktree").mockRejectedValueOnce(new Error("nope"));
|
||||||
|
const monitor = new HeartbeatMonitor({ store, taskStore, rootDir: "/repo" });
|
||||||
|
await monitor.executeHeartbeat({ agentId: "a1", source: "on_demand" });
|
||||||
|
expect(piModule.createFnAgent).not.toHaveBeenCalled();
|
||||||
|
expect(taskStore.moveTask).toHaveBeenCalledWith("FN-1", "todo", { preserveProgress: true });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -27,7 +27,13 @@ vi.mock("../pi.js", () => ({
|
|||||||
}),
|
}),
|
||||||
}));
|
}));
|
||||||
import { createFnAgent } from "../pi.js";
|
import { createFnAgent } from "../pi.js";
|
||||||
|
import { acquireTaskWorktree } from "../worktree-acquisition.js";
|
||||||
const mockedCreateFnAgent = vi.mocked(createFnAgent);
|
const mockedCreateFnAgent = vi.mocked(createFnAgent);
|
||||||
|
const mockedAcquireTaskWorktree = vi.mocked(acquireTaskWorktree);
|
||||||
|
|
||||||
|
vi.mock("../worktree-acquisition.js", () => ({
|
||||||
|
acquireTaskWorktree: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
describe("executeHeartbeat", () => {
|
describe("executeHeartbeat", () => {
|
||||||
let mockTaskStore: TaskStore;
|
let mockTaskStore: TaskStore;
|
||||||
@@ -57,6 +63,8 @@ describe("executeHeartbeat", () => {
|
|||||||
prompt: "# Test PROMPT.md\nSome content",
|
prompt: "# Test PROMPT.md\nSome content",
|
||||||
steps: [],
|
steps: [],
|
||||||
column: "todo",
|
column: "todo",
|
||||||
|
worktree: "/tmp/worktree-fn-001",
|
||||||
|
branch: "fusion/fn-001",
|
||||||
dependencies: [],
|
dependencies: [],
|
||||||
log: [],
|
log: [],
|
||||||
attachments: [],
|
attachments: [],
|
||||||
@@ -164,10 +172,17 @@ describe("executeHeartbeat", () => {
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
mockTaskStore = createMockTaskStore();
|
mockTaskStore = createMockTaskStore();
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
mockedAcquireTaskWorktree.mockResolvedValue({
|
||||||
|
worktreePath: "/tmp/worktree-fn-001",
|
||||||
|
branch: "fusion/fn-001",
|
||||||
|
source: "existing",
|
||||||
|
hydrated: false,
|
||||||
|
isResume: true,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
vi.restoreAllMocks();
|
vi.clearAllMocks();
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("reports health check", () => {
|
describe("reports health check", () => {
|
||||||
|
|||||||
@@ -204,10 +204,14 @@ vi.mock("node:child_process", () => {
|
|||||||
|
|
||||||
return { execSync: execSyncFn, exec: execFn, execFile: execFileFn, spawn: spawnFn };
|
return { execSync: execSyncFn, exec: execFn, execFile: execFileFn, spawn: spawnFn };
|
||||||
});
|
});
|
||||||
vi.mock("node:fs", () => ({
|
vi.mock("node:fs", async (importOriginal) => {
|
||||||
existsSync: vi.fn().mockReturnValue(true),
|
const actual = await importOriginal<typeof import("node:fs")>();
|
||||||
readdirSync: vi.fn().mockReturnValue([]),
|
return {
|
||||||
}));
|
...actual,
|
||||||
|
existsSync: vi.fn().mockReturnValue(true),
|
||||||
|
readdirSync: vi.fn().mockReturnValue([]),
|
||||||
|
};
|
||||||
|
});
|
||||||
vi.mock("node:fs/promises", () => ({
|
vi.mock("node:fs/promises", () => ({
|
||||||
readFile: vi.fn().mockResolvedValue("# Task prompt content"),
|
readFile: vi.fn().mockResolvedValue("# Task prompt content"),
|
||||||
}));
|
}));
|
||||||
@@ -592,7 +596,7 @@ describe("In-progress task resume after restart", () => {
|
|||||||
const executor = new TaskExecutor(store, "/tmp/test");
|
const executor = new TaskExecutor(store, "/tmp/test");
|
||||||
await executor.resumeOrphaned();
|
await executor.resumeOrphaned();
|
||||||
await waitForAsyncExpectation(() => {
|
await waitForAsyncExpectation(() => {
|
||||||
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(1);
|
expect(mockedCreateFnAgent).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
// Must NOT mark the step done — the reset invalidated the prior approval.
|
// Must NOT mark the step done — the reset invalidated the prior approval.
|
||||||
@@ -621,7 +625,7 @@ describe("In-progress task resume after restart", () => {
|
|||||||
const executor = new TaskExecutor(store, "/tmp/test");
|
const executor = new TaskExecutor(store, "/tmp/test");
|
||||||
await executor.resumeOrphaned();
|
await executor.resumeOrphaned();
|
||||||
await waitForAsyncExpectation(() => {
|
await waitForAsyncExpectation(() => {
|
||||||
expect(mockedCreateFnAgent).toHaveBeenCalledTimes(1);
|
expect(mockedCreateFnAgent).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
const updateStepDoneCalls = store.updateStep.mock.calls.filter(
|
const updateStepDoneCalls = store.updateStep.mock.calls.filter(
|
||||||
|
|||||||
88
packages/engine/src/__tests__/worktree-acquisition.test.ts
Normal file
88
packages/engine/src/__tests__/worktree-acquisition.test.ts
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||||
|
import { acquireTaskWorktree } from "../worktree-acquisition.js";
|
||||||
|
|
||||||
|
vi.mock("../worktree-pool.js", async () => {
|
||||||
|
const actual = await vi.importActual<any>("../worktree-pool.js");
|
||||||
|
return { ...actual, isUsableTaskWorktree: vi.fn().mockResolvedValue(true) };
|
||||||
|
});
|
||||||
|
|
||||||
|
vi.mock("../worktree-db-hydrate.js", () => ({
|
||||||
|
hydrateWorktreeDb: vi.fn().mockResolvedValue({ degraded: false, tasksCopied: 1, documentsCopied: 1 }),
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe("acquireTaskWorktree", () => {
|
||||||
|
const task = {
|
||||||
|
id: "FN-1",
|
||||||
|
title: "Task",
|
||||||
|
description: "Desc",
|
||||||
|
branch: null,
|
||||||
|
worktree: null,
|
||||||
|
} as any;
|
||||||
|
|
||||||
|
let store: any;
|
||||||
|
beforeEach(() => {
|
||||||
|
store = {
|
||||||
|
updateTask: vi.fn().mockResolvedValue(undefined),
|
||||||
|
logEntry: vi.fn().mockResolvedValue(undefined),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reuses existing usable worktree", async () => {
|
||||||
|
const result = await acquireTaskWorktree({
|
||||||
|
task: { ...task, worktree: process.cwd(), branch: "fusion/fn-1" },
|
||||||
|
rootDir: process.cwd(),
|
||||||
|
store,
|
||||||
|
settings: {},
|
||||||
|
createWorktree: vi.fn(),
|
||||||
|
});
|
||||||
|
expect(result.source).toBe("existing");
|
||||||
|
expect(result.worktreePath).toBe(process.cwd());
|
||||||
|
});
|
||||||
|
|
||||||
|
it("acquires from pool when enabled", async () => {
|
||||||
|
const prepareForTask = vi.fn().mockResolvedValue("fusion/fn-1");
|
||||||
|
const result = await acquireTaskWorktree({
|
||||||
|
task,
|
||||||
|
rootDir: process.cwd(),
|
||||||
|
store,
|
||||||
|
settings: { recycleWorktrees: true } as any,
|
||||||
|
pool: {
|
||||||
|
acquire: () => "/tmp/pooled",
|
||||||
|
prepareForTask,
|
||||||
|
release: vi.fn(),
|
||||||
|
} as any,
|
||||||
|
createWorktree: vi.fn(),
|
||||||
|
});
|
||||||
|
expect(result.source).toBe("pool");
|
||||||
|
expect(prepareForTask).toHaveBeenCalled();
|
||||||
|
expect(store.updateTask).toHaveBeenCalledWith("FN-1", { worktree: "/tmp/pooled", branch: "fusion/fn-1" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("creates fresh when pool disabled", async () => {
|
||||||
|
const createWorktree = vi.fn().mockResolvedValue({ path: "/tmp/new", branch: "fusion/fn-1" });
|
||||||
|
const result = await acquireTaskWorktree({
|
||||||
|
task,
|
||||||
|
rootDir: process.cwd(),
|
||||||
|
store,
|
||||||
|
settings: {},
|
||||||
|
createWorktree,
|
||||||
|
});
|
||||||
|
expect(result.source).toBe("fresh");
|
||||||
|
expect(createWorktree).toHaveBeenCalled();
|
||||||
|
expect(store.updateTask).toHaveBeenCalledWith("FN-1", { worktree: "/tmp/new", branch: "fusion/fn-1" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips init command when runInitCommand false", async () => {
|
||||||
|
const runConfiguredCommand = vi.fn();
|
||||||
|
await acquireTaskWorktree({
|
||||||
|
task,
|
||||||
|
rootDir: process.cwd(),
|
||||||
|
store,
|
||||||
|
settings: { worktreeInitCommand: "pnpm i" } as any,
|
||||||
|
createWorktree: vi.fn().mockResolvedValue({ path: "/tmp/new", branch: "fusion/fn-1" }),
|
||||||
|
runConfiguredCommand,
|
||||||
|
runInitCommand: false,
|
||||||
|
});
|
||||||
|
expect(runConfiguredCommand).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -31,6 +31,7 @@ import {
|
|||||||
} from "./agent-instructions.js";
|
} from "./agent-instructions.js";
|
||||||
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
|
import { buildPromptLayers, collapsePromptLayers } from "./prompt-layers.js";
|
||||||
import { heartbeatLog, formatError } from "./logger.js";
|
import { heartbeatLog, formatError } from "./logger.js";
|
||||||
|
import { acquireTaskWorktree } from "./worktree-acquisition.js";
|
||||||
import { createRunAuditor, type EngineRunContext } from "./run-audit.js";
|
import { createRunAuditor, type EngineRunContext } from "./run-audit.js";
|
||||||
import { promptWithFallback } from "./pi.js";
|
import { promptWithFallback } from "./pi.js";
|
||||||
import { createResolvedAgentSession, extractRuntimeHint, resolveHeartbeatSessionModels } from "./agent-session-helpers.js";
|
import { createResolvedAgentSession, extractRuntimeHint, resolveHeartbeatSessionModels } from "./agent-session-helpers.js";
|
||||||
@@ -2103,6 +2104,36 @@ export class HeartbeatMonitor {
|
|||||||
heartbeatLog.warn(`Failed to read heartbeat model settings for ${agentId}: ${settingsErr instanceof Error ? settingsErr.message : String(settingsErr)}`);
|
heartbeatLog.warn(`Failed to read heartbeat model settings for ${agentId}: ${settingsErr instanceof Error ? settingsErr.message : String(settingsErr)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let sessionCwd = rootDir;
|
||||||
|
if (!isNoTaskRun && taskDetail && heartbeatModelSettings) {
|
||||||
|
try {
|
||||||
|
const acquisition = await acquireTaskWorktree({
|
||||||
|
task: taskDetail,
|
||||||
|
rootDir,
|
||||||
|
store: taskStore,
|
||||||
|
settings: heartbeatModelSettings,
|
||||||
|
logger: heartbeatLog,
|
||||||
|
audit,
|
||||||
|
runContext,
|
||||||
|
runInitCommand: false,
|
||||||
|
});
|
||||||
|
sessionCwd = acquisition.worktreePath;
|
||||||
|
} catch (worktreeErr) {
|
||||||
|
const detail = worktreeErr instanceof Error ? worktreeErr.message : String(worktreeErr);
|
||||||
|
heartbeatLog.warn(`Heartbeat worktree acquisition failed for ${agentId}: ${detail}`);
|
||||||
|
if (taskDetail.column !== "done" && taskDetail.column !== "archived") {
|
||||||
|
await taskStore.moveTask(taskDetail.id, "todo", { preserveProgress: true });
|
||||||
|
}
|
||||||
|
await this.completeRun(agentId, run.id, {
|
||||||
|
status: "completed",
|
||||||
|
resultJson: { reason: "worktree_acquisition_failed", detail },
|
||||||
|
stderrExcerpt: detail,
|
||||||
|
skipStateTransition: true,
|
||||||
|
});
|
||||||
|
return (await this.store.getRunDetail(agentId, run.id))!;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const heartbeatSessionModels = resolveHeartbeatSessionModels(heartbeatModelSettings, agent.runtimeConfig);
|
const heartbeatSessionModels = resolveHeartbeatSessionModels(heartbeatModelSettings, agent.runtimeConfig);
|
||||||
|
|
||||||
// Create agent session
|
// Create agent session
|
||||||
@@ -2110,7 +2141,7 @@ export class HeartbeatMonitor {
|
|||||||
sessionPurpose: "heartbeat",
|
sessionPurpose: "heartbeat",
|
||||||
runtimeHint: extractRuntimeHint(agent.runtimeConfig),
|
runtimeHint: extractRuntimeHint(agent.runtimeConfig),
|
||||||
pluginRunner: this.pluginRunner,
|
pluginRunner: this.pluginRunner,
|
||||||
cwd: rootDir,
|
cwd: sessionCwd,
|
||||||
systemPrompt: systemPromptFinal,
|
systemPrompt: systemPromptFinal,
|
||||||
systemPromptLayers: heartbeatLayers,
|
systemPromptLayers: heartbeatLayers,
|
||||||
tools: "coding",
|
tools: "coding",
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import {
|
|||||||
VERIFICATION_LOG_MAX_CHARS,
|
VERIFICATION_LOG_MAX_CHARS,
|
||||||
type VerificationResult,
|
type VerificationResult,
|
||||||
} from "./verification-utils.js";
|
} from "./verification-utils.js";
|
||||||
import { generateWorktreeName, slugify } from "./worktree-names.js";
|
import { generateWorktreeName } from "./worktree-names.js";
|
||||||
import { Type, type Static } from "@mariozechner/pi-ai";
|
import { Type, type Static } from "@mariozechner/pi-ai";
|
||||||
import { describeModel, promptWithFallback, compactSessionContext } from "./pi.js";
|
import { describeModel, promptWithFallback, compactSessionContext } from "./pi.js";
|
||||||
import { accumulateSessionTokenUsage } from "./session-token-usage.js";
|
import { accumulateSessionTokenUsage } from "./session-token-usage.js";
|
||||||
@@ -36,7 +36,7 @@ import { buildSessionSkillContext } from "./session-skill-context.js";
|
|||||||
import { reviewStep, type ReviewVerdict } from "./reviewer.js";
|
import { reviewStep, type ReviewVerdict } from "./reviewer.js";
|
||||||
import { ModelRegistry, SessionManager, type ToolDefinition, type AgentSession } from "@mariozechner/pi-coding-agent";
|
import { ModelRegistry, SessionManager, type ToolDefinition, type AgentSession } from "@mariozechner/pi-coding-agent";
|
||||||
import { PRIORITY_EXECUTE, type AgentSemaphore } from "./concurrency.js";
|
import { PRIORITY_EXECUTE, type AgentSemaphore } from "./concurrency.js";
|
||||||
import { getRegisteredWorktreePaths, isGitRepository, isRegisteredGitWorktree, isUsableTaskWorktree, type WorktreePool } from "./worktree-pool.js";
|
import { getRegisteredWorktreePaths, isGitRepository, isRegisteredGitWorktree, type WorktreePool } from "./worktree-pool.js";
|
||||||
import { BranchConflictError, isBranchConflictError, inspectBranchConflict } from "./branch-conflicts.js";
|
import { BranchConflictError, isBranchConflictError, inspectBranchConflict } from "./branch-conflicts.js";
|
||||||
import { AgentLogger } from "./agent-logger.js";
|
import { AgentLogger } from "./agent-logger.js";
|
||||||
import { executorLog, reviewerLog, formatError } from "./logger.js";
|
import { executorLog, reviewerLog, formatError } from "./logger.js";
|
||||||
@@ -49,7 +49,7 @@ import type { StuckTaskDetector, StuckTaskEvent } from "./stuck-task-detector.js
|
|||||||
import type { PluginRunner } from "./plugin-runner.js";
|
import type { PluginRunner } from "./plugin-runner.js";
|
||||||
import { isContextLimitError } from "./context-limit-detector.js";
|
import { isContextLimitError } from "./context-limit-detector.js";
|
||||||
import { StepSessionExecutor } from "./step-session-executor.js";
|
import { StepSessionExecutor } from "./step-session-executor.js";
|
||||||
import { hydrateWorktreeDb } from "./worktree-db-hydrate.js";
|
import { acquireTaskWorktree } from "./worktree-acquisition.js";
|
||||||
import {
|
import {
|
||||||
resolveAgentInstructions,
|
resolveAgentInstructions,
|
||||||
buildSystemPromptWithInstructions,
|
buildSystemPromptWithInstructions,
|
||||||
@@ -2416,28 +2416,7 @@ export class TaskExecutor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Hoist worktreePath so it's accessible in the catch block for dep-abort cleanup
|
// Hoist worktreePath so it's accessible in the catch block for dep-abort cleanup
|
||||||
// Determine worktree name based on settings
|
let worktreePath = task.worktree ?? "";
|
||||||
let worktreePath: string;
|
|
||||||
if (task.worktree) {
|
|
||||||
worktreePath = task.worktree;
|
|
||||||
} else {
|
|
||||||
const naming = settings.worktreeNaming || "random";
|
|
||||||
let worktreeName: string;
|
|
||||||
|
|
||||||
switch (naming) {
|
|
||||||
case "task-id":
|
|
||||||
worktreeName = task.id.toLowerCase();
|
|
||||||
break;
|
|
||||||
case "task-title":
|
|
||||||
worktreeName = slugify(task.title || task.description.slice(0, 60));
|
|
||||||
break;
|
|
||||||
case "random":
|
|
||||||
default:
|
|
||||||
worktreeName = generateWorktreeName(this.rootDir);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
worktreePath = join(this.rootDir, ".worktrees", worktreeName);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set by stuck-abort handlers; the actual moveTask("todo") is deferred to
|
// Set by stuck-abort handlers; the actual moveTask("todo") is deferred to
|
||||||
// the finally block so this.executing is cleared first (prevents re-dispatch race).
|
// the finally block so this.executing is cleared first (prevents re-dispatch race).
|
||||||
@@ -2472,255 +2451,48 @@ export class TaskExecutor {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create or reuse worktree — try pool first when recycling is enabled
|
const acquisition = await acquireTaskWorktree({
|
||||||
// Prefer the persisted branch from a prior run so the agent resumes on the
|
task,
|
||||||
// same branch instead of creating a fresh fusion/fn-XXXX-2, -3, etc.
|
rootDir: this.rootDir,
|
||||||
const branchName = task.branch || `fusion/${task.id.toLowerCase()}`;
|
store: this.store,
|
||||||
// Use generateWorktreeName for human-friendly directory names (adjective-noun pattern)
|
settings,
|
||||||
// instead of task.id, so worktrees are named like ".worktrees/swift-falcon"
|
pool: this.options.pool,
|
||||||
let isResume = existsSync(worktreePath);
|
logger: executorLog,
|
||||||
let acquiredFromPool = false;
|
audit,
|
||||||
|
runContext: this.currentRunContext,
|
||||||
|
runInitCommand: true,
|
||||||
|
createWorktree: this.createWorktree.bind(this),
|
||||||
|
runConfiguredCommand,
|
||||||
|
taskEnv,
|
||||||
|
});
|
||||||
|
worktreePath = acquisition.worktreePath;
|
||||||
|
|
||||||
// Resolve the base branch — set by the scheduler when a dep is in-review
|
if (!acquisition.isResume && acquisition.source === "fresh" && settings.setupScript) {
|
||||||
const baseBranch = task.executionStartBranch || null;
|
const scriptCommand = settings.scripts?.[settings.setupScript];
|
||||||
const allowSiblingBranchRename = settings.executorAllowSiblingBranchRename === true;
|
if (scriptCommand) {
|
||||||
|
const setupStartedAt = Date.now();
|
||||||
if (task.worktree && isResume && !await isUsableTaskWorktree(this.rootDir, worktreePath)) {
|
|
||||||
const invalidWorktreePath = worktreePath;
|
|
||||||
executorLog.log(`${task.id}: assigned worktree is not usable; creating a fresh worktree instead: ${invalidWorktreePath}`);
|
|
||||||
await this.store.logEntry(
|
|
||||||
task.id,
|
|
||||||
`Assigned worktree is not a registered, usable git worktree; creating a fresh worktree instead`,
|
|
||||||
invalidWorktreePath,
|
|
||||||
this.currentRunContext,
|
|
||||||
);
|
|
||||||
await this.store.updateTask(task.id, { worktree: null, branch: null });
|
|
||||||
worktreePath = join(this.rootDir, ".worktrees", generateWorktreeName(this.rootDir));
|
|
||||||
isResume = existsSync(worktreePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isResume) {
|
|
||||||
|
|
||||||
// Try acquiring a warm worktree from the pool
|
|
||||||
if (this.options.pool && settings.recycleWorktrees) {
|
|
||||||
const pooled = this.options.pool.acquire();
|
|
||||||
if (pooled) {
|
|
||||||
try {
|
|
||||||
const actualBranch = await this.options.pool.prepareForTask(
|
|
||||||
pooled,
|
|
||||||
branchName,
|
|
||||||
baseBranch ?? undefined,
|
|
||||||
{ allowSiblingBranchRename, repoDir: this.rootDir },
|
|
||||||
);
|
|
||||||
worktreePath = pooled;
|
|
||||||
acquiredFromPool = true;
|
|
||||||
executorLog.log(`Acquired worktree from pool: ${pooled}`);
|
|
||||||
await this.store.updateTask(task.id, { worktree: worktreePath, branch: actualBranch });
|
|
||||||
// Audit trail: record worktree reuse (FN-1404)
|
|
||||||
await audit.git({ type: "worktree:reuse", target: worktreePath, metadata: { branch: actualBranch } });
|
|
||||||
if (actualBranch !== branchName) {
|
|
||||||
executorLog.log(`Branch conflict resolved: using ${actualBranch} instead of ${branchName}`);
|
|
||||||
await this.store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath} (branch conflict: using ${actualBranch})`, undefined, this.currentRunContext);
|
|
||||||
} else {
|
|
||||||
await this.store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath}`, undefined, this.currentRunContext);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.rootDir !== worktreePath) {
|
|
||||||
try {
|
|
||||||
const hydration = await hydrateWorktreeDb({
|
|
||||||
rootDir: this.rootDir,
|
|
||||||
worktreePath,
|
|
||||||
taskId: task.id,
|
|
||||||
store: this.store,
|
|
||||||
logger: executorLog,
|
|
||||||
});
|
|
||||||
if (hydration.degraded) {
|
|
||||||
await this.store.logEntry(
|
|
||||||
task.id,
|
|
||||||
`Worktree DB hydration degraded: ${hydration.reason ?? "unknown"}`,
|
|
||||||
undefined,
|
|
||||||
this.currentRunContext,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
await this.store.logEntry(
|
|
||||||
task.id,
|
|
||||||
`Hydrated worktree DB: ${hydration.tasksCopied} tasks, ${hydration.documentsCopied} task_documents`,
|
|
||||||
undefined,
|
|
||||||
this.currentRunContext,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
executorLog.warn(`${task.id}: worktree DB hydration failed: ${formatError(error)}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (poolErr: unknown) {
|
|
||||||
this.options.pool.release(pooled);
|
|
||||||
if (isBranchConflictError(poolErr)) {
|
|
||||||
throw poolErr;
|
|
||||||
}
|
|
||||||
const poolErrMessage = poolErr instanceof Error ? poolErr.message : String(poolErr);
|
|
||||||
executorLog.log(`Pool prepareForTask failed, falling through to fresh worktree: ${poolErrMessage}`);
|
|
||||||
await this.store.logEntry(
|
|
||||||
task.id,
|
|
||||||
`Pool worktree preparation failed (${poolErrMessage}), creating fresh worktree`,
|
|
||||||
undefined,
|
|
||||||
this.currentRunContext,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fall through to fresh worktree creation if pool had nothing
|
|
||||||
if (!acquiredFromPool) {
|
|
||||||
const created = await this.createWorktree(branchName, worktreePath, task.id, baseBranch ?? undefined, allowSiblingBranchRename);
|
|
||||||
worktreePath = created.path;
|
|
||||||
await this.store.updateTask(task.id, { worktree: created.path, branch: created.branch });
|
|
||||||
// Audit trail: record worktree creation and branch creation (FN-1404)
|
|
||||||
await audit.git({ type: "worktree:create", target: created.path, metadata: { branch: created.branch } });
|
|
||||||
await audit.git({ type: "branch:create", target: created.branch });
|
|
||||||
if (created.branch !== branchName) {
|
|
||||||
executorLog.log(`Branch conflict resolved: using ${created.branch} instead of ${branchName}`);
|
|
||||||
await this.store.logEntry(task.id, `Worktree created at ${worktreePath} (branch conflict: using ${created.branch})`, undefined, this.currentRunContext);
|
|
||||||
} else if (baseBranch) {
|
|
||||||
await this.store.logEntry(task.id, `Worktree created at ${worktreePath} (based on ${baseBranch})`, undefined, this.currentRunContext);
|
|
||||||
} else {
|
|
||||||
await this.store.logEntry(task.id, `Worktree created at ${worktreePath}`, undefined, this.currentRunContext);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run worktree init command for fresh worktrees (skip for pooled — caches are warm).
|
|
||||||
// The init command should deterministically install the full dependency
|
|
||||||
// graph required by test/typecheck/build commands (for pnpm workspaces,
|
|
||||||
// prefer `pnpm install --frozen-lockfile`) so transitive modules and
|
|
||||||
// declarations like @vitest/runner, loupe, debug, @types/express, and
|
|
||||||
// node-pty are present after bootstrap.
|
|
||||||
//
|
|
||||||
// NOTE: This is distinct from the separate workspace-export failure
|
|
||||||
// class where internal packages fail to resolve because exports point
|
|
||||||
// to missing dist/* outputs (e.g. "@fusion/core entry not found").
|
|
||||||
// 5-minute timeout accommodates larger dependency installs.
|
|
||||||
if (settings.worktreeInitCommand) {
|
|
||||||
const initStartedAt = Date.now();
|
|
||||||
try {
|
|
||||||
const initResult = await runConfiguredCommand(settings.worktreeInitCommand, worktreePath, 300_000, taskEnv);
|
|
||||||
if (initResult.spawnError || initResult.timedOut || initResult.exitCode !== 0) {
|
|
||||||
throw new Error(configuredCommandErrorMessage(initResult));
|
|
||||||
}
|
|
||||||
await this.store.logEntry(task.id, `[timing] Worktree init command completed in ${Date.now() - initStartedAt}ms`, settings.worktreeInitCommand, this.currentRunContext);
|
|
||||||
} catch (err: unknown) {
|
|
||||||
await this.store.logEntry(task.id, `[timing] Worktree init command failed after ${Date.now() - initStartedAt}ms`, undefined, this.currentRunContext);
|
|
||||||
const execError = err instanceof Error ? err : new Error(String(err));
|
|
||||||
const message = "stderr" in execError && typeof (execError as Record<string, unknown>).stderr === "string"
|
|
||||||
? String((execError as Record<string, unknown>).stderr)
|
|
||||||
: execError.message;
|
|
||||||
executorLog.error(`${task.id}: worktree init command failed — first test run will likely fail: ${message}`);
|
|
||||||
await this.store.logEntry(
|
|
||||||
task.id,
|
|
||||||
`Worktree init command failed (first test run will likely fail): ${message}`,
|
|
||||||
undefined,
|
|
||||||
this.currentRunContext,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run setup script for fresh worktrees (after worktreeInitCommand)
|
|
||||||
if (settings.setupScript) {
|
|
||||||
const scriptCommand = settings.scripts?.[settings.setupScript];
|
|
||||||
if (scriptCommand) {
|
|
||||||
const setupStartedAt = Date.now();
|
|
||||||
try {
|
|
||||||
const setupResult = await runConfiguredCommand(scriptCommand, worktreePath, 120_000, taskEnv);
|
|
||||||
if (setupResult.spawnError || setupResult.timedOut || setupResult.exitCode !== 0) {
|
|
||||||
throw new Error(configuredCommandErrorMessage(setupResult));
|
|
||||||
}
|
|
||||||
await this.store.logEntry(task.id, `[timing] Setup script '${settings.setupScript}' completed in ${Date.now() - setupStartedAt}ms`, scriptCommand, this.currentRunContext);
|
|
||||||
} catch (err: unknown) {
|
|
||||||
const execError = err instanceof Error ? err : new Error(String(err));
|
|
||||||
const message = "stderr" in execError && typeof (execError as Record<string, unknown>).stderr === "string"
|
|
||||||
? String((execError as Record<string, unknown>).stderr)
|
|
||||||
: execError.message;
|
|
||||||
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' failed: ${message}`, undefined, this.currentRunContext);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' not found in scripts map — skipping`, undefined, this.currentRunContext);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!acquiredFromPool) {
|
|
||||||
if (this.rootDir !== worktreePath) {
|
|
||||||
try {
|
|
||||||
const hydration = await hydrateWorktreeDb({
|
|
||||||
rootDir: this.rootDir,
|
|
||||||
worktreePath,
|
|
||||||
taskId: task.id,
|
|
||||||
store: this.store,
|
|
||||||
logger: executorLog,
|
|
||||||
});
|
|
||||||
if (hydration.degraded) {
|
|
||||||
await this.store.logEntry(
|
|
||||||
task.id,
|
|
||||||
`Worktree DB hydration degraded: ${hydration.reason ?? "unknown"}`,
|
|
||||||
undefined,
|
|
||||||
this.currentRunContext,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
await this.store.logEntry(
|
|
||||||
task.id,
|
|
||||||
`Hydrated worktree DB: ${hydration.tasksCopied} tasks, ${hydration.documentsCopied} task_documents`,
|
|
||||||
undefined,
|
|
||||||
this.currentRunContext,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
executorLog.warn(`${task.id}: worktree DB hydration failed: ${formatError(error)}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (task.worktree) {
|
|
||||||
// Task already had a worktree assigned and it exists on disk — reuse it
|
|
||||||
executorLog.log(`Reusing existing worktree: ${worktreePath}`);
|
|
||||||
if (this.rootDir !== worktreePath) {
|
|
||||||
try {
|
try {
|
||||||
const hydration = await hydrateWorktreeDb({
|
const setupResult = await runConfiguredCommand(scriptCommand, worktreePath, 120_000, taskEnv);
|
||||||
rootDir: this.rootDir,
|
if (setupResult.spawnError || setupResult.timedOut || setupResult.exitCode !== 0) {
|
||||||
worktreePath,
|
throw new Error(configuredCommandErrorMessage(setupResult));
|
||||||
taskId: task.id,
|
}
|
||||||
store: this.store,
|
await this.store.logEntry(task.id, `[timing] Setup script '${settings.setupScript}' completed in ${Date.now() - setupStartedAt}ms`, scriptCommand, this.currentRunContext);
|
||||||
logger: executorLog,
|
} catch (err: unknown) {
|
||||||
});
|
const execError = err instanceof Error ? err : new Error(String(err));
|
||||||
if (hydration.degraded) {
|
const message = "stderr" in execError && typeof (execError as Record<string, unknown>).stderr === "string"
|
||||||
await this.store.logEntry(
|
? String((execError as Record<string, unknown>).stderr)
|
||||||
task.id,
|
: execError.message;
|
||||||
`Worktree DB hydration degraded: ${hydration.reason ?? "unknown"}`,
|
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' failed: ${message}`, undefined, this.currentRunContext);
|
||||||
undefined,
|
|
||||||
this.currentRunContext,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
await this.store.logEntry(
|
|
||||||
task.id,
|
|
||||||
`Hydrated worktree DB: ${hydration.tasksCopied} tasks, ${hydration.documentsCopied} task_documents`,
|
|
||||||
undefined,
|
|
||||||
this.currentRunContext,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
executorLog.warn(`${task.id}: worktree DB hydration failed: ${formatError(error)}`);
|
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
await this.store.logEntry(task.id, `Setup script '${settings.setupScript}' not found in scripts map — skipping`, undefined, this.currentRunContext);
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
// Directory exists at generated path but task has no worktree — create via normal flow
|
|
||||||
const created = await this.createWorktree(branchName, worktreePath, task.id, undefined, allowSiblingBranchRename);
|
|
||||||
worktreePath = created.path;
|
|
||||||
await this.store.updateTask(task.id, { worktree: created.path, branch: created.branch });
|
|
||||||
// Audit trail: record worktree creation and branch creation (FN-1404)
|
|
||||||
await audit.git({ type: "worktree:create", target: created.path, metadata: { branch: created.branch } });
|
|
||||||
await audit.git({ type: "branch:create", target: created.branch });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Capture the base commit SHA for diff computation whenever a task
|
// Capture the base commit SHA for diff computation whenever a task
|
||||||
// starts with a newly assigned worktree. Recycled worktrees must
|
// starts with a newly assigned worktree. Recycled worktrees must
|
||||||
// overwrite any prior task baseline instead of inheriting it.
|
// overwrite any prior task baseline instead of inheriting it.
|
||||||
if (!isResume) {
|
if (!acquisition.isResume) {
|
||||||
try {
|
try {
|
||||||
const { stdout } = await execAsync("git rev-parse HEAD", {
|
const { stdout } = await execAsync("git rev-parse HEAD", {
|
||||||
cwd: worktreePath,
|
cwd: worktreePath,
|
||||||
@@ -2776,7 +2548,7 @@ export class TaskExecutor {
|
|||||||
|
|
||||||
// On resume (task.branch already set from a prior run), reconcile step
|
// On resume (task.branch already set from a prior run), reconcile step
|
||||||
// statuses from git history so the agent doesn't redo already-committed work.
|
// statuses from git history so the agent doesn't redo already-committed work.
|
||||||
if (isResume && task.branch && detail.steps.length > 0) {
|
if (acquisition.isResume && task.branch && detail.steps.length > 0) {
|
||||||
await this.reconcileStepsFromGitHistory(task.id, detail, worktreePath);
|
await this.reconcileStepsFromGitHistory(task.id, detail, worktreePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
207
packages/engine/src/worktree-acquisition.ts
Normal file
207
packages/engine/src/worktree-acquisition.ts
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import { exec } from "node:child_process";
|
||||||
|
import { promisify } from "node:util";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import type { RunMutationContext, Settings, Task, TaskStore } from "@fusion/core";
|
||||||
|
import { generateWorktreeName, slugify } from "./worktree-names.js";
|
||||||
|
import { hydrateWorktreeDb } from "./worktree-db-hydrate.js";
|
||||||
|
import { formatError } from "./logger.js";
|
||||||
|
import { isBranchConflictError } from "./branch-conflicts.js";
|
||||||
|
import { type WorktreePool, isUsableTaskWorktree } from "./worktree-pool.js";
|
||||||
|
import type { RunAuditor } from "./run-audit.js";
|
||||||
|
|
||||||
|
const execAsync = promisify(exec);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Worktree acquisition contract:
|
||||||
|
* - `runInitCommand=true` runs the init command only for newly-created worktrees (fresh, not pool/existing).
|
||||||
|
* - Heartbeat task runs should pass `runInitCommand=false`.
|
||||||
|
* - Executor may pass `runInitCommand=true`; if heartbeat created the worktree earlier, executor reuses it and init is skipped.
|
||||||
|
*/
|
||||||
|
export interface AcquireTaskWorktreeOptions {
|
||||||
|
task: Task;
|
||||||
|
rootDir: string;
|
||||||
|
store: TaskStore;
|
||||||
|
settings: Partial<Settings>;
|
||||||
|
pool?: WorktreePool;
|
||||||
|
logger?: { log: (m: string) => void; warn: (m: string) => void; error?: (m: string) => void };
|
||||||
|
audit?: Pick<RunAuditor, "git">;
|
||||||
|
runContext?: RunMutationContext;
|
||||||
|
runInitCommand?: boolean;
|
||||||
|
createWorktree?: (
|
||||||
|
branch: string,
|
||||||
|
path: string,
|
||||||
|
taskId: string,
|
||||||
|
startPoint?: string,
|
||||||
|
allowSiblingBranchRename?: boolean,
|
||||||
|
) => Promise<{ path: string; branch: string }>;
|
||||||
|
runConfiguredCommand?: (command: string, cwd: string, timeoutMs: number, env?: NodeJS.ProcessEnv) => Promise<{ spawnError?: string | Error; timedOut?: boolean; exitCode?: number | null }>;
|
||||||
|
taskEnv?: NodeJS.ProcessEnv;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AcquireTaskWorktreeResult {
|
||||||
|
worktreePath: string;
|
||||||
|
branch: string;
|
||||||
|
source: "existing" | "pool" | "fresh";
|
||||||
|
hydrated: boolean;
|
||||||
|
isResume: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function configuredCommandErrorMessage(result: { spawnError?: string | Error; timedOut?: boolean; exitCode?: number | null }): string {
|
||||||
|
if (result.spawnError) return `Failed to start command: ${result.spawnError}`;
|
||||||
|
if (result.timedOut) return "Command timed out";
|
||||||
|
return `Command exited with code ${result.exitCode ?? "unknown"}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createWorktreeFallback(
|
||||||
|
rootDir: string,
|
||||||
|
branch: string,
|
||||||
|
path: string,
|
||||||
|
startPoint?: string,
|
||||||
|
allowSiblingBranchRename = false,
|
||||||
|
): Promise<{ path: string; branch: string }> {
|
||||||
|
const escapedPath = JSON.stringify(path);
|
||||||
|
const escapedBranch = JSON.stringify(branch);
|
||||||
|
const escapedStart = startPoint ? JSON.stringify(startPoint) : undefined;
|
||||||
|
const startArg = escapedStart ? ` ${escapedStart}` : "";
|
||||||
|
|
||||||
|
try {
|
||||||
|
await execAsync(`git worktree add -b ${escapedBranch} ${escapedPath}${startArg}`, { cwd: rootDir });
|
||||||
|
return { path, branch };
|
||||||
|
} catch (error) {
|
||||||
|
if (!allowSiblingBranchRename) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let suffix = 2; suffix <= 50; suffix += 1) {
|
||||||
|
const candidate = `${branch}-${suffix}`;
|
||||||
|
const escapedCandidate = JSON.stringify(candidate);
|
||||||
|
try {
|
||||||
|
await execAsync(`git worktree add -b ${escapedCandidate} ${escapedPath}${startArg}`, { cwd: rootDir });
|
||||||
|
return { path, branch: candidate };
|
||||||
|
} catch {
|
||||||
|
// try next suffix
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function acquireTaskWorktree(opts: AcquireTaskWorktreeOptions): Promise<AcquireTaskWorktreeResult> {
|
||||||
|
const { task, rootDir, store, settings, pool, logger, audit, runContext, createWorktree, runConfiguredCommand, runInitCommand, taskEnv } = opts;
|
||||||
|
const branchName = task.branch || `fusion/${task.id.toLowerCase()}`;
|
||||||
|
const naming = settings.worktreeNaming || "random";
|
||||||
|
const allowSiblingBranchRename = settings.executorAllowSiblingBranchRename === true;
|
||||||
|
const baseBranch = task.executionStartBranch || null;
|
||||||
|
|
||||||
|
let worktreePath = task.worktree;
|
||||||
|
if (!worktreePath) {
|
||||||
|
const worktreeName = naming === "task-id"
|
||||||
|
? task.id.toLowerCase()
|
||||||
|
: naming === "task-title"
|
||||||
|
? slugify(task.title || task.description.slice(0, 60))
|
||||||
|
: generateWorktreeName(rootDir);
|
||||||
|
worktreePath = join(rootDir, ".worktrees", worktreeName);
|
||||||
|
}
|
||||||
|
|
||||||
|
let isResume = Boolean(task.worktree && existsSync(worktreePath));
|
||||||
|
if (task.worktree && isResume && !await isUsableTaskWorktree(rootDir, worktreePath)) {
|
||||||
|
logger?.log(`${task.id}: assigned worktree is not usable; creating a fresh worktree instead: ${worktreePath}`);
|
||||||
|
await store.logEntry(task.id, "Assigned worktree is not a registered, usable git worktree; creating a fresh worktree instead", worktreePath, runContext);
|
||||||
|
await store.updateTask(task.id, { worktree: null, branch: null });
|
||||||
|
worktreePath = join(rootDir, ".worktrees", generateWorktreeName(rootDir));
|
||||||
|
isResume = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const hydrate = async (path: string): Promise<boolean> => {
|
||||||
|
if (rootDir === path) return false;
|
||||||
|
try {
|
||||||
|
const hydration = await hydrateWorktreeDb({ rootDir, worktreePath: path, taskId: task.id, store, logger: logger ?? { warn: () => {} } });
|
||||||
|
if (hydration.degraded) {
|
||||||
|
await store.logEntry(task.id, `Worktree DB hydration degraded: ${hydration.reason ?? "unknown"}`, undefined, runContext);
|
||||||
|
} else {
|
||||||
|
await store.logEntry(task.id, `Hydrated worktree DB: ${hydration.tasksCopied} tasks, ${hydration.documentsCopied} task_documents`, undefined, runContext);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
logger?.warn(`${task.id}: worktree DB hydration failed: ${formatError(error)}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (task.worktree && isResume) {
|
||||||
|
logger?.log(`Reusing existing worktree: ${worktreePath}`);
|
||||||
|
const hydrated = await hydrate(worktreePath);
|
||||||
|
return { worktreePath, branch: task.branch ?? branchName, source: "existing", hydrated, isResume: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
let acquiredFromPool = false;
|
||||||
|
let branch = branchName;
|
||||||
|
|
||||||
|
if (!isResume && pool && settings.recycleWorktrees) {
|
||||||
|
const pooled = pool.acquire();
|
||||||
|
if (pooled) {
|
||||||
|
try {
|
||||||
|
const actualBranch = await pool.prepareForTask(pooled, branchName, baseBranch ?? undefined, { allowSiblingBranchRename, repoDir: rootDir });
|
||||||
|
worktreePath = pooled;
|
||||||
|
branch = actualBranch;
|
||||||
|
acquiredFromPool = true;
|
||||||
|
logger?.log(`Acquired worktree from pool: ${pooled}`);
|
||||||
|
await store.updateTask(task.id, { worktree: worktreePath, branch: actualBranch });
|
||||||
|
await audit?.git({ type: "worktree:reuse", target: worktreePath, metadata: { branch: actualBranch } });
|
||||||
|
if (actualBranch !== branchName) {
|
||||||
|
logger?.log(`Branch conflict resolved: using ${actualBranch} instead of ${branchName}`);
|
||||||
|
await store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath} (branch conflict: using ${actualBranch})`, undefined, runContext);
|
||||||
|
} else {
|
||||||
|
await store.logEntry(task.id, `Acquired worktree from pool: ${worktreePath}`, undefined, runContext);
|
||||||
|
}
|
||||||
|
const hydrated = await hydrate(worktreePath);
|
||||||
|
return { worktreePath, branch, source: "pool", hydrated, isResume: false };
|
||||||
|
} catch (poolErr) {
|
||||||
|
pool.release(pooled);
|
||||||
|
if (isBranchConflictError(poolErr)) throw poolErr;
|
||||||
|
const poolErrMessage = poolErr instanceof Error ? poolErr.message : String(poolErr);
|
||||||
|
logger?.log(`Pool prepareForTask failed, falling through to fresh worktree: ${poolErrMessage}`);
|
||||||
|
await store.logEntry(task.id, `Pool worktree preparation failed (${poolErrMessage}), creating fresh worktree`, undefined, runContext);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const createWorktreeImpl = createWorktree
|
||||||
|
? createWorktree
|
||||||
|
: (branch: string, path: string, _taskId: string, startPoint?: string, allowRename?: boolean) => createWorktreeFallback(rootDir, branch, path, startPoint, allowRename);
|
||||||
|
|
||||||
|
const created = await createWorktreeImpl(branchName, worktreePath, task.id, baseBranch ?? undefined, allowSiblingBranchRename);
|
||||||
|
worktreePath = created.path;
|
||||||
|
branch = created.branch;
|
||||||
|
await store.updateTask(task.id, { worktree: created.path, branch: created.branch });
|
||||||
|
await audit?.git({ type: "worktree:create", target: created.path, metadata: { branch: created.branch } });
|
||||||
|
await audit?.git({ type: "branch:create", target: created.branch });
|
||||||
|
if (created.branch !== branchName) {
|
||||||
|
logger?.log(`Branch conflict resolved: using ${created.branch} instead of ${branchName}`);
|
||||||
|
await store.logEntry(task.id, `Worktree created at ${worktreePath} (branch conflict: using ${created.branch})`, undefined, runContext);
|
||||||
|
} else if (baseBranch) {
|
||||||
|
await store.logEntry(task.id, `Worktree created at ${worktreePath} (based on ${baseBranch})`, undefined, runContext);
|
||||||
|
} else {
|
||||||
|
await store.logEntry(task.id, `Worktree created at ${worktreePath}`, undefined, runContext);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (runInitCommand && settings.worktreeInitCommand && runConfiguredCommand) {
|
||||||
|
const initStartedAt = Date.now();
|
||||||
|
try {
|
||||||
|
const initResult = await runConfiguredCommand(settings.worktreeInitCommand, worktreePath, 300_000, taskEnv);
|
||||||
|
if (initResult.spawnError || initResult.timedOut || initResult.exitCode !== 0) {
|
||||||
|
throw new Error(configuredCommandErrorMessage(initResult));
|
||||||
|
}
|
||||||
|
await store.logEntry(task.id, `[timing] Worktree init command completed in ${Date.now() - initStartedAt}ms`, settings.worktreeInitCommand, runContext);
|
||||||
|
} catch (err) {
|
||||||
|
await store.logEntry(task.id, `[timing] Worktree init command failed after ${Date.now() - initStartedAt}ms`, undefined, runContext);
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
logger?.error?.(`${task.id}: worktree init command failed — first test run will likely fail: ${message}`);
|
||||||
|
await store.logEntry(task.id, `Worktree init command failed (first test run will likely fail): ${message}`, undefined, runContext);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const hydrated = await hydrate(worktreePath);
|
||||||
|
return { worktreePath, branch, source: acquiredFromPool ? "pool" : "fresh", hydrated, isResume: false };
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user