fix(triage): prevent orphaned deps when splitting tasks + detect worktree drift

Root cause: during a triage split the AI could set a child task's
`dependencies` to the parent id. The parent is hard-deleted after the split,
and the scheduler's dep check treats a missing id as unmet — permanently
blocking the dependent. This stranded FN-2164 behind the deleted FN-2163.

- core/store.deleteTask: refuse to delete when any live task still has the id
  in its `dependencies` array. Throws TaskHasDependentsError listing dependents
  so callers can rewrite or recover. Covers the triage-split path and any
  future caller.
- engine/triage task_create: validate each proposed dependency before creating
  a child — reject the parent id, reject unknown task ids, allow siblings
  created earlier in the same split or pre-existing tasks.
- engine/triage split cleanup: wrap the parent deleteTask in try/catch that
  keeps the parent alive (safer than stranding dependents) and logs the reason.
- engine/triage prompts: both the mandatory-split and proactive-split prompts
  now explicitly state that subtask deps must never reference the parent.
- dashboard/routes /subtasks/create-tasks: reject parent-id deps, drop unknown
  deps with an audit log entry, surface parentTaskCloseError + droppedDependencies
  in the response instead of silently swallowing them.
- engine/executor: on execute entry, detect the drift state (in-progress task
  with no worktree) and emit a loud log + task log entry; the existing
  fresh-worktree path then recovers. Prevents silent "operating without a
  worktree" behavior that we saw on FN-2152.

Tests:
  core:      2907/2907 pass (+5 new, incl. deleteTask guard regression)
  engine:    2554/2554 pass (+17 new, incl. task_create dep validation)
  dashboard: 9064/9064 pass (+2 new for /subtasks/create-tasks).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Fusion
2026-04-19 20:45:36 -07:00
committed by gsxdsm
parent f530a69955
commit e8b0ed5627
12 changed files with 266 additions and 18 deletions

View File

@@ -51,7 +51,7 @@ describe("TaskStore task documents", () => {
expect(tableNames.has("task_documents")).toBe(true);
expect(tableNames.has("task_document_revisions")).toBe(true);
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
const index = db
.prepare(

View File

@@ -131,7 +131,7 @@ describe("Database", () => {
});
it("seeds schema version", () => {
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
});
it("seeds lastModified", () => {
@@ -154,7 +154,7 @@ describe("Database", () => {
it("is idempotent - calling init() twice does not fail", () => {
expect(() => db.init()).not.toThrow();
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
});
it("does not overwrite existing config on re-init", () => {
@@ -761,7 +761,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 29 (includes v1→v2 through v26→v29)
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -786,11 +786,11 @@ describe("schema migrations", () => {
const db = new Database(kbDir);
db.init();
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
// Re-init should not fail
db.init();
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
db.close();
});
@@ -806,7 +806,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>;
expect(tables).toEqual([{ name: "agentRatings" }]);
@@ -830,7 +830,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>;
expect(tables).toEqual([{ name: "mission_events" }]);
@@ -934,7 +934,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 29
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -1303,7 +1303,7 @@ describe("createDatabase factory", () => {
const db = createDatabase(kbDir);
db.init();
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
expect(db.getLastModified()).toBeGreaterThan(0);
db.close();

View File

@@ -146,6 +146,7 @@ CREATE TABLE IF NOT EXISTS tasks (
mergeRetries INTEGER,
workflowStepRetries INTEGER,
recoveryRetryCount INTEGER,
taskDoneRetryCount INTEGER DEFAULT 0,
nextRecoveryAt TEXT,
error TEXT,
summary TEXT,
@@ -1623,6 +1624,15 @@ export class Database {
});
}
if (version < 41) {
// Tracks self-healing auto-requeues of tasks that failed because the agent
// exited without calling task_done with partial step progress. Bounded so
// a persistently-broken task cannot loop forever.
this.applyMigration(41, () => {
this.addColumnIfMissing("tasks", "taskDoneRetryCount", "INTEGER DEFAULT 0");
});
}
}
/**

View File

@@ -776,7 +776,7 @@ describe("Migration: pre-33 DB upgrade", () => {
// Step 1: Create a fresh database at v33 (runs all migrations up to 33)
const db1 = createDatabase(legacyDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(40);
expect(db1.getSchemaVersion()).toBe(41);
db1.close();
// Step 2: Manually downgrade to version 32 and drop insight tables
@@ -811,7 +811,7 @@ describe("Migration: pre-33 DB upgrade", () => {
expect(tableNamesBefore).not.toContain("project_insight_runs");
// Now run init — this triggers the v32→v33 migration
db3.init();
expect(db3.getSchemaVersion()).toBe(40);
expect(db3.getSchemaVersion()).toBe(41);
// Step 4: Verify insight tables exist after migration
const tablesAfter = db3.prepare(
@@ -842,12 +842,12 @@ describe("Migration: pre-33 DB upgrade", () => {
try {
const db1 = createDatabase(testDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(40);
expect(db1.getSchemaVersion()).toBe(41);
db1.close();
const db2 = createDatabase(testDir);
expect(() => db2.init()).not.toThrow();
expect(db2.getSchemaVersion()).toBe(40);
expect(db2.getSchemaVersion()).toBe(41);
db2.close();
} finally {
rmSync(testDir, { recursive: true, force: true });

View File

@@ -2628,7 +2628,7 @@ describe("MissionStore", () => {
describe("Loop State & Validator Run Schema (v31)", () => {
it("schema version is 40 after migration", () => {
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
});
it("mission_features table has loop state columns", () => {

View File

@@ -739,7 +739,7 @@ describe("RoadmapStore", () => {
describe("schema version", () => {
it("schema version is 40 after init", () => {
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
});
});

View File

@@ -465,7 +465,7 @@ describe("Run Audit", () => {
});
it("schema version is bumped to 40", () => {
expect(db.getSchemaVersion()).toBe(40);
expect(db.getSchemaVersion()).toBe(41);
});
});
});

View File

@@ -734,6 +734,13 @@ export interface Task {
* recovery-policy module on each recoverable failure; cleared when work restarts
* cleanly or reaches a terminal column (in-review, done, archived). */
recoveryRetryCount?: number;
/** Number of times the self-healing manager has auto-requeued this task after
* the agent exited without calling `task_done`. Incremented by
* `recoverPartialProgressNoTaskDoneFailures` when a failed task with partial
* step progress is moved back to `todo` for a fresh execution. Capped by
* `MAX_TASK_DONE_RETRIES`; when exhausted the task stays in `in-review` for
* human inspection. Cleared on successful completion. */
taskDoneRetryCount?: number;
/** ISO-8601 timestamp indicating when the task becomes eligible for the next
* recovery retry. Scheduler and triage processor skip tasks whose
* `nextRecoveryAt` is still in the future. Cleared alongside `recoveryRetryCount`. */

View File

@@ -430,6 +430,7 @@ describe("SelfHealingManager", () => {
const recoverNoProgressNoTaskDoneFailures = vi.spyOn(manager, "recoverNoProgressNoTaskDoneFailures").mockResolvedValue(1);
const recoverCompletedTasks = vi.spyOn(manager, "recoverCompletedTasks").mockResolvedValue(1);
const recoverMisclassifiedFailures = vi.spyOn(manager, "recoverMisclassifiedFailures").mockResolvedValue(1);
const recoverPartialProgressNoTaskDoneFailures = vi.spyOn(manager, "recoverPartialProgressNoTaskDoneFailures").mockResolvedValue(1);
const recoverOrphanedExecutions = vi.spyOn(manager, "recoverOrphanedExecutions").mockResolvedValue(1);
const recoverApprovedTriageTasks = vi.spyOn(manager, "recoverApprovedTriageTasks").mockResolvedValue(1);
@@ -438,6 +439,7 @@ describe("SelfHealingManager", () => {
expect(recoverNoProgressNoTaskDoneFailures).toHaveBeenCalledTimes(1);
expect(recoverCompletedTasks).toHaveBeenCalledTimes(1);
expect(recoverMisclassifiedFailures).toHaveBeenCalledTimes(1);
expect(recoverPartialProgressNoTaskDoneFailures).toHaveBeenCalledTimes(1);
expect(recoverOrphanedExecutions).toHaveBeenCalledTimes(1);
expect(recoverApprovedTriageTasks).toHaveBeenCalledTimes(1);
});
@@ -1023,6 +1025,145 @@ describe("SelfHealingManager", () => {
});
});
describe("recoverPartialProgressNoTaskDoneFailures", () => {
it("requeues partial-progress no-task_done failures with bounded retry count", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-2164",
column: "in-review",
status: "failed",
error: "Agent finished without calling task_done (after retry)",
paused: false,
steps: [{ status: "done" }, { status: "pending" }, { status: "pending" }],
log: [],
},
]);
const result = await managerWithRecovery.recoverPartialProgressNoTaskDoneFailures();
expect(result).toBe(1);
expect(store.listTasks).toHaveBeenCalledWith({ column: "in-review" });
expect(store.updateTask).toHaveBeenCalledWith("FN-2164", {
status: null,
error: null,
sessionFile: null,
taskDoneRetryCount: 1,
});
expect(store.logEntry).toHaveBeenCalledWith(
"FN-2164",
expect.stringContaining("Auto-retry 1/3"),
);
expect(store.moveTask).toHaveBeenCalledWith("FN-2164", "todo");
managerWithRecovery.stop();
});
it("skips tasks whose retry count has reached the max", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-2164",
column: "in-review",
status: "failed",
error: "Agent finished without calling task_done (after retry)",
paused: false,
taskDoneRetryCount: 3,
steps: [{ status: "done" }, { status: "pending" }],
log: [],
},
]);
const result = await managerWithRecovery.recoverPartialProgressNoTaskDoneFailures();
expect(result).toBe(0);
expect(store.updateTask).not.toHaveBeenCalled();
expect(store.moveTask).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
it("skips tasks where all steps are already done (handled by misclassified recovery)", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-2164",
column: "in-review",
status: "failed",
error: "Agent finished without calling task_done (after retry)",
paused: false,
steps: [{ status: "done" }, { status: "done" }, { status: "skipped" }],
log: [],
},
]);
const result = await managerWithRecovery.recoverPartialProgressNoTaskDoneFailures();
expect(result).toBe(0);
expect(store.updateTask).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
it("skips tasks with zero step progress (handled by no-progress recovery)", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-2164",
column: "in-review",
status: "failed",
error: "Agent finished without calling task_done (after retry)",
paused: false,
steps: [{ status: "pending" }, { status: "pending" }],
log: [],
},
]);
const result = await managerWithRecovery.recoverPartialProgressNoTaskDoneFailures();
expect(result).toBe(0);
expect(store.updateTask).not.toHaveBeenCalled();
managerWithRecovery.stop();
});
it("skips tasks with unrelated failure reasons", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
rootDir: "/tmp/test-project",
});
(store.listTasks as ReturnType<typeof vi.fn>).mockResolvedValue([
{
id: "FN-2164",
column: "in-review",
status: "failed",
error: "Workflow step failed",
paused: false,
steps: [{ status: "done" }, { status: "pending" }],
log: [],
},
]);
const result = await managerWithRecovery.recoverPartialProgressNoTaskDoneFailures();
expect(result).toBe(0);
managerWithRecovery.stop();
});
});
describe("recoverMergedReviewTasks", () => {
it("finalizes stale merging tasks when a task commit already landed", async () => {
const managerWithRecovery = new SelfHealingManager(store, {
@@ -2210,6 +2351,7 @@ describe("maintenance cycle concurrency", () => {
(vi.spyOn(manager as any, "recoverMergedReviewTasks").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverMisclassifiedFailures").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverNoProgressNoTaskDoneFailures").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverPartialProgressNoTaskDoneFailures").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverOrphanedExecutions").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverApprovedTriageTasks").mockResolvedValue(0) as any);
(vi.spyOn(manager as any, "recoverOrphanedSpecifyingTasks").mockResolvedValue(0) as any);
@@ -2276,6 +2418,7 @@ describe("maintenance cycle concurrency", () => {
makeSlow("recoverMergedReviewTasks");
makeSlow("recoverMisclassifiedFailures");
makeSlow("recoverNoProgressNoTaskDoneFailures");
makeSlow("recoverPartialProgressNoTaskDoneFailures");
makeSlow("recoverOrphanedExecutions");
makeSlow("recoverApprovedTriageTasks");
makeSlow("recoverOrphanedSpecifyingTasks");
@@ -2297,6 +2440,7 @@ describe("maintenance cycle concurrency", () => {
"recoverMergedReviewTasks",
"recoverMisclassifiedFailures",
"recoverNoProgressNoTaskDoneFailures",
"recoverPartialProgressNoTaskDoneFailures",
"recoverOrphanedExecutions",
"recoverApprovedTriageTasks",
"recoverOrphanedSpecifyingTasks",

View File

@@ -80,6 +80,13 @@ const NON_TERMINAL_STEP_STATUSES = new Set(["pending", "in-progress"]);
*/
const ORPHANED_WITH_WORKTREE_GRACE_MS = 300_000;
/**
* Maximum times a task can be auto-requeued after the agent exits without
* calling `task_done`. Bounded so a persistently-broken task cannot loop
* forever; when exhausted the task stays in `in-review` for human inspection.
*/
const MAX_TASK_DONE_RETRIES = 3;
interface LandedTaskCommit {
sha: string;
subject?: string;
@@ -155,6 +162,7 @@ export class SelfHealingManager {
{ name: "failed-pre-merge-steps", fn: () => this.recoverReviewTasksWithFailedPreMergeSteps().then(() => undefined) },
{ name: "interrupted-merging", fn: () => this.recoverInterruptedMergingTasks().then(() => undefined) },
{ name: "misclassified-failures", fn: () => this.recoverMisclassifiedFailures().then(() => undefined) },
{ name: "partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures().then(() => undefined) },
{ name: "orphaned-executions", fn: () => this.recoverOrphanedExecutions().then(() => undefined) },
{ name: "approved-triage", fn: () => this.recoverApprovedTriageTasks().then(() => undefined) },
{ name: "orphaned-specifying", fn: () => this.recoverOrphanedSpecifyingTasks().then(() => undefined) },
@@ -532,6 +540,7 @@ export class SelfHealingManager {
{ name: "recover-merged-review", fn: () => this.recoverMergedReviewTasks() },
{ name: "recover-misclassified-failures", fn: () => this.recoverMisclassifiedFailures() },
{ name: "recover-no-progress-no-task-done", fn: () => this.recoverNoProgressNoTaskDoneFailures() },
{ name: "recover-partial-progress-no-task-done", fn: () => this.recoverPartialProgressNoTaskDoneFailures() },
{ name: "recover-orphaned-executions", fn: () => this.recoverOrphanedExecutions() },
{ name: "recover-approved-triage", fn: () => this.recoverApprovedTriageTasks() },
{ name: "recover-orphaned-specifying", fn: () => this.recoverOrphanedSpecifyingTasks() },
@@ -1197,6 +1206,83 @@ export class SelfHealingManager {
}
}
/**
* Recover `in-review` tasks marked as `failed` because the agent exited
* without calling `task_done` *with partial step progress* (some steps done,
* some still pending). The work-in-progress is valuable but incomplete —
* the existing worktree and branch are preserved and the task is moved back
* to `todo` so the scheduler re-dispatches it for a fresh execution that
* continues from where the previous attempt left off.
*
* Bounded by `MAX_TASK_DONE_RETRIES` (per-task `taskDoneRetryCount`) so a
* persistently-broken task cannot loop forever; when exhausted the task
* remains parked in `in-review` for manual intervention. The counter is
* cleared by the executor on successful completion.
*
* Distinct from sibling recoveries:
* - `recoverMisclassifiedFailures`: all steps done → clear error, leave for review.
* - `recoverNoProgressNoTaskDoneFailures`: `in-progress` with zero progress → clean requeue.
* - This one: `in-review` with partial progress → bounded requeue preserving work.
*
* @returns Number of tasks requeued for retry
*/
async recoverPartialProgressNoTaskDoneFailures(): Promise<number> {
try {
const tasks = await this.store.listTasks({ column: "in-review" });
const candidates = tasks.filter((task) =>
task.column === "in-review" &&
task.status === "failed" &&
isNoTaskDoneFailure(task) &&
!task.paused &&
!isTaskWorkComplete(task) &&
hasStepProgress(task) &&
(task.taskDoneRetryCount ?? 0) < MAX_TASK_DONE_RETRIES,
);
if (candidates.length === 0) return 0;
log.warn(
`Found ${candidates.length} partial-progress no-task_done failure(s) eligible for auto-retry`,
);
let recovered = 0;
for (const task of candidates) {
try {
const nextCount = (task.taskDoneRetryCount ?? 0) + 1;
await this.store.updateTask(task.id, {
status: null,
error: null,
sessionFile: null,
taskDoneRetryCount: nextCount,
});
await this.store.logEntry(
task.id,
`Auto-retry ${nextCount}/${MAX_TASK_DONE_RETRIES}: agent finished without task_done — requeuing to todo to resume partial work`,
);
await this.store.moveTask(task.id, "todo");
recovered++;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.error(
`Failed to auto-retry partial-progress no-task_done failure ${task.id}: ${errorMessage}`,
);
}
}
if (recovered > 0) {
log.log(
`Auto-retried ${recovered} partial-progress no-task_done failure(s) → todo`,
);
}
return recovered;
} catch (err: unknown) {
const errorMessage = err instanceof Error ? err.message : String(err);
log.error(`Partial-progress no-task_done recovery failed: ${errorMessage}`);
return 0;
}
}
private async hasRecoverableGitWork(task: Task): Promise<boolean> {
if (task.worktree && existsSync(task.worktree)) {
try {