fix(engine): make the planning->plan-review handoff atomic so planned cards stop stranding in Todo

Triage announced specification completion before its finally block marked the
plan work item terminal, so the Plan Review seeder saw its own still-running
predecessor as an "active continuation", bailed, and the discarded result
silently stranded the card until FN-8592 self-healing re-seeded it ~10 minutes
later (529 occurrences in 18 days).

- seedStrandedPlanReviewContinuation gains retirePredecessorId: idle check
  excludes the named predecessor, then retires it and installs the successor in
  ONE transaction under the task lock; a bailed seed mutates nothing.
- triage threads planningWorkItemId through PlanningHandoffReport; the runtime
  reaction passes it as retirePredecessorId.
- reactToSpecificationComplete consumes the seed result: typed quiet parks
  (incl. new "no-pre-release-plan-review"), bounded retries with a fresh
  task/IR snapshot per attempt (mid-retry pause/needs-replan honored), loud
  warning naming self-healing on exhaustion.
- Tests: PG both-orderings/no-mutation-on-bail/cross-task cases, direct engine
  seeder handoff cases, reaction retry/park/pause/replan cases.
- docs/solutions: new planning-handoff-race writeup; graph-entry-contract doc
  reclassifies the FN-8592 sweep as backstop-only.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-08-12 21:24:53 -07:00
parent ea53cbd4ff
commit 19dffe36f6
12 changed files with 574 additions and 27 deletions

View File

@@ -285,6 +285,71 @@ function seedStore(): { store: TaskStore; seeded: () => number } {
return { store, seeded: () => seeds };
}
/*
FNXC:PlanningHandoffAtomicity 2026-08-13-04:20:
THE INVARIANT: the normal planning handoff must not bail on its OWN predecessor.
Triage announces specification completion before its finally block marks the
planning work item terminal, so the seeder routinely observes that row still
`running`. Naming it via `retirePredecessorId` excludes it from the engine-side
active pre-check and routes to the atomic store op that retires it with the
successor install; any OTHER active row still blocks, and omitting the option
keeps the historical bail (which is what self-healing's idle-graph repair needs).
*/
describe("#2b the handoff seeder does not bail on its own named predecessor", () => {
const runningPlanItem = { id: "wi-plan", state: "running" } as WorkflowWorkItem;
it("seeds atomically past a still-running named predecessor", async () => {
const { store } = seedStore();
(store.listWorkflowWorkItemsForTask as ReturnType<typeof vi.fn>).mockResolvedValue([runningPlanItem]);
const result = await seedPreReleasePlanReviewContinuation(store, task(), planInPlaceIr(), {
retirePredecessorId: "wi-plan",
});
expect(result.seeded).toBe(true);
expect(store.seedStrandedPlanReviewContinuation).toHaveBeenCalledWith(
expect.objectContaining({ taskId: "FN-1", nodeId: PLAN_REVIEW_GROUP_ID }),
{ retirePredecessorId: "wi-plan" },
);
expect(store.replaceActiveTaskWorkflowContinuation).not.toHaveBeenCalled();
});
it("still bails on the same running row when no predecessor is named (opt-in)", async () => {
const { store, seeded } = seedStore();
(store.listWorkflowWorkItemsForTask as ReturnType<typeof vi.fn>).mockResolvedValue([runningPlanItem]);
const result = await seedPreReleasePlanReviewContinuation(store, task(), planInPlaceIr());
expect(result).toEqual({ seeded: false, reason: "active-continuation" });
expect(seeded()).toBe(0);
});
it("still bails when a DIFFERENT active row exists alongside the named predecessor", async () => {
const { store, seeded } = seedStore();
(store.listWorkflowWorkItemsForTask as ReturnType<typeof vi.fn>).mockResolvedValue([
runningPlanItem,
{ id: "wi-other", state: "runnable" } as WorkflowWorkItem,
]);
const result = await seedPreReleasePlanReviewContinuation(store, task(), planInPlaceIr(), {
retirePredecessorId: "wi-plan",
});
expect(result).toEqual({ seeded: false, reason: "active-continuation" });
expect(seeded()).toBe(0);
});
it("reports a workflow without a pre-release Plan Review node as its own quiet reason", async () => {
const { store, seeded } = seedStore();
const result = await seedPreReleasePlanReviewContinuation(store, task(), releaseIr(), {
retirePredecessorId: "wi-plan",
});
expect(result).toEqual({ seeded: false, reason: "no-pre-release-plan-review" });
expect(seeded()).toBe(0);
});
});
describe("#2 neither continuation seeder arms a run for a card blocked on approval", () => {
it("seeds for an ordinary specified card (the control)", async () => {
const { store, seeded } = seedStore();

View File

@@ -31,6 +31,7 @@ import { describe, expect, it, vi } from "vitest";
import type { Task, WorkflowIr } from "@fusion/core";
import { reactToSpecificationComplete } from "../runtimes/in-process-runtime.js";
import type { PlanReviewSeedBailReason } from "../plan-review-continuation.js";
import type { PlanningHandoffOutcome } from "../triage.js";
const IR = { version: "v2", name: "wf", columns: [], nodes: [], edges: [] } as unknown as WorkflowIr;
@@ -38,22 +39,39 @@ const IR = { version: "v2", name: "wf", columns: [], nodes: [], edges: [] } as u
// `null` means the row vanished. NOT `undefined`: that would trigger the default
// parameter and silently hand the reaction a live task, turning the
// vanished-task case into a duplicate of the control.
function harness(task: Task | null = { id: "FN-1", column: "todo" } as Task) {
function harness(
task: Task | null = { id: "FN-1", column: "todo" } as Task,
seedImpl?: (t: Task) => Promise<{ seeded: boolean; reason?: PlanReviewSeedBailReason }>,
// getTask is invoked once per attempt (the stale-snapshot fix); an impl override
// lets a test change the task's state between retry attempts.
getTaskImpl?: (call: number) => Task | null,
) {
const seeded: string[] = [];
const kicks: string[] = [];
const logs: string[] = [];
const warns: string[] = [];
const sleeps: number[] = [];
let getTaskCalls = 0;
return {
seeded,
kicks,
logs,
warns,
sleeps,
run: (outcome: PlanningHandoffOutcome) => reactToSpecificationComplete({
taskId: "FN-1",
outcome,
getTask: async () => task ?? undefined,
getTask: async () => {
getTaskCalls += 1;
const resolved = getTaskImpl ? getTaskImpl(getTaskCalls) : task;
return resolved ?? undefined;
},
resolveIr: async () => IR,
seed: async (t) => { seeded.push(t.id); return { seeded: true }; },
seed: async (t) => { seeded.push(t.id); return seedImpl ? seedImpl(t) : { seeded: true }; },
kick: () => { kicks.push("kick"); },
log: (m) => { logs.push(m); },
warn: (m) => { warns.push(m); },
sleep: async (ms) => { sleeps.push(ms); },
}),
};
}
@@ -125,9 +143,128 @@ describe("specification-complete reaction arms a plan review only on a real hand
seed: async () => ({ seeded: true }),
kick: () => {},
log: () => {},
warn: () => {},
});
expect(getTask).not.toHaveBeenCalled();
expect(resolveIr).not.toHaveBeenCalled();
});
});
/*
FNXC:PlanningHandoffAtomicity 2026-08-13-03:49:
THE INVARIANT: a released card's Plan Review handoff is never dropped silently.
The seed result used to be discarded, so a bail (the seeder racing triage's own
finally-block terminal transition and seeing its caller as an "active
continuation") left the card stranded until self-healing re-seeded it ~10 minutes
later — 529 times in 18 days. The reaction must consume the result: quiet parks
stay quiet, anomalies retry a bounded number of times, and a final failure is
warned loudly with the recovery owner named.
*/
describe("specification-complete reaction never drops a failed handoff silently", () => {
for (const reason of ["awaiting-approval", "paused", "plan-review-passed", "no-pre-release-plan-review"] as const) {
it(`treats ${reason} as a quiet park: no retry, no warning`, async () => {
const h = harness(undefined, async () => ({ seeded: false, reason }));
await h.run("released");
expect(h.seeded).toEqual(["FN-1"]);
expect(h.kicks).toEqual([]);
expect(h.warns).toEqual([]);
expect(h.sleeps).toEqual([]);
expect(h.logs.some((m) => m.includes(reason))).toBe(true);
});
}
it("retries an active-continuation bail and succeeds when the writer clears", async () => {
let calls = 0;
const h = harness(undefined, async () => {
calls += 1;
return calls < 2 ? { seeded: false, reason: "active-continuation" } : { seeded: true };
});
await h.run("released");
expect(calls).toBe(2);
expect(h.kicks).toEqual(["kick"]);
expect(h.sleeps).toEqual([1_000]);
expect(h.warns).toHaveLength(1);
expect(h.warns[0]).toContain("retrying");
});
it("retries a thrown store error the same way as a bail", async () => {
let calls = 0;
const h = harness(undefined, async () => {
calls += 1;
if (calls < 2) throw new Error("transient db error");
return { seeded: true };
});
await h.run("released");
expect(calls).toBe(2);
expect(h.kicks).toEqual(["kick"]);
expect(h.warns[0]).toContain("transient db error");
});
it("exhausts the bounded retries and names the self-healing recovery owner", async () => {
const h = harness(undefined, async () => ({ seeded: false, reason: "active-continuation" }));
await h.run("released");
expect(h.seeded).toEqual(["FN-1", "FN-1", "FN-1"]);
expect(h.sleeps).toEqual([1_000, 5_000]);
expect(h.kicks).toEqual([]);
const final = h.warns[h.warns.length - 1];
expect(final).toContain("failed after 3 attempts");
expect(final).toContain("self-healing");
});
/*
FNXC:PlanningHandoffAtomicity 2026-08-13-04:20:
The task snapshot is re-fetched per attempt, so operator state landing during a
retry delay is honored: a pause stops the loop without another seed, and a
needs-replan status (the plan was superseded mid-retry) exits quietly to the
replan loop. A single pre-loop snapshot could not honor either.
*/
it("honors an operator pause that lands between retry attempts", async () => {
const h = harness(
undefined,
async () => ({ seeded: false, reason: "active-continuation" }),
(call) => (call === 1
? ({ id: "FN-1", column: "todo" } as Task)
: ({ id: "FN-1", column: "todo", paused: true } as Task)),
);
await h.run("released");
expect(h.seeded).toEqual(["FN-1"]);
expect(h.kicks).toEqual([]);
expect(h.warns).toHaveLength(1);
});
it("exits quietly when a replan supersedes the plan between retry attempts", async () => {
const h = harness(
undefined,
async () => ({ seeded: false, reason: "active-continuation" }),
(call) => (call === 1
? ({ id: "FN-1", column: "todo" } as Task)
: ({ id: "FN-1", column: "todo", status: "needs-replan" } as Task)),
);
await h.run("released");
expect(h.seeded).toEqual(["FN-1"]);
expect(h.kicks).toEqual([]);
expect(h.logs.some((m) => m.includes("needs-replan"))).toBe(true);
});
it("never arms Plan Review for a card that is needs-replan at reaction time", async () => {
const h = harness({ id: "FN-1", column: "todo", status: "needs-replan" } as Task);
await h.run("released");
expect(h.seeded).toEqual([]);
expect(h.kicks).toEqual([]);
});
});

View File

@@ -37,18 +37,33 @@ export type ApprovedPlanReviewHandoffResult = {
* an active non-task continuation means the graph is not idle, even though the
* newly seeded continuation itself remains kind `task` for processor parity.
*/
/*
FNXC:PlanningHandoffAtomicity 2026-08-13-04:20:
The bail-reason union is exported so the runtime reaction's quiet-park set stays type-linked to it:
a rename or new reason here must be a compile error at the consumer, not a silent behavior change.
"no-pre-release-plan-review" exists because a workflow with no pre-release Plan Review node (or a
plan-review gate living in a WIP column) is a legitimate configuration — the reaction must treat
that bail as a quiet park, not an anomaly worth retries and warnings.
*/
export type PlanReviewSeedBailReason =
| "no-pre-release-plan-review"
| "active-continuation"
| "plan-review-passed"
| "awaiting-approval"
| "paused";
export async function seedPreReleasePlanReviewContinuation(
store: TaskStore,
task: Task,
ir: WorkflowIr,
options: { atomic?: boolean } = {},
options: { atomic?: boolean; retirePredecessorId?: string } = {},
): Promise<{
seeded: boolean;
reason?: "active-continuation" | "plan-review-passed" | "awaiting-approval" | "paused";
reason?: PlanReviewSeedBailReason;
workItemId?: string;
}> {
const node = resolvePreReleasePlanReviewNode(ir);
if (!node || node.column !== task.column) return { seeded: false };
if (!node || node.column !== task.column) return { seeded: false, reason: "no-pre-release-plan-review" };
/*
FNXC:PlanApprovalHold 2026-07-27-19:30 (U7 / R4):
Arming a runnable continuation is starting AI work on this card, so the parks
@@ -70,7 +85,18 @@ export async function seedPreReleasePlanReviewContinuation(
if (isTaskBlockedOnApproval(task)) return { seeded: false, reason: "awaiting-approval" };
if (task.paused === true || task.userPaused === true) return { seeded: false, reason: "paused" };
const items = await store.listWorkflowWorkItemsForTask(task.id);
const active = items.filter((item) => ACTIVE_WORKFLOW_WORK_ITEM_STATES.includes(item.state));
/*
FNXC:PlanningHandoffAtomicity 2026-08-13-03:49:
The normal planning handoff names its own just-finished plan work item via
`retirePredecessorId`. That row is often still `running` here because triage
announces completion BEFORE its finally block marks the row terminal, so counting
it as "active" made the seeder bail on its own predecessor and strand the card
until FN-8592 self-healing re-seeded it ~10 minutes later. The predecessor is
excluded from this advisory pre-check and retired atomically with the successor
install inside the store operation; any OTHER active row still blocks the seed.
*/
const active = items.filter((item) =>
ACTIVE_WORKFLOW_WORK_ITEM_STATES.includes(item.state) && item.id !== options.retirePredecessorId);
if (active.length > 0) return { seeded: false, reason: "active-continuation" };
// FNXC:StrandedHoldContinuation 2026-07-26-16:10:
// A terminal predecessor is still part of this task's durable run history.
@@ -91,6 +117,16 @@ export async function seedPreReleasePlanReviewContinuation(
targetColumn: task.column,
irHash: computeWorkflowIrPin(ir, node.id).irHash,
};
/*
FNXC:PlanningHandoffAtomicity 2026-08-13-03:49:
A caller that names a predecessor gets the atomic conditional seed regardless of the
`atomic` flag: the pre-check above is advisory (outside any transaction), so the
retire-predecessor + idle-recheck + successor-install must all land in ONE store
transaction under the task lock for the handoff to be ordering-proof.
*/
if (options.retirePredecessorId) {
return store.seedStrandedPlanReviewContinuation(input, { retirePredecessorId: options.retirePredecessorId });
}
if (options.atomic) return store.seedStrandedPlanReviewContinuation(input);
const item = await store.replaceActiveTaskWorkflowContinuation(input);
return { seeded: true, workItemId: item.id };
@@ -120,9 +156,13 @@ export async function resumeApprovedPlanReviewHandoff(
const seeded = await seedPreReleasePlanReviewContinuation(store, task, ir, { atomic: true });
if (seeded.seeded) return { resumed: true, reason: "seeded", workItemId: seeded.workItemId };
// FNXC:PlanningHandoffAtomicity 2026-08-13-04:20: the no-node bail now carries its own reason
// token; this seam already pre-checked the node above, so map it to its legacy result name.
return {
resumed: false,
reason: seeded.reason ?? "not-plan-in-place",
reason: seeded.reason === undefined || seeded.reason === "no-pre-release-plan-review"
? "not-plan-in-place"
: seeded.reason,
};
}

View File

@@ -66,7 +66,7 @@ import { validateProjectNodeMapping } from "../project/node-dispatch-validation.
import { attachAgentLinkSync } from "../agents/task-agent-sync.js";
import { createRunAuditor, generateSyntheticRunId } from "../util/run-audit.js";
import { setImmediate as setImmediateCb } from "node:timers";
import { seedPreReleasePlanReviewContinuation } from "../plan-review-continuation.js";
import { seedPreReleasePlanReviewContinuation, type PlanReviewSeedBailReason } from "../plan-review-continuation.js";
import {
formatAdmissionCapacityQueuedReason,
persistedTopLevelAgentTaskIdsFromStore,
@@ -372,9 +372,18 @@ export interface SpecificationCompleteReactionDeps {
outcome: PlanningHandoffOutcome;
getTask: (taskId: string) => Promise<Task | undefined>;
resolveIr: (taskId: string) => Promise<WorkflowIr>;
seed: (task: Task, ir: WorkflowIr) => Promise<{ seeded: boolean; reason?: string }>;
seed: (task: Task, ir: WorkflowIr) => Promise<{ seeded: boolean; reason?: PlanReviewSeedBailReason }>;
kick: () => void;
log: (message: string) => void;
/*
FNXC:PlanningHandoffAtomicity 2026-08-13-03:49:
A seed outcome that is neither success nor a legitimate operator park is a broken
handoff that used to be dropped silently; it must be surfaced loudly because the
only remaining recovery owner is the FN-8592 self-healing sweep (~10 min later).
*/
warn: (message: string) => void;
/** Injectable delay for the bounded transient-failure retry; defaults to setTimeout. */
sleep?: (ms: number) => Promise<void>;
}
/**
@@ -414,11 +423,69 @@ export async function reactToSpecificationComplete(
return;
}
deps.log(`Specified ${deps.taskId} → todo`);
const live = await deps.getTask(deps.taskId);
if (!live || live.paused || live.userPaused) return;
const ir = await deps.resolveIr(live.id);
await deps.seed(live, ir);
deps.kick();
/*
FNXC:PlanningHandoffAtomicity 2026-08-13-03:49:
The seed outcome was previously discarded, so a bailed handoff was invisible: the
seeder saw the caller's own still-running plan work item as an "active
continuation", returned {seeded:false}, and the card stranded in the hold column
until FN-8592 self-healing re-seeded it ~10 minutes later. The seed is now atomic
(it retires the named predecessor in the same transaction), so a bail here is a
genuine anomaly. Consume the result: retry a bounded number of times to absorb a
transient store error or a racing writer, then warn loudly — self-healing remains
the durable backstop, never the primary handoff.
FNXC:PlanningHandoffAtomicity 2026-08-13-04:20:
The task and IR are re-fetched at the TOP OF EVERY ATTEMPT (review finding: a
single pre-loop snapshot let an operator pause or a replan landing during the
retry delays be ignored, arming Plan Review for a plan the operator had just
parked or superseded). The seeder re-applies the pause/approval guards from the
task object it is given, so a fresh snapshot per attempt is what makes those
guards current. A needs-replan status means the plan this handoff belongs to is
already superseded — quiet exit, the replan loop owns the card now.
QUIET_PARK_REASONS is typed against the seeder's exported reason union so a
renamed or added reason is a compile error here, not a silent misroute into the
retry/warn path.
*/
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
const QUIET_PARK_REASONS: ReadonlySet<PlanReviewSeedBailReason> = new Set<PlanReviewSeedBailReason>([
"no-pre-release-plan-review",
"awaiting-approval",
"paused",
"plan-review-passed",
]);
const RETRY_DELAYS_MS = [1_000, 5_000];
for (let attempt = 0; ; attempt++) {
let failure: string | null = null;
try {
const live = await deps.getTask(deps.taskId);
if (!live || live.paused || live.userPaused) return;
if (live.status === "needs-replan") {
deps.log(`Plan Review handoff for ${deps.taskId} not armed (needs-replan)`);
return;
}
const ir = await deps.resolveIr(live.id);
const result = await deps.seed(live, ir);
if (result.seeded) {
deps.kick();
return;
}
if (result.reason && QUIET_PARK_REASONS.has(result.reason)) {
deps.log(`Plan Review handoff for ${deps.taskId} not armed (${result.reason})`);
return;
}
failure = result.reason ?? "not-seeded";
} catch (error) {
failure = error instanceof Error ? error.message : String(error);
}
if (attempt >= RETRY_DELAYS_MS.length) {
deps.warn(
`Plan Review handoff for ${deps.taskId} failed after ${attempt + 1} attempts (${failure}) — stranded-continuation self-healing will recover it`,
);
return;
}
deps.warn(`Plan Review handoff for ${deps.taskId} did not seed (${failure}) — retrying`);
await sleep(RETRY_DELAYS_MS[attempt]);
}
}
/** Everything the drain pass touches, injected so the pass is exercisable without
@@ -1711,9 +1778,19 @@ export class InProcessRuntime
outcome: report.outcome,
getTask: (id) => Promise.resolve(this.taskStore.getTask(id)),
resolveIr: (id) => resolveWorkflowIrForTask(this.taskStore, id),
seed: (task, ir) => seedPreReleasePlanReviewContinuation(this.taskStore, task, ir),
/*
FNXC:PlanningHandoffAtomicity 2026-08-13-03:49:
The report carries the planning session's own durable work item id so the
seeder can retire that exact predecessor row atomically with the successor
install. Without it the seeder raced triage's finally-block terminal
transition and bailed on its own caller (529 stranded cards in 18 days).
*/
seed: (task, ir) => seedPreReleasePlanReviewContinuation(this.taskStore, task, ir, {
retirePredecessorId: report.planningWorkItemId,
}),
kick: () => this.kickWorkflowContinuationProcessor(),
log: (message) => runtimeLog.log(message),
warn: (message) => runtimeLog.warn(message),
}).catch((error) => {
runtimeLog.error(`Failed to start Todo plan review for ${t.id}:`, error);
});

View File

@@ -348,6 +348,16 @@ export type PlanningHandoffOutcome = "released" | "parked" | "withheld";
* `finalizeApprovedTask` for why this is a report object and not a return value. */
export interface PlanningHandoffReport {
outcome: PlanningHandoffOutcome;
/*
FNXC:PlanningHandoffAtomicity 2026-08-13-03:49:
The durable work item this planning session ran under. The runtime's
specification-complete reaction passes it to the Plan Review seeder so the
successor install can atomically retire this exact predecessor row instead of
bailing on it: triage announces completion BEFORE its finally block transitions
the row to succeeded, so without this id the seeder saw its own caller as an
"active continuation" and silently stranded the card for self-healing to repair.
*/
planningWorkItemId?: string;
}
@@ -3586,7 +3596,7 @@ export class TriageProcessor {
// FNXC:TriagePlanningRetry 2026-08-03-00:20: A duplicate remains a separate closure
// path, but fallback-authored or inherited markers cannot bypass clean-attempt admission.
const duplicateReport: PlanningHandoffReport = { outcome: "parked" };
const duplicateReport: PlanningHandoffReport = { outcome: "parked", planningWorkItemId };
if (await this.tryFinalizeExplicitDuplicateMarker(task, written, settings, {
isReplan,
feedback,
@@ -3609,6 +3619,7 @@ export class TriageProcessor {
isReplan,
feedback,
});
finalizeReport.planningWorkItemId = planningWorkItemId;
this.options.onSpecifyComplete?.(task, finalizeReport);
} finally {
this.activeSessions.delete(task.id);