test(FN-5168): complete Step 5 — reliability interaction regression
Fusion-Task-Id: FN-5168 Fusion-Task-Lineage: 0c865d3e-0886-4e51-a7ff-cb4c713dcc54
This commit is contained in:
committed by
gsxdsm
parent
368c4c37fe
commit
7ac8a73f1b
@@ -0,0 +1,198 @@
|
|||||||
|
import { EventEmitter } from "node:events";
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import "../executor-test-helpers.js";
|
||||||
|
import type { Task, TaskStore } from "@fusion/core";
|
||||||
|
import { TaskExecutor } from "../../executor.js";
|
||||||
|
import { SelfHealingManager } from "../../self-healing.js";
|
||||||
|
import { StuckTaskDetector } from "../../stuck-task-detector.js";
|
||||||
|
|
||||||
|
function createStore(task: Task, settings: Record<string, unknown> = {}): TaskStore & EventEmitter {
|
||||||
|
const emitter = new EventEmitter() as TaskStore & EventEmitter;
|
||||||
|
(emitter as any).getSettings = vi.fn().mockResolvedValue({
|
||||||
|
autoMerge: true,
|
||||||
|
globalPause: false,
|
||||||
|
enginePaused: false,
|
||||||
|
taskStuckTimeoutMs: 60_000,
|
||||||
|
maxStuckKills: 6,
|
||||||
|
inReviewStallDeadlockThreshold: 3,
|
||||||
|
inReviewStalledThresholdMs: 3_600_000,
|
||||||
|
stalePausedReviewThresholdMs: 3_600_000,
|
||||||
|
...settings,
|
||||||
|
});
|
||||||
|
(emitter as any).getTask = vi.fn().mockImplementation(async () => task);
|
||||||
|
(emitter as any).listTasks = vi.fn().mockImplementation(async ({ column }: { column?: string } = {}) => {
|
||||||
|
if (!column || task.column === column) return [task];
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
(emitter as any).updateTask = vi.fn().mockImplementation(async (_taskId: string, patch: Partial<Task>) => {
|
||||||
|
Object.assign(task, patch, { updatedAt: new Date(Date.now()).toISOString() });
|
||||||
|
});
|
||||||
|
(emitter as any).moveTask = vi.fn().mockImplementation(async (_taskId: string, column: string) => {
|
||||||
|
task.column = column as any;
|
||||||
|
task.updatedAt = new Date(Date.now()).toISOString();
|
||||||
|
});
|
||||||
|
(emitter as any).logEntry = vi.fn().mockImplementation(async (_taskId: string, action: string) => {
|
||||||
|
task.log = task.log ?? [];
|
||||||
|
task.log.push({ timestamp: new Date(Date.now()).toISOString(), action });
|
||||||
|
});
|
||||||
|
(emitter as any).recordRunAuditEvent = vi.fn().mockResolvedValue(undefined);
|
||||||
|
return emitter;
|
||||||
|
}
|
||||||
|
|
||||||
|
function baseTask(overrides: Record<string, unknown> = {}): Task {
|
||||||
|
return {
|
||||||
|
id: "FN-5168-RI",
|
||||||
|
lineageId: "lin-fn-5168-ri",
|
||||||
|
title: "non-progress churn",
|
||||||
|
description: "reliability interaction harness",
|
||||||
|
column: "in-progress",
|
||||||
|
status: null,
|
||||||
|
error: null,
|
||||||
|
steps: [{ name: "Implement", status: "in-progress" }],
|
||||||
|
workflowStepResults: [],
|
||||||
|
log: [],
|
||||||
|
dependencies: [],
|
||||||
|
createdAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
columnMovedAt: "2026-01-01T00:00:00.000Z",
|
||||||
|
stuckKillCount: 0,
|
||||||
|
...overrides,
|
||||||
|
} as any;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runNoProgressChurnFlow(options: { autoMerge?: boolean } = {}) {
|
||||||
|
const task = baseTask();
|
||||||
|
const store = createStore(task, { autoMerge: options.autoMerge ?? true });
|
||||||
|
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
|
||||||
|
const detector = new StuckTaskDetector(store, {
|
||||||
|
beforeRequeue: (taskId, reason, event) => manager.checkStuckBudget(taskId, reason, event),
|
||||||
|
});
|
||||||
|
const executor = new TaskExecutor(store, "/tmp/repo", { stuckTaskDetector: detector });
|
||||||
|
(detector as any).onLoopDetected = (event: any) => executor.handleLoopDetected(event);
|
||||||
|
|
||||||
|
const session = {
|
||||||
|
compact: vi.fn(async () => ({ summary: "Compacted conversation", tokensBefore: 150000 })),
|
||||||
|
steer: vi.fn(async () => {}),
|
||||||
|
dispose: vi.fn(),
|
||||||
|
sessionManager: { getLeafId: vi.fn().mockReturnValue("leaf-1") },
|
||||||
|
state: {},
|
||||||
|
};
|
||||||
|
(executor as any).activeSessions.set(task.id, {
|
||||||
|
session,
|
||||||
|
seenSteeringIds: new Set(),
|
||||||
|
});
|
||||||
|
|
||||||
|
detector.trackTask(task.id, session as any);
|
||||||
|
vi.advanceTimersByTime(61_000);
|
||||||
|
for (let i = 0; i < 80; i++) {
|
||||||
|
detector.recordActivity(task.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
await detector.killAndRetry(task.id, 60_000);
|
||||||
|
detector.recordProgress(task.id);
|
||||||
|
|
||||||
|
vi.advanceTimersByTime(61_000);
|
||||||
|
for (let i = 0; i < 25; i++) {
|
||||||
|
detector.recordIgnoredStepUpdate(task.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
await detector.checkNow();
|
||||||
|
|
||||||
|
return { task, store, manager, detector, executor, session };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("reliability interactions: non-progress churn", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("compacts once, then escalates repeated rebuffs to no-progress-churn without a second compact attempt", async () => {
|
||||||
|
const { task, session, manager } = await runNoProgressChurnFlow();
|
||||||
|
|
||||||
|
expect(session.compact).toHaveBeenCalledTimes(1);
|
||||||
|
expect(task.error).toMatch(/^STUCK_NO_PROGRESS_CHURN:/);
|
||||||
|
expect(task.column).toBe("in-review");
|
||||||
|
|
||||||
|
manager.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("beforeRequeue prevents requeue and parks the task in in-review failed", async () => {
|
||||||
|
const { task, store, manager } = await runNoProgressChurnFlow();
|
||||||
|
|
||||||
|
expect(task.status).toBe("failed");
|
||||||
|
expect(task.column).toBe("in-review");
|
||||||
|
expect(task.error).toMatch(/^STUCK_NO_PROGRESS_CHURN:/);
|
||||||
|
expect(store.moveTask).toHaveBeenCalledWith(task.id, "in-review");
|
||||||
|
|
||||||
|
manager.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("writes the operator guidance log entry", async () => {
|
||||||
|
const { task, manager } = await runNoProgressChurnFlow();
|
||||||
|
|
||||||
|
expect(task.log.some((entry) => entry.action.includes("manually decompose the work via fn_task_create child tasks"))).toBe(true);
|
||||||
|
|
||||||
|
manager.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits the no-progress churn audit event payload", async () => {
|
||||||
|
const { task, store, manager } = await runNoProgressChurnFlow();
|
||||||
|
|
||||||
|
expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
domain: "database",
|
||||||
|
mutationType: "task:stuck-no-progress-churn-terminalized",
|
||||||
|
target: task.id,
|
||||||
|
metadata: expect.objectContaining({
|
||||||
|
taskId: task.id,
|
||||||
|
ignoredStepUpdateCount: 25,
|
||||||
|
stuckKillStreak: 1,
|
||||||
|
lastReason: "no-progress-churn",
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
manager.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("respects FN-5147 autoMerge false by leaving the terminalized task in in-review", async () => {
|
||||||
|
const { task, manager } = await runNoProgressChurnFlow({ autoMerge: false });
|
||||||
|
|
||||||
|
expect(task.column).toBe("in-review");
|
||||||
|
expect(task.status).toBe("failed");
|
||||||
|
|
||||||
|
expect(await manager.surfaceInReviewStalls()).toBe(0);
|
||||||
|
expect(await manager.surfaceInReviewStalled()).toBe(0);
|
||||||
|
expect(task.column).toBe("in-review");
|
||||||
|
|
||||||
|
manager.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves STUCK_LOOP_EXHAUSTED when the churn signal does not fire", async () => {
|
||||||
|
const task = baseTask({ id: "FN-5168-LOOP", stuckKillCount: 6 });
|
||||||
|
const store = createStore(task);
|
||||||
|
const manager = new SelfHealingManager(store, { rootDir: "/tmp/repo" });
|
||||||
|
const detector = new StuckTaskDetector(store, {
|
||||||
|
beforeRequeue: (taskId, reason, event) => manager.checkStuckBudget(taskId, reason, event),
|
||||||
|
onLoopDetected: vi.fn().mockResolvedValue(false),
|
||||||
|
});
|
||||||
|
const session = {
|
||||||
|
dispose: vi.fn(),
|
||||||
|
};
|
||||||
|
|
||||||
|
detector.trackTask(task.id, session as any);
|
||||||
|
vi.advanceTimersByTime(61_000);
|
||||||
|
for (let i = 0; i < 80; i++) {
|
||||||
|
detector.recordActivity(task.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
await detector.killAndRetry(task.id, 60_000);
|
||||||
|
|
||||||
|
expect(task.error).toBe("STUCK_LOOP_EXHAUSTED: stuck kill budget exhausted (7/6) after last reason=loop.");
|
||||||
|
expect(task.column).toBe("in-review");
|
||||||
|
|
||||||
|
manager.stop();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -301,7 +301,6 @@ export class StuckTaskDetector {
|
|||||||
entry.lastProgressAt = Date.now();
|
entry.lastProgressAt = Date.now();
|
||||||
entry.activitySinceProgress = 0;
|
entry.activitySinceProgress = 0;
|
||||||
entry.ignoredStepUpdateCount = 0;
|
entry.ignoredStepUpdateCount = 0;
|
||||||
entry.loopObservedInLifecycle = false;
|
|
||||||
entry.recoveryInProgress = false;
|
entry.recoveryInProgress = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user