fix(FN-6043): recover stuck task processing

Fusion-Task-Id: FN-6043
This commit is contained in:
gsxdsm
2026-06-08 18:08:42 -07:00
parent ed0dc4a7b9
commit aa8bd3dc92
21 changed files with 643 additions and 164 deletions

View File

@@ -5,7 +5,18 @@ import { PullRequestView, type PrDetail } from "../components/PullRequestView";
// Icons → simple stubs so assertions key on text/testids, not SVG internals.
vi.mock("lucide-react", () => {
const Stub = () => <span />;
return new Proxy({}, { get: () => Stub });
return {
AlertTriangle: Stub,
CheckCircle: Stub,
Clock: Stub,
ExternalLink: Stub,
GitMerge: Stub,
GitPullRequest: Stub,
MessageSquare: Stub,
RotateCcw: Stub,
ThumbsUp: Stub,
XCircle: Stub,
};
});
function makeSummary(over: Partial<PrDetail["summary"]> = {}): PrDetail["summary"] {

View File

@@ -6,7 +6,9 @@ import type { Task } from "@fusion/core";
vi.mock("lucide-react", () => {
const Stub = () => null;
return new Proxy({}, { get: () => Stub });
return new Proxy({}, {
get: (_target, prop) => prop === "then" ? undefined : Stub,
});
});
vi.mock("../ProviderIcon", () => ({

View File

@@ -13,6 +13,14 @@ vi.mock("../../api", () => ({
fetchModels: vi.fn(),
fetchAgents: vi.fn(),
fetchDiscoveredSkills: vi.fn(),
fetchWorkflowStepTemplates: vi.fn().mockResolvedValue({ templates: [] }),
fetchPluginWorkflowStepTemplates: vi.fn().mockResolvedValue({ templates: [] }),
fetchConfig: vi.fn(),
fetchSettings: vi.fn(),
updateSettings: vi.fn(),
updateGlobalSettings: vi.fn(),
fetchWorkflowSettingValues: vi.fn().mockResolvedValue({ stored: {}, effective: {}, orphaned: [] }),
updateWorkflowSettingValues: vi.fn().mockResolvedValue({ stored: {}, effective: {}, orphaned: [] }),
}));
import {
@@ -21,6 +29,8 @@ import {
fetchStepParsers,
updateWorkflow,
fetchModels,
fetchConfig,
fetchSettings,
} from "../../api";
import type { TraitCatalogEntry } from "../../api";
import { WorkflowNodeEditor } from "../WorkflowNodeEditor";
@@ -68,6 +78,8 @@ describe("WorkflowNodeEditor — cli-agent executor (U15)", () => {
vi.mocked(fetchTraits).mockResolvedValue(TRAIT_CATALOG);
vi.mocked(fetchStepParsers).mockResolvedValue([]);
vi.mocked(fetchModels).mockResolvedValue({ models: [] });
vi.mocked(fetchConfig).mockResolvedValue({ maxConcurrent: 2, rootDir: "/tmp/project" });
vi.mocked(fetchSettings).mockResolvedValue({ autoMerge: true });
vi.mocked(updateWorkflow).mockResolvedValue(promptDef());
// Stub the adapter-catalog fetch.
vi.stubGlobal(

View File

@@ -127,6 +127,56 @@ async function spawnWrapperTree(signal: NodeJS.Signals) {
await waitFor(() => !isProcessAlive(pids!.childPid) && !isProcessAlive(pids!.grandchildPid));
}
async function spawnWrapperTreeUntilTimeout() {
const { pidFile, childPath, grandchildPath } = createStubProcessTree();
const wrapper = spawn(
process.execPath,
[wrapperPath, "--heap=6144", "run", "--project", "dashboard-api-quality"],
{
cwd: dashboardRoot,
stdio: "pipe",
env: {
...process.env,
FUSION_RUN_VITEST_TIMEOUT_MS: "100",
FUSION_RUN_VITEST_KILL_GRACE_MS: "50",
FUSION_RUN_VITEST_SPAWN_OVERRIDE: JSON.stringify({
command: process.execPath,
args: [childPath, pidFile, grandchildPath],
}),
},
},
);
activeWrappers.add(wrapper);
let pids: { childPid: number; grandchildPid: number } | null = null;
await waitFor(() => {
try {
pids = JSON.parse(readFileSync(pidFile, "utf8")) as { childPid: number; grandchildPid: number };
return Boolean(
pids &&
Number.isInteger(pids.childPid) &&
Number.isInteger(pids.grandchildPid) &&
isProcessAlive(pids.childPid) &&
isProcessAlive(pids.grandchildPid),
);
} catch {
return false;
}
});
registerGroupLeader(pids!.childPid);
registerPid(pids!.grandchildPid);
const exitCode = await new Promise<number | null>((resolve, reject) => {
wrapper.once("error", reject);
wrapper.once("close", (code) => resolve(code));
});
activeWrappers.delete(wrapper);
expect(exitCode).toBe(124);
await waitFor(() => !isProcessAlive(pids!.childPid) && !isProcessAlive(pids!.grandchildPid));
}
afterEach(async () => {
for (const wrapper of activeWrappers) {
wrapper.kill("SIGKILL");
@@ -169,4 +219,8 @@ describe("run-vitest-with-heap", () => {
it("reaps the spawned process group on SIGINT", async () => {
await spawnWrapperTree("SIGINT");
});
it("times out and reaps the spawned process group", async () => {
await spawnWrapperTreeUntilTimeout();
});
});

View File

@@ -1,5 +1,5 @@
#!/usr/bin/env node
/* global clearInterval, console, process, setInterval */
/* global clearInterval, clearTimeout, console, process, setInterval, setTimeout */
import { spawn } from "node:child_process";
@@ -16,6 +16,8 @@ if (vitestArgs.length === 0) {
const nodeOptions = [`--max-old-space-size=${heapMb}`, process.env.NODE_OPTIONS || ""]
.join(" ")
.trim();
const timeoutMs = Number.parseInt(process.env.FUSION_RUN_VITEST_TIMEOUT_MS || "900000", 10);
const forceKillGraceMs = Number.parseInt(process.env.FUSION_RUN_VITEST_KILL_GRACE_MS || "5000", 10);
function resolveSpawnCommand() {
const override = process.env.FUSION_RUN_VITEST_SPAWN_OVERRIDE;
@@ -51,11 +53,31 @@ const child = spawn(command, args, {
const heartbeat = setInterval(() => {
console.log(`[dashboard-vitest] still running: ${vitestArgs.join(" ")}`);
}, 5_000);
let timeoutExitCode = null;
let forceKillTimer = null;
const timeout = Number.isFinite(timeoutMs) && timeoutMs > 0
? setTimeout(() => {
timeoutExitCode = 124;
console.error(`[dashboard-vitest] timeout after ${timeoutMs}ms: ${vitestArgs.join(" ")}`);
forwardSignal("SIGTERM");
forceKillTimer = setTimeout(() => {
forwardSignal("SIGKILL");
}, Math.max(1, forceKillGraceMs));
forceKillTimer.unref();
}, timeoutMs)
: null;
timeout?.unref();
function clearHeartbeat() {
clearInterval(heartbeat);
}
function clearTimers() {
clearHeartbeat();
if (timeout) clearTimeout(timeout);
if (forceKillTimer) clearTimeout(forceKillTimer);
}
function forwardSignal(signal) {
clearHeartbeat();
@@ -86,7 +108,7 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
}
process.on("exit", () => {
clearHeartbeat();
clearTimers();
try {
process.kill(-child.pid, "SIGTERM");
} catch (error) {
@@ -101,13 +123,16 @@ process.on("exit", () => {
});
child.on("error", (error) => {
clearHeartbeat();
clearTimers();
console.error(error);
process.exit(1);
});
child.on("close", (code, signal) => {
clearHeartbeat();
clearTimers();
if (timeoutExitCode !== null) {
process.exit(timeoutExitCode);
}
if (signal) {
process.kill(process.pid, signal);
return;

View File

@@ -209,9 +209,7 @@ const batchedQualityAppComponentTestsA = batchedQualityAppComponentTests.slice(0
const batchedQualityAppComponentTestsB = batchedQualityAppComponentTests.slice(batchedQualityAppSplitIndex);
function buildComponentQualityInclude(testNames: readonly string[]): string[] {
return testNames.length > 0
? [`app/components/__tests__/{${testNames.join(",")}}.test.{ts,tsx}`]
: [];
return testNames.map((testName) => `app/components/__tests__/${testName}.test.{ts,tsx}`);
}
const qualityAppTests = [