fix(FN-6043): recover stuck task processing
Fusion-Task-Id: FN-6043
This commit is contained in:
@@ -5,7 +5,18 @@ import { PullRequestView, type PrDetail } from "../components/PullRequestView";
|
||||
// Icons → simple stubs so assertions key on text/testids, not SVG internals.
|
||||
vi.mock("lucide-react", () => {
|
||||
const Stub = () => <span />;
|
||||
return new Proxy({}, { get: () => Stub });
|
||||
return {
|
||||
AlertTriangle: Stub,
|
||||
CheckCircle: Stub,
|
||||
Clock: Stub,
|
||||
ExternalLink: Stub,
|
||||
GitMerge: Stub,
|
||||
GitPullRequest: Stub,
|
||||
MessageSquare: Stub,
|
||||
RotateCcw: Stub,
|
||||
ThumbsUp: Stub,
|
||||
XCircle: Stub,
|
||||
};
|
||||
});
|
||||
|
||||
function makeSummary(over: Partial<PrDetail["summary"]> = {}): PrDetail["summary"] {
|
||||
|
||||
@@ -6,7 +6,9 @@ import type { Task } from "@fusion/core";
|
||||
|
||||
vi.mock("lucide-react", () => {
|
||||
const Stub = () => null;
|
||||
return new Proxy({}, { get: () => Stub });
|
||||
return new Proxy({}, {
|
||||
get: (_target, prop) => prop === "then" ? undefined : Stub,
|
||||
});
|
||||
});
|
||||
|
||||
vi.mock("../ProviderIcon", () => ({
|
||||
|
||||
@@ -13,6 +13,14 @@ vi.mock("../../api", () => ({
|
||||
fetchModels: vi.fn(),
|
||||
fetchAgents: vi.fn(),
|
||||
fetchDiscoveredSkills: vi.fn(),
|
||||
fetchWorkflowStepTemplates: vi.fn().mockResolvedValue({ templates: [] }),
|
||||
fetchPluginWorkflowStepTemplates: vi.fn().mockResolvedValue({ templates: [] }),
|
||||
fetchConfig: vi.fn(),
|
||||
fetchSettings: vi.fn(),
|
||||
updateSettings: vi.fn(),
|
||||
updateGlobalSettings: vi.fn(),
|
||||
fetchWorkflowSettingValues: vi.fn().mockResolvedValue({ stored: {}, effective: {}, orphaned: [] }),
|
||||
updateWorkflowSettingValues: vi.fn().mockResolvedValue({ stored: {}, effective: {}, orphaned: [] }),
|
||||
}));
|
||||
|
||||
import {
|
||||
@@ -21,6 +29,8 @@ import {
|
||||
fetchStepParsers,
|
||||
updateWorkflow,
|
||||
fetchModels,
|
||||
fetchConfig,
|
||||
fetchSettings,
|
||||
} from "../../api";
|
||||
import type { TraitCatalogEntry } from "../../api";
|
||||
import { WorkflowNodeEditor } from "../WorkflowNodeEditor";
|
||||
@@ -68,6 +78,8 @@ describe("WorkflowNodeEditor — cli-agent executor (U15)", () => {
|
||||
vi.mocked(fetchTraits).mockResolvedValue(TRAIT_CATALOG);
|
||||
vi.mocked(fetchStepParsers).mockResolvedValue([]);
|
||||
vi.mocked(fetchModels).mockResolvedValue({ models: [] });
|
||||
vi.mocked(fetchConfig).mockResolvedValue({ maxConcurrent: 2, rootDir: "/tmp/project" });
|
||||
vi.mocked(fetchSettings).mockResolvedValue({ autoMerge: true });
|
||||
vi.mocked(updateWorkflow).mockResolvedValue(promptDef());
|
||||
// Stub the adapter-catalog fetch.
|
||||
vi.stubGlobal(
|
||||
|
||||
@@ -127,6 +127,56 @@ async function spawnWrapperTree(signal: NodeJS.Signals) {
|
||||
await waitFor(() => !isProcessAlive(pids!.childPid) && !isProcessAlive(pids!.grandchildPid));
|
||||
}
|
||||
|
||||
async function spawnWrapperTreeUntilTimeout() {
|
||||
const { pidFile, childPath, grandchildPath } = createStubProcessTree();
|
||||
const wrapper = spawn(
|
||||
process.execPath,
|
||||
[wrapperPath, "--heap=6144", "run", "--project", "dashboard-api-quality"],
|
||||
{
|
||||
cwd: dashboardRoot,
|
||||
stdio: "pipe",
|
||||
env: {
|
||||
...process.env,
|
||||
FUSION_RUN_VITEST_TIMEOUT_MS: "100",
|
||||
FUSION_RUN_VITEST_KILL_GRACE_MS: "50",
|
||||
FUSION_RUN_VITEST_SPAWN_OVERRIDE: JSON.stringify({
|
||||
command: process.execPath,
|
||||
args: [childPath, pidFile, grandchildPath],
|
||||
}),
|
||||
},
|
||||
},
|
||||
);
|
||||
activeWrappers.add(wrapper);
|
||||
|
||||
let pids: { childPid: number; grandchildPid: number } | null = null;
|
||||
await waitFor(() => {
|
||||
try {
|
||||
pids = JSON.parse(readFileSync(pidFile, "utf8")) as { childPid: number; grandchildPid: number };
|
||||
return Boolean(
|
||||
pids &&
|
||||
Number.isInteger(pids.childPid) &&
|
||||
Number.isInteger(pids.grandchildPid) &&
|
||||
isProcessAlive(pids.childPid) &&
|
||||
isProcessAlive(pids.grandchildPid),
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
registerGroupLeader(pids!.childPid);
|
||||
registerPid(pids!.grandchildPid);
|
||||
|
||||
const exitCode = await new Promise<number | null>((resolve, reject) => {
|
||||
wrapper.once("error", reject);
|
||||
wrapper.once("close", (code) => resolve(code));
|
||||
});
|
||||
activeWrappers.delete(wrapper);
|
||||
|
||||
expect(exitCode).toBe(124);
|
||||
await waitFor(() => !isProcessAlive(pids!.childPid) && !isProcessAlive(pids!.grandchildPid));
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const wrapper of activeWrappers) {
|
||||
wrapper.kill("SIGKILL");
|
||||
@@ -169,4 +219,8 @@ describe("run-vitest-with-heap", () => {
|
||||
it("reaps the spawned process group on SIGINT", async () => {
|
||||
await spawnWrapperTree("SIGINT");
|
||||
});
|
||||
|
||||
it("times out and reaps the spawned process group", async () => {
|
||||
await spawnWrapperTreeUntilTimeout();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env node
|
||||
/* global clearInterval, console, process, setInterval */
|
||||
/* global clearInterval, clearTimeout, console, process, setInterval, setTimeout */
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
@@ -16,6 +16,8 @@ if (vitestArgs.length === 0) {
|
||||
const nodeOptions = [`--max-old-space-size=${heapMb}`, process.env.NODE_OPTIONS || ""]
|
||||
.join(" ")
|
||||
.trim();
|
||||
const timeoutMs = Number.parseInt(process.env.FUSION_RUN_VITEST_TIMEOUT_MS || "900000", 10);
|
||||
const forceKillGraceMs = Number.parseInt(process.env.FUSION_RUN_VITEST_KILL_GRACE_MS || "5000", 10);
|
||||
|
||||
function resolveSpawnCommand() {
|
||||
const override = process.env.FUSION_RUN_VITEST_SPAWN_OVERRIDE;
|
||||
@@ -51,11 +53,31 @@ const child = spawn(command, args, {
|
||||
const heartbeat = setInterval(() => {
|
||||
console.log(`[dashboard-vitest] still running: ${vitestArgs.join(" ")}`);
|
||||
}, 5_000);
|
||||
let timeoutExitCode = null;
|
||||
let forceKillTimer = null;
|
||||
const timeout = Number.isFinite(timeoutMs) && timeoutMs > 0
|
||||
? setTimeout(() => {
|
||||
timeoutExitCode = 124;
|
||||
console.error(`[dashboard-vitest] timeout after ${timeoutMs}ms: ${vitestArgs.join(" ")}`);
|
||||
forwardSignal("SIGTERM");
|
||||
forceKillTimer = setTimeout(() => {
|
||||
forwardSignal("SIGKILL");
|
||||
}, Math.max(1, forceKillGraceMs));
|
||||
forceKillTimer.unref();
|
||||
}, timeoutMs)
|
||||
: null;
|
||||
timeout?.unref();
|
||||
|
||||
function clearHeartbeat() {
|
||||
clearInterval(heartbeat);
|
||||
}
|
||||
|
||||
function clearTimers() {
|
||||
clearHeartbeat();
|
||||
if (timeout) clearTimeout(timeout);
|
||||
if (forceKillTimer) clearTimeout(forceKillTimer);
|
||||
}
|
||||
|
||||
function forwardSignal(signal) {
|
||||
clearHeartbeat();
|
||||
|
||||
@@ -86,7 +108,7 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
||||
}
|
||||
|
||||
process.on("exit", () => {
|
||||
clearHeartbeat();
|
||||
clearTimers();
|
||||
try {
|
||||
process.kill(-child.pid, "SIGTERM");
|
||||
} catch (error) {
|
||||
@@ -101,13 +123,16 @@ process.on("exit", () => {
|
||||
});
|
||||
|
||||
child.on("error", (error) => {
|
||||
clearHeartbeat();
|
||||
clearTimers();
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
child.on("close", (code, signal) => {
|
||||
clearHeartbeat();
|
||||
clearTimers();
|
||||
if (timeoutExitCode !== null) {
|
||||
process.exit(timeoutExitCode);
|
||||
}
|
||||
if (signal) {
|
||||
process.kill(process.pid, signal);
|
||||
return;
|
||||
|
||||
@@ -209,9 +209,7 @@ const batchedQualityAppComponentTestsA = batchedQualityAppComponentTests.slice(0
|
||||
const batchedQualityAppComponentTestsB = batchedQualityAppComponentTests.slice(batchedQualityAppSplitIndex);
|
||||
|
||||
function buildComponentQualityInclude(testNames: readonly string[]): string[] {
|
||||
return testNames.length > 0
|
||||
? [`app/components/__tests__/{${testNames.join(",")}}.test.{ts,tsx}`]
|
||||
: [];
|
||||
return testNames.map((testName) => `app/components/__tests__/${testName}.test.{ts,tsx}`);
|
||||
}
|
||||
|
||||
const qualityAppTests = [
|
||||
|
||||
Reference in New Issue
Block a user