fix: report engine available when another fusion process owns it

The dashboard's engine-availability health check only counted engines
this process started. A second launch (e.g. `pnpm dev dashboard`
alongside an already-running `fusion`) is correctly refused the
per-machine engine singleton lock, so its engine map stays empty and
the dashboard showed a false "engine not running" banner even though an
engine was live on the machine.

ProjectEngineManager now records projects whose singleton lock is held
by another process (via EngineAlreadyRunningError) and exposes
hasRunningEngine(), which the health endpoint consults so the banner
reflects machine-level truth. Reconciliation still retries so this
process takes over if the other exits, and the "refusing to start" log
fires once per project instead of on every 30s reconciliation tick.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-06-21 03:09:35 -07:00
parent 4f9c4cf10d
commit 7635ba8682
5 changed files with 166 additions and 5 deletions

View File

@@ -0,0 +1,5 @@
---
"@runfusion/fusion": patch
---
Fix a false "engine not running" banner when another fusion process on the same machine already owns the engine. The dashboard's health check only counted engines this process started, so a second launch (e.g. `pnpm dev dashboard` alongside an already-running `fusion`) that was correctly refused the per-machine engine singleton lock reported the engine as unavailable — even though one was running. The `ProjectEngineManager` now tracks engines owned by another process (detected via `EngineAlreadyRunningError` from the singleton lock) and exposes `hasRunningEngine()`, which the dashboard health endpoint uses so the banner reflects machine-level truth. Reconciliation still retries so this process takes over if the other exits, and the "refusing to start" log is emitted once per project instead of on every reconciliation tick.

View File

@@ -415,6 +415,25 @@ describe("createServer health and headless mode", () => {
expect(res.body.engine).toEqual({ available: true });
});
it("reports the engine available when another fusion process owns it", async () => {
// The manager could not acquire the singleton lock (engine owned by another
// process on this machine) so it owns no engine instances, but it knows one
// is running. The banner must stay hidden.
const store = createMockStore();
const app = createServer(store, {
engineManager: {
getAllEngines: vi.fn().mockReturnValue(new Map()),
hasRunningEngine: vi.fn().mockReturnValue(true),
getEngine: vi.fn(),
} as any,
});
const res = await GET(app, "/api/health");
expect(res.status).toBe(200);
expect(res.body.engine).toEqual({ available: true });
});
it("reports degraded status when database corruption is detected", async () => {
const store = createMockStore({
getDatabaseHealth: vi.fn().mockReturnValue({

View File

@@ -479,8 +479,19 @@ export interface ServerOptions {
}
function hasDashboardEngine(options?: ServerOptions): boolean {
const engines = options?.engineManager?.getAllEngines?.();
return Boolean(options?.engine || (engines && engines.size > 0));
if (options?.engine) return true;
const manager = options?.engineManager;
if (!manager) return false;
// `hasRunningEngine` counts engines owned by this process AND engines owned
// by another fusion process on the machine (detected via the singleton
// lock). Without the latter, a UI-only launch alongside an already-running
// engine would show a false "engine not running" banner. Fall back to the
// owned-engine map for older manager instances / test doubles.
if (typeof manager.hasRunningEngine === "function") {
return manager.hasRunningEngine();
}
const engines = manager.getAllEngines?.();
return Boolean(engines && engines.size > 0);
}
type DashboardExpressApp = ReturnType<typeof express> & {

View File

@@ -37,6 +37,10 @@ vi.mock("../project-engine.js", () => {
import { ProjectEngineManager } from "../project-engine-manager.js";
import { ProjectEngine } from "../project-engine.js";
import {
acquireEngineSingleton,
EngineAlreadyRunningError,
} from "../engine-singleton-lock.js";
import type { RegisteredProject, CentralCore } from "@fusion/core";
function createMockCentralCore(projects: RegisteredProject[]): CentralCore {
@@ -702,4 +706,92 @@ describe("ProjectEngineManager", () => {
await manager.stopAll();
});
});
describe("engine owned by another fusion process", () => {
// These tests override the singleton-lock mock with persistent rejections.
// Restore the default "acquire succeeds" behaviour after each so later
// tests/describes are unaffected even if an assertion throws mid-test.
afterEach(() => {
const acquire = acquireEngineSingleton as ReturnType<typeof vi.fn>;
acquire.mockReset();
acquire.mockResolvedValue({
release: vi.fn().mockResolvedValue(undefined),
socketPath: "/tmp/test.sock",
lockFilePath: "/tmp/test.lock",
});
});
it("reports a running engine when the singleton lock is held elsewhere", async () => {
const manager = new ProjectEngineManager(centralCore);
// Another fusion process on this machine already owns the engine: the
// singleton lock cannot be acquired.
(acquireEngineSingleton as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
new EngineAlreadyRunningError("proj_aaa", "socket"),
);
await expect(manager.ensureEngine("proj_aaa")).rejects.toBeInstanceOf(
EngineAlreadyRunningError,
);
// We do not own an engine instance...
expect(manager.getEngine("proj_aaa")).toBeUndefined();
expect(manager.getAllEngines().size).toBe(0);
// ...but an engine IS running on the machine, so the dashboard must not
// show an "engine not running" banner.
expect(manager.hasRunningEngine()).toBe(true);
expect(manager.getExternalEngineIds().has("proj_aaa")).toBe(true);
});
it("logs the refusal only once across repeated reconciliation attempts", async () => {
const manager = new ProjectEngineManager(centralCore);
const acquire = acquireEngineSingleton as ReturnType<typeof vi.fn>;
acquire.mockRejectedValue(
new EngineAlreadyRunningError("proj_aaa", "socket"),
);
const warnSpy = vi.spyOn(
(await import("../logger.js")).runtimeLog,
"warn",
);
await expect(manager.ensureEngine("proj_aaa")).rejects.toBeInstanceOf(
EngineAlreadyRunningError,
);
await expect(manager.ensureEngine("proj_aaa")).rejects.toBeInstanceOf(
EngineAlreadyRunningError,
);
const refusals = warnSpy.mock.calls.filter(([msg]) =>
String(msg).includes("Refusing to start engine for proj_aaa"),
);
expect(refusals).toHaveLength(1);
warnSpy.mockRestore();
});
it("takes over and clears the external marker once the lock is free", async () => {
const manager = new ProjectEngineManager(centralCore);
const acquire = acquireEngineSingleton as ReturnType<typeof vi.fn>;
acquire.mockRejectedValueOnce(
new EngineAlreadyRunningError("proj_aaa", "socket"),
);
await expect(manager.ensureEngine("proj_aaa")).rejects.toBeInstanceOf(
EngineAlreadyRunningError,
);
expect(manager.getExternalEngineIds().has("proj_aaa")).toBe(true);
// The other process exits; the next attempt acquires the lock (the
// default mock resolves) and we take ownership.
const engine = await manager.ensureEngine("proj_aaa");
expect(engine).toBeDefined();
expect(manager.getEngine("proj_aaa")).toBeDefined();
expect(manager.getExternalEngineIds().has("proj_aaa")).toBe(false);
expect(manager.hasRunningEngine()).toBe(true);
});
});
});

View File

@@ -51,6 +51,15 @@ export class ProjectEngineManager {
private engines = new Map<string, ProjectEngine>();
private starting = new Map<string, Promise<ProjectEngine>>();
private singletonLocks = new Map<string, EngineSingletonLock>();
/**
* Projects whose engine is owned by ANOTHER fusion process on this machine.
* Populated when `acquireEngineSingleton` rejects with
* {@link EngineAlreadyRunningError} — that error is positive proof an engine
* is live for the project, just not owned by us. We keep retrying to start
* (so we take over if the other process dies), but the dashboard must report
* the engine as available rather than showing an "engine not running" banner.
*/
private externalEngines = new Set<string>();
private stopped = false;
/**
@@ -110,6 +119,21 @@ export class ProjectEngineManager {
return this.engines;
}
/**
* Whether an engine is running for any project on this machine — including
* engines owned by another fusion process (detected via the singleton lock).
* Drives the dashboard's "engine available" health so a UI-only launch
* alongside an already-running engine does not show a false banner.
*/
hasRunningEngine(): boolean {
return this.engines.size > 0 || this.externalEngines.size > 0;
}
/** Project ids whose engine is owned by another fusion process on this machine. */
getExternalEngineIds(): ReadonlySet<string> {
return this.externalEngines;
}
/** Get the TaskStore for a project from its engine. */
getStore(projectId: string): TaskStore | undefined {
return this.engines.get(projectId)?.getTaskStore();
@@ -262,6 +286,7 @@ export class ProjectEngineManager {
await Promise.all(stops);
this.engines.clear();
this.starting.clear();
this.externalEngines.clear();
// Release all singleton locks so another fusion process can take over.
const releases = Array.from(this.singletonLocks.values()).map((lock) =>
@@ -427,9 +452,16 @@ export class ProjectEngineManager {
},
).catch((err) => {
if (err instanceof EngineAlreadyRunningError) {
runtimeLog.warn(
`Refusing to start engine for ${projectId}: ${err.message}`,
);
// An engine IS running for this project — another fusion process owns
// it. Record it so the dashboard reports the engine as available, and
// log only on the first detection to avoid spamming every 30s
// reconciliation tick while the other process stays alive.
if (!this.externalEngines.has(projectId)) {
runtimeLog.warn(
`Refusing to start engine for ${projectId}: ${err.message}`,
);
}
this.externalEngines.add(projectId);
}
throw err;
});
@@ -452,6 +484,8 @@ export class ProjectEngineManager {
this.engines.set(projectId, engine);
this.starting.delete(projectId);
// We now own the engine — clear any prior "owned by another process" marker.
this.externalEngines.delete(projectId);
runtimeLog.log(
`Started engine for ${project.name ?? projectId} (${projectId})`,
);