feat(FN-2523): add safe restart restore lifecycle diagnostics

- Add ProjectEngine restore lifecycle core to perform safe restarts and surface detailed restore state transitions
- Expose restore diagnostics through remote-access status types and settings/memory route context, including legacy API mapping updates
- Add comprehensive regression coverage for restore lifecycle behavior in engine and dashboard headless remote-access tests
- Document the restore lifecycle contract in architecture/settings docs and include a patch changeset for @runfusion/fusion
This commit is contained in:
Fusion
2026-04-26 05:18:12 -07:00
committed by gsxdsm
parent 2ce8b1c249
commit dd81a12e77
12 changed files with 956 additions and 19 deletions

View File

@@ -425,9 +425,17 @@ export interface RemoteSettings {
export interface RemoteStatus {
provider: "tailscale" | "cloudflare" | null;
state: "stopped" | "starting" | "running" | "error";
state: "stopped" | "starting" | "running" | "stopping" | "failed";
url: string | null;
lastError: string | null;
lastErrorCode?: string | null;
restore?: {
outcome: "applied" | "skipped" | "failed";
reason: string;
at: string;
provider: "tailscale" | "cloudflare" | null;
message?: string;
};
}
export function fetchRemoteSettings(projectId?: string): Promise<{ settings: RemoteSettings }> {

View File

@@ -0,0 +1,217 @@
// @vitest-environment node
import { describe, expect, it, vi } from "vitest";
import type { TaskStore } from "@fusion/core";
import { createServer } from "../server.js";
import { get as performGet, request as performRequest } from "../test-request.js";
function createMockStore(overrides: Partial<TaskStore> = {}): TaskStore {
return {
getTask: vi.fn(),
listTasks: vi.fn().mockResolvedValue([]),
createTask: vi.fn(),
moveTask: vi.fn(),
updateTask: vi.fn(),
deleteTask: vi.fn(),
mergeTask: vi.fn(),
archiveTask: vi.fn(),
unarchiveTask: vi.fn(),
getSettings: vi.fn().mockResolvedValue({}),
updateSettings: vi.fn(),
logEntry: vi.fn().mockResolvedValue(undefined),
getAgentLogs: vi.fn().mockResolvedValue([]),
addSteeringComment: vi.fn(),
updatePrInfo: vi.fn().mockResolvedValue(undefined),
updateIssueInfo: vi.fn().mockResolvedValue(undefined),
getRootDir: vi.fn().mockReturnValue("/fake/root"),
getFusionDir: vi.fn().mockReturnValue("/fake/root/.fusion"),
getDatabase: vi.fn().mockReturnValue({
exec: vi.fn(),
prepare: vi.fn().mockReturnValue({ run: vi.fn().mockReturnValue({ changes: 0 }), get: vi.fn(), all: vi.fn().mockReturnValue([]) }),
}),
getMissionStore: vi.fn().mockReturnValue({
listMissions: vi.fn().mockReturnValue([]),
createMission: vi.fn(),
getMissionWithHierarchy: vi.fn(),
updateMission: vi.fn(),
getMission: vi.fn(),
deleteMission: vi.fn(),
listMilestonesByMission: vi.fn().mockReturnValue([]),
createMilestone: vi.fn(),
updateMilestone: vi.fn(),
getMilestone: vi.fn(),
deleteMilestone: vi.fn(),
listTasksByMilestone: vi.fn().mockReturnValue([]),
createMissionTask: vi.fn(),
updateMissionTask: vi.fn(),
getMissionTask: vi.fn(),
deleteMissionTask: vi.fn(),
}),
on: vi.fn(),
off: vi.fn(),
...overrides,
} as unknown as TaskStore;
}
async function GET(app: ReturnType<typeof createServer>, path: string) {
return performGet(app, path);
}
async function REQUEST(
app: ReturnType<typeof createServer>,
method: string,
path: string,
body?: unknown,
) {
return performRequest(
app,
method,
path,
body === undefined ? undefined : JSON.stringify(body),
body === undefined ? {} : { "Content-Type": "application/json" },
);
}
describe("remote access headless parity", () => {
function buildRemoteAccessSettings() {
return {
enabled: true,
activeProvider: "cloudflare" as const,
providers: {
tailscale: {
enabled: false,
hostname: "tail.example.ts.net",
targetPort: 4040,
acceptRoutes: false,
},
cloudflare: {
enabled: true,
tunnelName: "demo",
tunnelToken: "cf-secret-token",
ingressUrl: "https://demo.example.com",
},
},
tokenStrategy: {
persistent: {
enabled: true,
token: "frt_persistent_token",
},
shortLived: {
enabled: true,
ttlMs: 120000,
maxTtlMs: 86400000,
},
},
lifecycle: {
rememberLastRunning: true,
wasRunningOnShutdown: true,
lastRunningProvider: "cloudflare" as const,
},
};
}
function buildServer(headless: boolean) {
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({
remoteAccess: buildRemoteAccessSettings(),
}),
});
const status = {
provider: "cloudflare" as const,
state: "running" as const,
pid: 12345,
startedAt: new Date().toISOString(),
stoppedAt: null,
url: "https://live.example.com",
lastError: null,
};
const engine = {
getTaskStore: vi.fn().mockReturnValue(store),
getAutomationStore: vi.fn(),
getRuntime: vi.fn().mockReturnValue({
getMissionAutopilot: vi.fn(),
getMissionExecutionLoop: vi.fn(),
}),
getHeartbeatMonitor: vi.fn(),
getWorkingDirectory: vi.fn().mockReturnValue("/fake/root"),
getRoutineStore: vi.fn(),
getRoutineRunner: vi.fn(),
onMerge: vi.fn(),
getRemoteTunnelManager: vi.fn().mockReturnValue({
getStatus: vi.fn().mockReturnValue(status),
}),
getRemoteTunnelRestoreDiagnostics: vi.fn().mockReturnValue({
outcome: "skipped",
reason: "runtime_prerequisite_missing",
at: new Date().toISOString(),
provider: "cloudflare",
}),
startRemoteTunnel: vi.fn().mockResolvedValue(status),
stopRemoteTunnel: vi.fn().mockResolvedValue({
...status,
provider: null,
state: "stopped",
pid: null,
stoppedAt: new Date().toISOString(),
url: null,
}),
};
return {
app: createServer(store, { headless, engine: engine as never }),
engine,
};
}
it("returns parity-compatible /api/remote/status payload between headless and non-headless", async () => {
const dashboard = buildServer(false);
const headless = buildServer(true);
const [dashboardRes, headlessRes] = await Promise.all([
GET(dashboard.app, "/api/remote/status"),
GET(headless.app, "/api/remote/status"),
]);
expect(dashboardRes.status).toBe(200);
expect(headlessRes.status).toBe(200);
expect(dashboardRes.body).toMatchObject({
provider: "cloudflare",
state: "running",
restore: {
outcome: "skipped",
reason: "runtime_prerequisite_missing",
provider: "cloudflare",
},
});
expect(headlessRes.body).toMatchObject({
provider: "cloudflare",
state: "running",
restore: {
outcome: "skipped",
reason: "runtime_prerequisite_missing",
provider: "cloudflare",
},
});
expect(JSON.stringify(dashboardRes.body)).not.toContain("cf-secret-token");
expect(JSON.stringify(headlessRes.body)).not.toContain("cf-secret-token");
});
it("uses engine lifecycle controls for /api/remote/tunnel/start and /api/remote/tunnel/stop", async () => {
const { app, engine } = buildServer(true);
const startRes = await REQUEST(app, "POST", "/api/remote/tunnel/start", {});
const stopRes = await REQUEST(app, "POST", "/api/remote/tunnel/stop", {});
expect(startRes.status).toBe(200);
expect(stopRes.status).toBe(200);
expect(engine.startRemoteTunnel).toHaveBeenCalledTimes(1);
expect(engine.stopRemoteTunnel).toHaveBeenCalledTimes(1);
expect(startRes.body).toMatchObject({ state: "running", provider: "cloudflare" });
expect(stopRes.body).toMatchObject({ state: "stopped", provider: null });
});
});

View File

@@ -96,6 +96,14 @@ export async function getProjectContext(
}
}
if (!projectId && options?.engine) {
try {
return { store: options.engine.getTaskStore(), engine: options.engine, projectId };
} catch {
// Fall back to scoped store resolution.
}
}
const scopedStore = await getScopedStore(req, store);
return { store: scopedStore, engine: undefined, projectId };
}

View File

@@ -303,13 +303,24 @@ export function registerSettingsMemoryRoutes(ctx: ApiRoutesContext, deps: Settin
router.get("/remote/status", async (req, res) => {
try {
const { store: scopedStore } = await getProjectContext(req);
const { store: scopedStore, engine } = await getProjectContext(req);
const settings = await scopedStore.getSettings();
const manager = engine?.getRemoteTunnelManager();
const tunnelStatus = manager?.getStatus();
const restore = engine?.getRemoteTunnelRestoreDiagnostics();
res.json({
provider: settings.remoteAccess?.activeProvider ?? null,
state: "stopped",
url: null,
lastError: null,
provider: tunnelStatus?.provider ?? settings.remoteAccess?.activeProvider ?? null,
state: tunnelStatus?.state ?? "stopped",
url: tunnelStatus?.url ?? null,
lastError: tunnelStatus?.lastError?.message ?? null,
lastErrorCode: tunnelStatus?.lastError?.code ?? null,
restore: restore ?? {
outcome: "skipped",
reason: "not_attempted",
at: new Date().toISOString(),
provider: null,
},
});
} catch (err: unknown) {
if (err instanceof ApiError) throw err;
@@ -345,25 +356,57 @@ export function registerSettingsMemoryRoutes(ctx: ApiRoutesContext, deps: Settin
router.post("/remote/tunnel/start", async (req, res) => {
try {
const { store: scopedStore } = await getProjectContext(req);
const { store: scopedStore, engine } = await getProjectContext(req);
const settings = await scopedStore.getSettings();
const provider = settings.remoteAccess?.activeProvider ?? null;
if (!provider) {
throw new ApiError(409, "No active provider configured", { code: "NO_ACTIVE_PROVIDER" });
}
res.json({ state: "starting", provider });
if (!engine) {
res.json({ state: "starting", provider });
return;
}
const status = await engine.startRemoteTunnel();
res.json({
state: status.state,
provider: status.provider,
url: status.url,
lastError: status.lastError?.message ?? null,
lastErrorCode: status.lastError?.code ?? null,
});
} catch (err: unknown) {
if (err instanceof ApiError) throw err;
const message = err instanceof Error ? err.message : String(err);
if (message.startsWith("invalid_config:") || message.startsWith("runtime_prerequisite_missing:")) {
throw new ApiError(409, message.split(":").slice(1).join(":") || "Remote tunnel prerequisites are not met", {
code: "REMOTE_TUNNEL_PREREQUISITE_MISSING",
});
}
rethrowAsApiError(err, "Failed to start remote tunnel");
}
});
router.post("/remote/tunnel/stop", async (req, res) => {
try {
const { store: scopedStore } = await getProjectContext(req);
const { store: scopedStore, engine } = await getProjectContext(req);
const settings = await scopedStore.getSettings();
const provider = settings.remoteAccess?.activeProvider ?? null;
res.json({ state: "stopped", provider });
if (!engine) {
res.json({ state: "stopped", provider });
return;
}
const status = await engine.stopRemoteTunnel();
res.json({
state: status.state,
provider: status.provider,
url: status.url,
lastError: status.lastError?.message ?? null,
lastErrorCode: status.lastError?.code ?? null,
});
} catch (err: unknown) {
if (err instanceof ApiError) throw err;
rethrowAsApiError(err, "Failed to stop remote tunnel");