feat(FN-1607): add process lifecycle diagnostics and fix resource leaks

- Add process lifecycle diagnostics for dashboard and serve commands
- Add SQLite database health check to diagnostics endpoint
- Add store listener count diagnostics for debugging subscription leaks
- Audit and fix SSE connection management to prevent connection leaks
- Audit and fix timer/interval cleanup in engine and CLI shutdown handlers
- Fix res.on() call guard for test mocks compatibility
- Fix variable declaration ordering in serve.ts
- Update memory with diagnostic findings for future debugging

packages/cli/src/commands/dashboard.ts

@@ -16,22 +16,173 @@ export { promptForPort };
 type LoginCallbacks = Parameters<AuthStorage["login"]>[1];
 
 let processDiagnosticsRegistered = false;
+let diagnosticIntervalHandle: ReturnType<typeof setInterval> | null = null;
+const DIAGNOSTIC_INTERVAL_MS = 30 * 60 * 1000; // 30 minutes
+let diagnosticStartTime = 0;
+let diagnosticDbHealthCheck: (() => boolean) | null = null;
+let diagnosticStoreListenerCheck: (() => Record<string, number>) | null = null;
 
+/**
+ * Format bytes to human-readable string
+ */
+function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes}B`;
+  if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)}KB`;
+  if (bytes < 1024 * 1024 * 1024) return `${Math.round(bytes / (1024 * 1024))}MB`;
+  return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
+}
+
+/**
+ * Format milliseconds to human-readable uptime string
+ */
+function formatUptime(ms: number): string {
+  const seconds = Math.floor(ms / 1000);
+  const minutes = Math.floor(seconds / 60);
+  const hours = Math.floor(minutes / 60);
+  const days = Math.floor(hours / 24);
+
+  if (days > 0) return `${days}d${hours % 24}h`;
+  if (hours > 0) return `${hours}h${minutes % 60}m`;
+  if (minutes > 0) return `${minutes}m${seconds % 60}s`;
+  return `${seconds}s`;
+}
+
+/**
+ * Get and log current process diagnostics (memory, handles, requests)
+ * @param prefix - Log prefix (e.g., "dashboard", "serve")
+ * @param startTime - Process start timestamp
+ * @param dbHealthCheck - Optional function to check database health
+ */
+function logDiagnostics(prefix: string, startTime: number, dbHealthCheck?: () => boolean): void {
+  const mem = process.memoryUsage();
+  const uptime = Date.now() - startTime;
+
+  // Get active handles/requests if available (Node.js internal)
+  let handleCount = -1;
+  let requestCount = -1;
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    handleCount = (process as any)._getActiveHandles?.()?.length ?? -1;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    requestCount = (process as any)._getActiveRequests?.()?.length ?? -1;
+  } catch {
+    // Ignore errors if these internal APIs are not available
+  }
+
+  // Check database health if provided
+  let dbHealth = "unknown";
+  if (dbHealthCheck) {
+    try {
+      dbHealth = dbHealthCheck() ? "ok" : "failed";
+    } catch {
+      dbHealth = "error";
+    }
+  }
+
+  // Get listener counts if provided
+  let listenerInfo = "";
+  if (diagnosticStoreListenerCheck) {
+    try {
+      const counts = diagnosticStoreListenerCheck();
+      const listenerEntries = Object.entries(counts)
+        .map(([event, count]) => `${event}:${count}`)
+        .join(",");
+      listenerInfo = ` listeners=${listenerEntries}`;
+    } catch {
+      // Ignore errors getting listener counts
+    }
+  }
+
+  const logLine = `[${prefix}] diagnostics: uptime=${formatUptime(uptime)} ` +
+    `rss=${formatBytes(mem.rss)} heap=${formatBytes(mem.heapUsed)}/${formatBytes(mem.heapTotal)} ` +
+    `external=${formatBytes(mem.external)} arrayBuffers=${formatBytes(mem.arrayBuffers)} ` +
+    `handles=${handleCount} requests=${requestCount} db=${dbHealth}${listenerInfo}`;
+
+  console.log(logLine);
+}
+
+/**
+ * Register process lifecycle diagnostics for long-running process monitoring.
+ * Logs memory usage, handle counts, and uptime at startup and every 30 minutes.
+ * Also logs beforeExit and exit events for shutdown analysis.
+ */
 function ensureProcessDiagnostics(): void {
   if (processDiagnosticsRegistered) {
     return;
   }
   processDiagnosticsRegistered = true;
 
+  diagnosticStartTime = Date.now();
+
+  // Log initial diagnostics at startup (before store is created)
+  logDiagnostics("dashboard", diagnosticStartTime);
+
+  // Register periodic diagnostics every 30 minutes
+  diagnosticIntervalHandle = setInterval(() => {
+    logDiagnostics("dashboard", diagnosticStartTime, diagnosticDbHealthCheck ?? undefined);
+  }, DIAGNOSTIC_INTERVAL_MS);
+  diagnosticIntervalHandle.unref?.(); // Don't prevent process exit
+
+  // Log beforeExit when event loop drains naturally
+  process.on("beforeExit", (code: number) => {
+    const uptime = Date.now() - diagnosticStartTime;
+    let handleCount = -1;
+    let requestCount = -1;
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      handleCount = (process as any)._getActiveHandles?.()?.length ?? -1;
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      requestCount = (process as any)._getActiveRequests?.()?.length ?? -1;
+    } catch {
+      // Ignore
+    }
+    console.log(`[dashboard] beforeExit code=${code} uptime=${formatUptime(uptime)} handles=${handleCount} requests=${requestCount}`);
+  });
+
+  // Log exit event with exit code and uptime
+  process.on("exit", (code: number) => {
+    const uptime = Date.now() - diagnosticStartTime;
+    console.log(`[dashboard] exit code=${code} uptime=${formatUptime(uptime)}`);
+  });
+
+  // Log uncaught exceptions
   process.on("uncaughtExceptionMonitor", (error: Error) => {
     console.error(`[dashboard] uncaught exception pid=${process.pid}: ${error.stack || error.message}`);
   });
+
+  // Log unhandled rejections
   process.on("unhandledRejection", (reason: unknown) => {
     const message = reason instanceof Error ? reason.stack || reason.message : String(reason);
     console.error(`[dashboard] unhandled rejection pid=${process.pid}: ${message}`);
   });
 }
 
+/**
+ * Stop the diagnostic interval timer. Call during shutdown.
+ */
+function stopDiagnosticInterval(): void {
+  if (diagnosticIntervalHandle) {
+    clearInterval(diagnosticIntervalHandle);
+    diagnosticIntervalHandle = null;
+  }
+}
+
+/**
+ * Set the database health check function for diagnostics.
+ * Call this after the TaskStore is created.
+ */
+function setDiagnosticDbHealthCheck(check: () => boolean): void {
+  diagnosticDbHealthCheck = check;
+}
+
+/**
+ * Set the store listener count check function for diagnostics.
+ * Call this after the TaskStore is created.
+ */
+function setDiagnosticStoreListenerCheck(check: () => Record<string, number>): void {
+  diagnosticStoreListenerCheck = check;
+}
+
 interface DashboardAuthStorage {
   reload(): void;
   getOAuthProviders(): Array<{ id: string; name: string }>;
@@ -127,6 +278,19 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
   await store.init();
   await store.watch();
 
+  // Set up database health check for diagnostics
+  setDiagnosticDbHealthCheck(() => store.healthCheck());
+
+  // Set up store listener count check for diagnostics
+  setDiagnosticStoreListenerCheck(() => ({
+    "task:created": store.listenerCount("task:created"),
+    "task:moved": store.listenerCount("task:moved"),
+    "task:updated": store.listenerCount("task:updated"),
+    "task:deleted": store.listenerCount("task:deleted"),
+    "settings:updated": store.listenerCount("settings:updated"),
+    "agent:log": store.listenerCount("agent:log"),
+  }));
+
   const handlers: Array<{
     target: NodeJS.EventEmitter;
     event: string | symbol;
@@ -1155,8 +1319,29 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
     const shutdown = async (signal: NodeJS.Signals) => {
       if (shutdownInProgress) return;
       shutdownInProgress = true;
+
+      // Log active handles at shutdown for diagnostics
+      const handleTypes: Record<string, number> = {};
+      try {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const handles = (process as any)._getActiveHandles?.() ?? [];
+        for (const handle of handles) {
+          const type = handle.constructor?.name ?? "unknown";
+          handleTypes[type] = (handleTypes[type] ?? 0) + 1;
+        }
+        const handleSummary = Object.entries(handleTypes)
+          .sort((a, b) => b[1] - a[1])
+          .map(([type, count]) => `${type}:${count}`)
+          .join(", ");
+        console.log(`[dashboard] active handles at shutdown: ${handleSummary}`);
+      } catch {
+        // Ignore errors getting handle types
+      }
+
       await logShutdownDiagnostics(signal);
       dispose();
+      stopDiagnosticInterval();
+
       // Stop heartbeat components first (they reference agentStore)
       if (triggerScheduler) triggerScheduler.stop();
       if (heartbeatMonitor) heartbeatMonitor.stop();
@@ -1180,8 +1365,28 @@ export async function runDashboard(port: number, opts: { paused?: boolean; dev?:
     const devShutdown = async (signal: NodeJS.Signals) => {
       if (shutdownInProgress) return;
       shutdownInProgress = true;
+
+      // Log active handles at shutdown for diagnostics
+      const handleTypes: Record<string, number> = {};
+      try {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const handles = (process as any)._getActiveHandles?.() ?? [];
+        for (const handle of handles) {
+          const type = handle.constructor?.name ?? "unknown";
+          handleTypes[type] = (handleTypes[type] ?? 0) + 1;
+        }
+        const handleSummary = Object.entries(handleTypes)
+          .sort((a, b) => b[1] - a[1])
+          .map(([type, count]) => `${type}:${count}`)
+          .join(", ");
+        console.log(`[dashboard] active handles at shutdown: ${handleSummary}`);
+      } catch {
+        // Ignore errors getting handle types
+      }
+
       await logShutdownDiagnostics(signal);
       dispose();
+      stopDiagnosticInterval();
       if (triggerScheduler) triggerScheduler.stop();
       if (heartbeatMonitor) heartbeatMonitor.stop();
       notifier.stop();

packages/cli/src/commands/serve.ts

@@ -63,10 +63,160 @@ import {
 } from "./task-lifecycle.js";
 import { promptForPort } from "./port-prompt.js";
 
+const DIAGNOSTIC_INTERVAL_MS = 30 * 60 * 1000; // 30 minutes
+let diagnosticIntervalHandle: ReturnType<typeof setInterval> | null = null;
+let serveStartTime = 0;
+let serveDbHealthCheck: (() => boolean) | null = null;
+
+/**
+ * Format bytes to human-readable string
+ */
+function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes}B`;
+  if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)}KB`;
+  if (bytes < 1024 * 1024 * 1024) return `${Math.round(bytes / (1024 * 1024))}MB`;
+  return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
+}
+
+/**
+ * Format milliseconds to human-readable uptime string
+ */
+function formatUptime(ms: number): string {
+  const seconds = Math.floor(ms / 1000);
+  const minutes = Math.floor(seconds / 60);
+  const hours = Math.floor(minutes / 60);
+  const days = Math.floor(hours / 24);
+
+  if (days > 0) return `${days}d${hours % 24}h`;
+  if (hours > 0) return `${hours}h${minutes % 60}m`;
+  if (minutes > 0) return `${minutes}m${seconds % 60}s`;
+  return `${seconds}s`;
+}
+
+/**
+ * Get and log current process diagnostics (memory, handles, requests)
+ * @param prefix - Log prefix (e.g., "dashboard", "serve")
+ * @param dbHealthCheck - Optional function to check database health
+ */
+function logDiagnostics(prefix: string, dbHealthCheck?: () => boolean): void {
+  const mem = process.memoryUsage();
+  const uptime = Date.now() - serveStartTime;
+
+  // Get active handles/requests if available (Node.js internal)
+  let handleCount = -1;
+  let requestCount = -1;
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    handleCount = (process as any)._getActiveHandles?.()?.length ?? -1;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    requestCount = (process as any)._getActiveRequests?.()?.length ?? -1;
+  } catch {
+    // Ignore errors if these internal APIs are not available
+  }
+
+  // Check database health if provided
+  let dbHealth = "unknown";
+  if (dbHealthCheck) {
+    try {
+      dbHealth = dbHealthCheck() ? "ok" : "failed";
+    } catch {
+      dbHealth = "error";
+    }
+  }
+
+  // Get listener counts if provided
+  let listenerInfo = "";
+  if (serveDbHealthCheck) {
+    try {
+      // This would be for store listener counts - not applicable in serve without store
+      listenerInfo = "";
+    } catch {
+      // Ignore errors getting listener counts
+    }
+  }
+
+  const logLine = `[${prefix}] diagnostics: uptime=${formatUptime(uptime)} ` +
+    `rss=${formatBytes(mem.rss)} heap=${formatBytes(mem.heapUsed)}/${formatBytes(mem.heapTotal)} ` +
+    `external=${formatBytes(mem.external)} arrayBuffers=${formatBytes(mem.arrayBuffers)} ` +
+    `handles=${handleCount} requests=${requestCount} db=${dbHealth}${listenerInfo}`;
+
+  console.log(logLine);
+}
+
+/**
+ * Stop the diagnostic interval timer. Call during shutdown.
+ */
+function stopDiagnosticInterval(): void {
+  if (diagnosticIntervalHandle) {
+    clearInterval(diagnosticIntervalHandle);
+    diagnosticIntervalHandle = null;
+  }
+}
+
+/**
+ * Set the database health check function for diagnostics.
+ * Call this after the TaskStore is created.
+ */
+function setServeDbHealthCheck(check: () => boolean): void {
+  serveDbHealthCheck = check;
+}
+
+/**
+ * Register process lifecycle diagnostics for long-running process monitoring.
+ * Logs memory usage, handle counts, and uptime at startup and every 30 minutes.
+ * Also logs beforeExit and exit events for shutdown analysis.
+ */
+function ensureProcessDiagnostics(): void {
+  // Log initial diagnostics at startup (before store is created)
+  logDiagnostics("serve");
+
+  // Register periodic diagnostics every 30 minutes
+  diagnosticIntervalHandle = setInterval(() => {
+    logDiagnostics("serve", serveDbHealthCheck ?? undefined);
+  }, DIAGNOSTIC_INTERVAL_MS);
+  diagnosticIntervalHandle.unref?.(); // Don't prevent process exit
+
+  // Log beforeExit when event loop drains naturally
+  process.on("beforeExit", (code: number) => {
+    const uptime = Date.now() - serveStartTime;
+    let handleCount = -1;
+    let requestCount = -1;
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      handleCount = (process as any)._getActiveHandles?.()?.length ?? -1;
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      requestCount = (process as any)._getActiveRequests?.()?.length ?? -1;
+    } catch {
+      // Ignore
+    }
+    console.log(`[serve] beforeExit code=${code} uptime=${formatUptime(uptime)} handles=${handleCount} requests=${requestCount}`);
+  });
+
+  // Log exit event with exit code and uptime
+  process.on("exit", (code: number) => {
+    const uptime = Date.now() - serveStartTime;
+    console.log(`[serve] exit code=${code} uptime=${formatUptime(uptime)}`);
+  });
+
+  // Log uncaught exceptions
+  process.on("uncaughtExceptionMonitor", (error: Error) => {
+    console.error(`[serve] uncaught exception pid=${process.pid}: ${error.stack || error.message}`);
+  });
+
+  // Log unhandled rejections
+  process.on("unhandledRejection", (reason: unknown) => {
+    const message = reason instanceof Error ? reason.stack || reason.message : String(reason);
+    console.error(`[serve] unhandled rejection pid=${process.pid}: ${message}`);
+  });
+}
+
 export async function runServe(
   port: number,
   opts: { interactive?: boolean; paused?: boolean; host?: string } = {},
 ) {
+  serveStartTime = Date.now();
+  ensureProcessDiagnostics();
+
   let selectedPort = port;
   if (opts.interactive) {
     try {
@@ -87,6 +237,9 @@ export async function runServe(
   await store.init();
   await store.watch();
 
+  // Set up database health check for diagnostics
+  setServeDbHealthCheck(() => store.healthCheck());
+
   const automationStore = new AutomationStore(cwd);
   await automationStore.init();
 
@@ -907,11 +1060,14 @@ export async function runServe(
     }
   });
 
+  let shuttingDown = false;
   let mergeRetryTimer: ReturnType<typeof setTimeout> | null = null;
   async function scheduleMergeRetry(): Promise<void> {
+    if (shuttingDown) return;
     const currentSettings = await store.getSettings().catch(() => settings);
     const interval = currentSettings.pollIntervalMs ?? 15_000;
     mergeRetryTimer = setTimeout(async () => {
+      if (shuttingDown) return;
       try {
         const s = await store.getSettings();
         cachedMaxConcurrent = s.maxConcurrent;
@@ -926,7 +1082,9 @@ export async function runServe(
       } catch {
         // ignore errors in periodic sweep
       }
-      void scheduleMergeRetry();
+      if (!shuttingDown) {
+        void scheduleMergeRetry();
+      }
     }, interval);
   }
   void scheduleMergeRetry();
@@ -968,11 +1126,28 @@ export async function runServe(
   console.log(`  Press Ctrl+C to stop`);
   console.log();
 
-  let shuttingDown = false;
   const shutdown = async () => {
     if (shuttingDown) return;
     shuttingDown = true;
 
+    // Log active handles at shutdown for diagnostics
+    const handleTypes: Record<string, number> = {};
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const handles = (process as any)._getActiveHandles?.() ?? [];
+      for (const handle of handles) {
+        const type = handle.constructor?.name ?? "unknown";
+        handleTypes[type] = (handleTypes[type] ?? 0) + 1;
+      }
+      const handleSummary = Object.entries(handleTypes)
+        .sort((a, b) => b[1] - a[1])
+        .map(([type, count]) => `${type}:${count}`)
+        .join(", ");
+      console.log(`[serve] active handles at shutdown: ${handleSummary}`);
+    } catch {
+      // Ignore errors getting handle types
+    }
+
     // Stop heartbeat components first (they reference agentStore)
     if (triggerScheduler) triggerScheduler.stop();
     if (heartbeatMonitor) heartbeatMonitor.stop();
@@ -1013,6 +1188,7 @@ export async function runServe(
       // best-effort
     }
 
+    stopDiagnosticInterval();
     store.close();
     process.exit(0);
   };

packages/core/src/store.ts

@@ -4206,6 +4206,21 @@ ${stepsSection}`;
     return this.db;
   }
 
+  /**
+   * Perform a simple database health check.
+   * Returns true if the database responds correctly, false otherwise.
+   * Used for periodic health diagnostics.
+   */
+  healthCheck(): boolean {
+    try {
+      // Simple query to verify database responsiveness
+      this.db.prepare("SELECT 1").get();
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
   private generateSpecifiedPrompt(task: Task): string {
     const deps =
       task.dependencies.length > 0

packages/dashboard/src/sse.ts

@@ -3,12 +3,18 @@ import type { TaskStore, MissionStore, PluginStore, PluginInstallation, PluginSt
 import type { AiSessionStore } from "./ai-session-store.js";
 
 let activeConnections = 0;
+let highWaterMark = 0;
 
 /** Returns the current number of active SSE connections. */
 export function getActiveSSEConnections(): number {
   return activeConnections;
 }
 
+/** Returns the high water mark of SSE connections. */
+export function getSSEHighWaterMark(): number {
+  return highWaterMark;
+}
+
 /**
  * Safely write to an SSE response stream.
  * Returns `true` if the write succeeded, `false` if the connection is dead.
@@ -177,6 +183,11 @@ export function createSSE(
     res.flushHeaders();
 
     activeConnections++;
+    // Track high water mark and log when new highs are reached
+    if (activeConnections > highWaterMark) {
+      highWaterMark = activeConnections;
+      console.log(`[sse] active connections: ${activeConnections} (high water mark: ${highWaterMark})`);
+    }
 
     // Send initial heartbeat
     res.write(": connected\n\n");
@@ -415,6 +426,15 @@ export function createSSE(
       send("event: heartbeat\ndata: \n\n");
     }, 30_000);
 
+    // Register cleanup on request close (primary path for HTTP/1.1)
     _req.on("close", cleanup);
+
+    // Also register on response close as a safety net for edge cases
+    // (e.g., proxy timeouts, HTTP/2 stream resets). This ensures cleanup
+    // fires even if the request object doesn't emit "close".
+    // Guard with typeof check for test mocks that may not have on method.
+    if (typeof res.on === "function") {
+      res.on("close", cleanup);
+    }
   };
 }