diff --git a/.changeset/central-db-delete-journal-sigbus.md b/.changeset/central-db-delete-journal-sigbus.md new file mode 100644 index 0000000000..47b6a8f087 --- /dev/null +++ b/.changeset/central-db-delete-journal-sigbus.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fix random fusion crashes when multiple dashboards/CLIs run on one host. +category: fix +dev: Central DB (~/.fusion/fusion-central.db) now uses journal_mode=DELETE instead of WAL. WAL coordinates concurrent processes via a memory-mapped `-shm` wal-index that SIGBUSes a reader (walIndexReadHdr / `cluster_pagein past EOF`) on macOS/APFS when another process resizes it mid-checkpoint, killing the node process with no JS stack. DELETE mode removes the `-shm` mmap surface and coordinates via POSIX locks (busy_timeout absorbs the added writer serialization). Per-project DBs (db.ts) are unchanged. See central-db.ts open() and central-db.test.ts regression. diff --git a/packages/core/src/__tests__/central-db.test.ts b/packages/core/src/__tests__/central-db.test.ts index 40365341aa..042c22455a 100644 --- a/packages/core/src/__tests__/central-db.test.ts +++ b/packages/core/src/__tests__/central-db.test.ts @@ -1,8 +1,9 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, rmSync, statSync } from "node:fs"; +import { mkdtempSync, rmSync, statSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { CentralDatabase, createCentralDatabase, toJson, fromJson } from "../central-db.js"; +import { DatabaseSync } from "../sqlite-adapter.js"; describe("CentralDatabase", () => { let tempDir: string; @@ -42,32 +43,78 @@ describe("CentralDatabase", () => { expect(db.getSchemaVersion()).toBe(13); }); - it("should enable WAL mode and busy_timeout", () => { + it("should use DELETE (rollback-journal) mode and busy_timeout, not WAL", () => { db.init(); const journalMode = db.prepare("PRAGMA journal_mode").get() as { journal_mode: string }; const busyTimeout = db.prepare("PRAGMA busy_timeout").get() as Record; - expect(journalMode.journal_mode).toBe("wal"); + // Regression: the central DB must NOT run in WAL mode. WAL coordinates the + // many concurrent fusion processes through a memory-mapped `-shm` wal-index, + // which on macOS/APFS SIGBUSes a reader (walIndexReadHdr / `cluster_pagein + // past EOF`) when another process resizes it mid-checkpoint — observed 3× + // in 3 days (Jun 22–24 2026). DELETE mode removes the `-shm` mmap surface. + expect(journalMode.journal_mode).toBe("delete"); expect(Object.values(busyTimeout)[0]).toBe(5000); }); - it("should bound WAL growth and durability like the per-project DB", () => { + it("should never create a `-shm` wal-index file (the SIGBUS surface)", () => { db.init(); + // Drive real write traffic; under WAL this materializes `-shm` + `-wal`. + db.bumpLastModified(); + db.prepare("SELECT * FROM globalConcurrency WHERE id = 1").get(); + + const dbPath = db.getPath(); + // The wal-index shared-memory file is the exact thing that was memmap'd + // and faulted. Its absence proves the crashing surface is gone. + expect(existsSync(`${dbPath}-shm`)).toBe(false); + expect(existsSync(`${dbPath}-wal`)).toBe(false); const synchronous = db.prepare("PRAGMA synchronous").get() as { synchronous: number }; - const autoCheckpoint = db - .prepare("PRAGMA wal_autocheckpoint") - .get() as { wal_autocheckpoint: number }; - const journalSizeLimit = db - .prepare("PRAGMA journal_size_limit") - .get() as { journal_size_limit: number }; + expect(synchronous.synchronous).toBe(2); // FULL — durability posture preserved + }); - expect(synchronous.synchronous).toBe(2); // FULL - expect(autoCheckpoint.wal_autocheckpoint).toBe(1000); - // Previously unset (-1 / unbounded), which let the central WAL bloat and - // slow every reader. Now capped at 4 MB to match db.ts. - expect(journalSizeLimit.journal_size_limit).toBe(4_194_304); + it("warns (does not throw) when a WAL holder blocks the DELETE migration", () => { + // Migration-path regression: during a rolling upgrade an old-version process + // can still hold the central DB open in WAL mode. WAL→DELETE needs an exclusive + // lock it cannot get, so SQLite keeps WAL and the PRAGMA *returns* "wal" instead + // of throwing. The new connection must surface that loudly rather than silently + // run with the SIGBUS `-shm` surface still present. + const dbFile = join(tempDir, "fusion-central.db"); + const walHolder = new DatabaseSync(dbFile); + walHolder.exec("PRAGMA journal_mode = WAL"); + walHolder.exec("CREATE TABLE IF NOT EXISTS lock_probe (id INTEGER PRIMARY KEY)"); + walHolder.exec("INSERT INTO lock_probe (id) VALUES (1)"); + // Hold an open read transaction so the switch cannot checkpoint/truncate the WAL. + walHolder.exec("BEGIN"); + walHolder.prepare("SELECT * FROM lock_probe").all(); + + const warnings: string[] = []; + const originalWarn = console.warn; + console.warn = (...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }; + + let blocked: CentralDatabase | undefined; + try { + // busyTimeoutMs:0 → the failed switch returns immediately instead of waiting. + expect(() => { + blocked = new CentralDatabase(tempDir, { busyTimeoutMs: 0 }); + }).not.toThrow(); + + const mode = blocked!.prepare("PRAGMA journal_mode").get() as { journal_mode: string }; + // The switch failed: this connection is still WAL (documents the known gap)… + expect(mode.journal_mode).toBe("wal"); + // …and the failure was surfaced, not swallowed. + expect( + warnings.some((w) => /journal_mode=DELETE did not take effect/.test(w)), + ).toBe(true); + } finally { + console.warn = originalWarn; + blocked?.close(); + walHolder.exec("ROLLBACK"); + walHolder.close(); + } }); it("should seed lastModified on init", () => { diff --git a/packages/core/src/central-db.ts b/packages/core/src/central-db.ts index b37502a5ee..a86fabdbc7 100644 --- a/packages/core/src/central-db.ts +++ b/packages/core/src/central-db.ts @@ -564,24 +564,63 @@ export class CentralDatabase implements CentralClaimStore { // Wait up to the configured timeout for locks to clear before returning SQLITE_BUSY. // Set this before other PRAGMAs so they also benefit. this.db.exec(`PRAGMA busy_timeout = ${this.busyTimeoutMs}`); - // Enable WAL mode for concurrent reader/writer access - this.db.exec("PRAGMA journal_mode = WAL"); - // FNXC:Database 2026-06-20-12:30: - // Mirror the per-project DB durability/maintenance PRAGMAs (see db.ts). The - // central DB is shared across every project and cluster node, so it sees the - // most cross-process read/write traffic. - // - synchronous=FULL and wal_autocheckpoint=1000 are already SQLite's - // compiled-in defaults (FULL stays in effect under WAL — NORMAL is a - // common WAL *recommendation* but not the default). Set explicitly so the - // durability posture is intentional and visible, and so a future change to - // synchronous=NORMAL is a deliberate edit, not an accidental drift. - // - journal_size_limit is the load-bearing one: it defaults to -1 - // (unbounded), so without it the central WAL never truncates back down - // after a checkpoint and every reader pays an ever-growing WAL-index scan - // — a direct read-contention source. Cap it at 4 MB like the per-project DB. + // FNXC:Database 2026-06-24-22:30: + // The central DB runs in DELETE (rollback-journal) mode, NOT WAL. It is the + // one DB opened concurrently by every fusion process on the host (multiple + // dashboards/CLIs across worktrees all attach ~/.fusion/fusion-central.db). + // WAL coordinates those connections through a memory-mapped `-shm` wal-index; + // on macOS/APFS, when one process resizes/rebuilds `-shm` during a checkpoint + // while another has it mmap'd, the reader takes a SIGBUS (`FS pagein error` / + // `cluster_pagein past EOF`) inside walIndexReadHdr → the whole node process + // dies with no JS stack and no log. Observed 3× in 3 days (Jun 22–24 2026). + // node:sqlite cannot catch a hardware memory fault, so the only durable fix + // is to remove the `-shm` mmap surface. Rollback-journal mode uses no `-shm` + // and coordinates cross-process access via plain POSIX byte-range locks + // instead; busy_timeout above absorbs the writer-serialization contention + // that DELETE mode trades for WAL's reader/writer concurrency. + // + // FNXC:Database 2026-06-25-07:10: + // The WAL→DELETE switch is NOT silent-safe: SQLite needs an exclusive lock to + // checkpoint and drop `-wal`/`-shm`. If another connection still holds the DB + // open in WAL mode (the rolling-upgrade window, where an old-version process is + // still running) the switch cannot complete, and SQLite signals this in one of + // TWO ways depending on busy_timeout: it throws SQLITE_BUSY ("database is + // locked"), or it no-ops and the PRAGMA *returns the current mode* ("wal"). + // `exec()` would swallow the return value and let the throw abort the + // constructor, so we capture both: try the switch, treat a throw or a non-DELETE + // result identically, and warn loudly. We deliberately DO NOT rethrow — the + // condition is transient and self-healing (the next start after the last WAL + // holder exits migrates cleanly), and the residual SIGBUS surface during the + // window is no worse than the pre-fix status quo. Hard-failing here would make + // the central DB unopenable during the very upgrade window this describes. + let journalMode: string | undefined; + let switchError: unknown; + try { + const journalRow = this.db.prepare("PRAGMA journal_mode = DELETE").get() as + | { journal_mode?: string } + | undefined; + journalMode = journalRow?.journal_mode?.toLowerCase(); + } catch (error) { + switchError = error; + } + if (journalMode !== "delete") { + const detail = switchError + ? `failed: ${switchError instanceof Error ? switchError.message : String(switchError)}` + : `current mode: ${journalMode ?? "unknown"}`; + console.warn( + `[fusion:central-db] PRAGMA journal_mode=DELETE did not take effect ` + + `(${detail}) at ${this.dbPath}. Another process likely still holds the ` + + `database open in WAL mode; this connection keeps the WAL -shm mmap ` + + `(SIGBUS) surface until all WAL-mode holders exit and a fresh process ` + + `re-runs the migration.`, + ); + } + // synchronous=FULL is SQLite's compiled-in default; set explicitly so the + // durability posture is intentional and visible, and so a future change to + // synchronous=NORMAL is a deliberate edit, not an accidental drift. The WAL-only + // PRAGMAs (wal_autocheckpoint, journal_size_limit) were dropped with WAL — they + // are no-ops under DELETE mode, where the journal file is removed after each commit. this.db.exec("PRAGMA synchronous = FULL"); - this.db.exec("PRAGMA wal_autocheckpoint = 1000"); - this.db.exec("PRAGMA journal_size_limit = 4194304"); // Enable foreign key enforcement this.db.exec("PRAGMA foreign_keys = ON"); }