fix(db): use rollback-journal mode for central DB to stop SIGBUS crashes
The central DB (~/.fusion/fusion-central.db) is opened concurrently by every fusion process on a host. In WAL mode those connections coordinate through a memory-mapped `-shm` wal-index; on macOS/APFS a reader takes a SIGBUS (walIndexReadHdr / `cluster_pagein past EOF`) when another process resizes it mid-checkpoint, killing the node process with no JS stack or log. Observed 3x in 3 days. Switch the central DB to journal_mode=DELETE, which uses no `-shm` mmap and coordinates cross-process access via POSIX byte-range locks instead; busy_timeout absorbs the added writer serialization. Per-project DBs keep WAL. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
7
.changeset/central-db-delete-journal-sigbus.md
Normal file
7
.changeset/central-db-delete-journal-sigbus.md
Normal file
@@ -0,0 +1,7 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
summary: Fix random fusion crashes when multiple dashboards/CLIs run on one host.
|
||||
category: fix
|
||||
dev: Central DB (~/.fusion/fusion-central.db) now uses journal_mode=DELETE instead of WAL. WAL coordinates concurrent processes via a memory-mapped `-shm` wal-index that SIGBUSes a reader (walIndexReadHdr / `cluster_pagein past EOF`) on macOS/APFS when another process resizes it mid-checkpoint, killing the node process with no JS stack. DELETE mode removes the `-shm` mmap surface and coordinates via POSIX locks (busy_timeout absorbs the added writer serialization). Per-project DBs (db.ts) are unchanged. See central-db.ts open() and central-db.test.ts regression.
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { mkdtempSync, rmSync, statSync } from "node:fs";
|
||||
import { mkdtempSync, rmSync, statSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { CentralDatabase, createCentralDatabase, toJson, fromJson } from "../central-db.js";
|
||||
@@ -42,32 +42,35 @@ describe("CentralDatabase", () => {
|
||||
expect(db.getSchemaVersion()).toBe(13);
|
||||
});
|
||||
|
||||
it("should enable WAL mode and busy_timeout", () => {
|
||||
it("should use DELETE (rollback-journal) mode and busy_timeout, not WAL", () => {
|
||||
db.init();
|
||||
|
||||
const journalMode = db.prepare("PRAGMA journal_mode").get() as { journal_mode: string };
|
||||
const busyTimeout = db.prepare("PRAGMA busy_timeout").get() as Record<string, number>;
|
||||
|
||||
expect(journalMode.journal_mode).toBe("wal");
|
||||
// Regression: the central DB must NOT run in WAL mode. WAL coordinates the
|
||||
// many concurrent fusion processes through a memory-mapped `-shm` wal-index,
|
||||
// which on macOS/APFS SIGBUSes a reader (walIndexReadHdr / `cluster_pagein
|
||||
// past EOF`) when another process resizes it mid-checkpoint — observed 3×
|
||||
// in 3 days (Jun 22–24 2026). DELETE mode removes the `-shm` mmap surface.
|
||||
expect(journalMode.journal_mode).toBe("delete");
|
||||
expect(Object.values(busyTimeout)[0]).toBe(5000);
|
||||
});
|
||||
|
||||
it("should bound WAL growth and durability like the per-project DB", () => {
|
||||
it("should never create a `-shm` wal-index file (the SIGBUS surface)", () => {
|
||||
db.init();
|
||||
// Drive real write traffic; under WAL this materializes `-shm` + `-wal`.
|
||||
db.bumpLastModified();
|
||||
db.prepare("SELECT * FROM globalConcurrency WHERE id = 1").get();
|
||||
|
||||
const dbPath = db.getPath();
|
||||
// The wal-index shared-memory file is the exact thing that was memmap'd
|
||||
// and faulted. Its absence proves the crashing surface is gone.
|
||||
expect(existsSync(`${dbPath}-shm`)).toBe(false);
|
||||
expect(existsSync(`${dbPath}-wal`)).toBe(false);
|
||||
|
||||
const synchronous = db.prepare("PRAGMA synchronous").get() as { synchronous: number };
|
||||
const autoCheckpoint = db
|
||||
.prepare("PRAGMA wal_autocheckpoint")
|
||||
.get() as { wal_autocheckpoint: number };
|
||||
const journalSizeLimit = db
|
||||
.prepare("PRAGMA journal_size_limit")
|
||||
.get() as { journal_size_limit: number };
|
||||
|
||||
expect(synchronous.synchronous).toBe(2); // FULL
|
||||
expect(autoCheckpoint.wal_autocheckpoint).toBe(1000);
|
||||
// Previously unset (-1 / unbounded), which let the central WAL bloat and
|
||||
// slow every reader. Now capped at 4 MB to match db.ts.
|
||||
expect(journalSizeLimit.journal_size_limit).toBe(4_194_304);
|
||||
expect(synchronous.synchronous).toBe(2); // FULL — durability posture preserved
|
||||
});
|
||||
|
||||
it("should seed lastModified on init", () => {
|
||||
|
||||
@@ -564,24 +564,27 @@ export class CentralDatabase implements CentralClaimStore {
|
||||
// Wait up to the configured timeout for locks to clear before returning SQLITE_BUSY.
|
||||
// Set this before other PRAGMAs so they also benefit.
|
||||
this.db.exec(`PRAGMA busy_timeout = ${this.busyTimeoutMs}`);
|
||||
// Enable WAL mode for concurrent reader/writer access
|
||||
this.db.exec("PRAGMA journal_mode = WAL");
|
||||
// FNXC:Database 2026-06-20-12:30:
|
||||
// Mirror the per-project DB durability/maintenance PRAGMAs (see db.ts). The
|
||||
// central DB is shared across every project and cluster node, so it sees the
|
||||
// most cross-process read/write traffic.
|
||||
// - synchronous=FULL and wal_autocheckpoint=1000 are already SQLite's
|
||||
// compiled-in defaults (FULL stays in effect under WAL — NORMAL is a
|
||||
// common WAL *recommendation* but not the default). Set explicitly so the
|
||||
// durability posture is intentional and visible, and so a future change to
|
||||
// synchronous=NORMAL is a deliberate edit, not an accidental drift.
|
||||
// - journal_size_limit is the load-bearing one: it defaults to -1
|
||||
// (unbounded), so without it the central WAL never truncates back down
|
||||
// after a checkpoint and every reader pays an ever-growing WAL-index scan
|
||||
// — a direct read-contention source. Cap it at 4 MB like the per-project DB.
|
||||
// FNXC:Database 2026-06-24-22:30:
|
||||
// The central DB runs in DELETE (rollback-journal) mode, NOT WAL. It is the
|
||||
// one DB opened concurrently by every fusion process on the host (multiple
|
||||
// dashboards/CLIs across worktrees all attach ~/.fusion/fusion-central.db).
|
||||
// WAL coordinates those connections through a memory-mapped `-shm` wal-index;
|
||||
// on macOS/APFS, when one process resizes/rebuilds `-shm` during a checkpoint
|
||||
// while another has it mmap'd, the reader takes a SIGBUS (`FS pagein error` /
|
||||
// `cluster_pagein past EOF`) inside walIndexReadHdr → the whole node process
|
||||
// dies with no JS stack and no log. Observed 3× in 3 days (Jun 22–24 2026).
|
||||
// node:sqlite cannot catch a hardware memory fault, so the only durable fix
|
||||
// is to remove the `-shm` mmap surface. Rollback-journal mode uses no `-shm`
|
||||
// and coordinates cross-process access via plain POSIX byte-range locks
|
||||
// instead; busy_timeout above absorbs the writer-serialization contention
|
||||
// that DELETE mode trades for WAL's reader/writer concurrency.
|
||||
this.db.exec("PRAGMA journal_mode = DELETE");
|
||||
// synchronous=FULL is SQLite's compiled-in default; set explicitly so the
|
||||
// durability posture is intentional and visible, and so a future change to
|
||||
// synchronous=NORMAL is a deliberate edit, not an accidental drift. The WAL-only
|
||||
// PRAGMAs (wal_autocheckpoint, journal_size_limit) were dropped with WAL — they
|
||||
// are no-ops under DELETE mode, where the journal file is removed after each commit.
|
||||
this.db.exec("PRAGMA synchronous = FULL");
|
||||
this.db.exec("PRAGMA wal_autocheckpoint = 1000");
|
||||
this.db.exec("PRAGMA journal_size_limit = 4194304");
|
||||
// Enable foreign key enforcement
|
||||
this.db.exec("PRAGMA foreign_keys = ON");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user