Harden fusion.db against recurring corruption

Root cause: node:sqlite SIGSEGVs inside pager_write leave the B-tree
malformed in a way that still opens but fails integrity checks; large
operational-log tables widen the write window where the crash strikes.

- backup: verify every copy with PRAGMA quick_check, quarantine corrupt
  copies as *.corrupt, and never rotate out the last verified-good backup
- db: add Database.recoverIfCorrupt() startup guard (wired into
  TaskStore.init, disk-backed only, opt out via FUSION_DISABLE_DB_AUTORECOVER)
  that rebuilds a malformed db via sqlite3 .recover, preserving the corrupt
  original; also fixes the latent `.recover main` invalid-option bug that made
  recoverDatabase() always fail
- db: drop lost_and_found* scratch tables on init; add pruneOperationalLogs()
- settings: add operationalLogRetentionDays (default 30, 0 = off) and prune
  activityLog/agentLogEntries/runAuditEvents/agentHeartbeats during maintenance
- dashboard: expose retention in Settings -> Backups -> Database Maintenance

Tests: backup 59/59, db 135/135 (incl. real corrupt->recover->reopen),
self-healing cleanup/corruption 10/10, settings 77/77, SettingsModal 460/460.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-06-01 21:55:10 -07:00
parent e561290174
commit f9e551317f
10 changed files with 713 additions and 16 deletions

View File

@@ -1,5 +1,6 @@
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { mkdtempSync, writeFileSync, existsSync, readFileSync } from "node:fs";
import { spawnSync } from "node:child_process";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { rm, mkdir, writeFile, readdir } from "node:fs/promises";
@@ -18,6 +19,21 @@ import { RoutineStore } from "../routine-store.js";
import { TaskStore } from "../store.js";
import type { ProjectSettings } from "../types.js";
/**
* Write a real SQLite database file so the production backup path's
* `PRAGMA quick_check` verification passes. Falls back to a placeholder string
* when the `sqlite3` CLI is unavailable — in that environment verification also
* no-ops, so the backup still succeeds and the assertions hold either way.
*/
function writeTestDb(path: string): void {
const result = spawnSync("sqlite3", [path, "CREATE TABLE IF NOT EXISTS t(x); INSERT INTO t VALUES (1);"], {
encoding: "utf-8",
});
if (result.error || result.status !== 0) {
writeFileSync(path, "dummy database content");
}
}
describe("BackupManager", () => {
let tempDir: string;
let fusionDir: string;
@@ -36,6 +52,10 @@ describe("BackupManager", () => {
writeFileSync(join(fusionDir, "fusion-central.db"), "dummy central database content");
backupManager = new BackupManager(fusionDir, {
centralDbPath: join(fusionDir, "fusion-central.db"),
// These tests use dummy (non-SQLite) files as the source db, so the
// PRAGMA quick_check verification cannot run against them. Integrity
// verification has its own dedicated tests with real SQLite databases.
verifyIntegrity: false,
});
});
@@ -75,6 +95,7 @@ describe("BackupManager", () => {
const manager = new BackupManager(fusionDir, {
backupDir: customBackupDir,
centralDbPath: join(fusionDir, "fusion-central.db"),
verifyIntegrity: false,
});
const customBackupPath = join(tempDir, customBackupDir);
@@ -96,6 +117,7 @@ describe("BackupManager", () => {
it("skips central backup when central DB is missing", async () => {
const manager = new BackupManager(fusionDir, {
centralDbPath: join(fusionDir, "does-not-exist.db"),
verifyIntegrity: false,
});
const backup = await manager.createBackup();
expect(backup.centralBackup).toEqual({ skipped: "missing" });
@@ -105,6 +127,7 @@ describe("BackupManager", () => {
const manager = new BackupManager(fusionDir, {
centralDbPath: join(fusionDir, "fusion-central.db"),
includeCentralDb: false,
verifyIntegrity: false,
});
const backup = await manager.createBackup();
expect(backup.centralBackup).toEqual({ skipped: "disabled" });
@@ -132,7 +155,7 @@ describe("BackupManager", () => {
it("continues central copy when checkpoint open fails", async () => {
const notSqlitePath = join(fusionDir, "not-sqlite.db");
writeFileSync(notSqlitePath, "definitely not sqlite");
const manager = new BackupManager(fusionDir, { centralDbPath: notSqlitePath });
const manager = new BackupManager(fusionDir, { centralDbPath: notSqlitePath, verifyIntegrity: false });
const backup = await manager.createBackup();
expect(backup.centralBackup && "filename" in backup.centralBackup).toBe(true);
if (backup.centralBackup && "filename" in backup.centralBackup) {
@@ -141,7 +164,7 @@ describe("BackupManager", () => {
});
it("keeps project backup when central copy fails", async () => {
const manager = new BackupManager(fusionDir, { centralDbPath: fusionDir });
const manager = new BackupManager(fusionDir, { centralDbPath: fusionDir, verifyIntegrity: false });
const backup = await manager.createBackup();
expect(existsSync(backup.path)).toBe(true);
expect(backup.centralBackup && "failed" in backup.centralBackup).toBe(true);
@@ -279,7 +302,7 @@ describe("BackupManager", () => {
});
it("should delete oldest backups exceeding retention", async () => {
const manager = new BackupManager(fusionDir, { retention: 2, centralDbPath: join(fusionDir, "fusion-central.db") });
const manager = new BackupManager(fusionDir, { retention: 2, centralDbPath: join(fusionDir, "fusion-central.db"), verifyIntegrity: false });
// Create 4 backups by advancing time deterministically
for (let i = 0; i < 4; i++) {
@@ -295,7 +318,7 @@ describe("BackupManager", () => {
});
it("should keep the newest backups after cleanup", async () => {
const manager = new BackupManager(fusionDir, { retention: 2, centralDbPath: join(fusionDir, "fusion-central.db") });
const manager = new BackupManager(fusionDir, { retention: 2, centralDbPath: join(fusionDir, "fusion-central.db"), verifyIntegrity: false });
// Create 4 backups and record their names by advancing time
const backupNames: string[] = [];
@@ -318,7 +341,7 @@ describe("BackupManager", () => {
});
it("deletes sibling central backup when deleting project backup", async () => {
const manager = new BackupManager(fusionDir, { retention: 1, centralDbPath: join(fusionDir, "fusion-central.db") });
const manager = new BackupManager(fusionDir, { retention: 1, centralDbPath: join(fusionDir, "fusion-central.db"), verifyIntegrity: false });
await manager.createBackup();
vi.setSystemTime(new Date("2026-01-01T00:00:01.000Z"));
await manager.createBackup();
@@ -427,6 +450,83 @@ describe("BackupManager", () => {
});
});
describe("backup integrity verification", () => {
let tempDir: string;
let fusionDir: string;
let sqlite3Available: boolean;
beforeEach(async () => {
tempDir = mkdtempSync(join(tmpdir(), "kb-backup-verify-"));
fusionDir = join(tempDir, ".fusion");
await mkdir(fusionDir, { recursive: true });
// Detect sqlite3 once: verification (and the corruption assertions that
// depend on it) only run meaningfully where the CLI exists.
const probe = spawnSync("sqlite3", ["--version"], { encoding: "utf-8" });
sqlite3Available = !probe.error && probe.status === 0;
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it("verifyDatabaseIntegrity returns ok for a real SQLite db", async () => {
if (!sqlite3Available) return;
const { verifyDatabaseIntegrity } = await import("../backup.js");
const dbPath = join(fusionDir, "fusion.db");
writeTestDb(dbPath);
const result = verifyDatabaseIntegrity(dbPath);
expect(result.ok).toBe(true);
expect(result.verified).toBe(true);
});
it("verifyDatabaseIntegrity flags a non-SQLite file as corrupt", async () => {
if (!sqlite3Available) return;
const { verifyDatabaseIntegrity } = await import("../backup.js");
const dbPath = join(fusionDir, "fusion.db");
writeFileSync(dbPath, "definitely not a sqlite database");
const result = verifyDatabaseIntegrity(dbPath);
expect(result.ok).toBe(false);
expect(result.verified).toBe(true);
});
it("createBackup refuses to keep a corrupt copy and quarantines it", async () => {
if (!sqlite3Available) return;
// Source db is not a valid SQLite file → verification must reject it.
writeFileSync(join(fusionDir, "fusion.db"), "corrupt source");
const manager = new BackupManager(fusionDir, {
centralDbPath: join(fusionDir, "fusion-central.db"),
includeCentralDb: false,
});
await expect(manager.createBackup()).rejects.toThrow(/verification failed/i);
// No listed (good) backup remains; the corrupt copy is quarantined as *.corrupt.
const backups = await manager.listBackups();
expect(backups).toEqual([]);
const files = await readdir(join(tempDir, ".fusion/backups"));
expect(files.some((f) => f.endsWith(".corrupt"))).toBe(true);
});
it("cleanupOldBackups never deletes the last verified-good backup", async () => {
if (!sqlite3Available) return;
const backupDir = join(tempDir, ".fusion/backups");
await mkdir(backupDir, { recursive: true });
// One real (good) backup, older than two newer corrupt ones.
writeTestDb(join(backupDir, "fusion-2026-01-01-000000.db"));
writeFileSync(join(backupDir, "fusion-2026-01-02-000000.db"), "corrupt newer 1");
writeFileSync(join(backupDir, "fusion-2026-01-03-000000.db"), "corrupt newer 2");
const manager = new BackupManager(fusionDir, { retention: 2, includeCentralDb: false });
await manager.cleanupOldBackups();
// Retention=2 would normally delete the oldest, but it is the only good one,
// so it must survive.
const remaining = (await manager.listBackups()).map((b) => b.filename);
expect(remaining).toContain("fusion-2026-01-01-000000.db");
});
});
describe("generateBackupFilename", () => {
it("should generate filename with correct pattern", () => {
const filename = generateBackupFilename();
@@ -520,7 +620,7 @@ describe("createBackupManager", () => {
const tempDir = mkdtempSync(join(tmpdir(), "kb-backup-test-"));
const fusionDir = join(tempDir, ".fusion");
await mkdir(fusionDir, { recursive: true });
writeFileSync(join(fusionDir, "fusion.db"), "test");
writeTestDb(join(fusionDir, "fusion.db"));
const settings: Partial<ProjectSettings> = {
autoBackupDir: "custom/backups",
@@ -547,7 +647,7 @@ describe("createBackupManager", () => {
const tempDir = mkdtempSync(join(tmpdir(), "kb-backup-test-"));
const fusionDir = join(tempDir, ".fusion");
await mkdir(fusionDir, { recursive: true });
writeFileSync(join(fusionDir, "fusion.db"), "test");
writeTestDb(join(fusionDir, "fusion.db"));
const settings: Partial<ProjectSettings> = {
autoBackupDir: ".kb/backups", // Legacy value
@@ -572,7 +672,7 @@ describe("createBackupManager", () => {
const tempDir = mkdtempSync(join(tmpdir(), "kb-backup-test-"));
const fusionDir = join(tempDir, ".fusion");
await mkdir(fusionDir, { recursive: true });
writeFileSync(join(fusionDir, "fusion.db"), "test");
writeTestDb(join(fusionDir, "fusion.db"));
const settings: Partial<ProjectSettings> = {
autoBackupDir: ".kb/my-custom-backups", // Custom path, not the legacy default
@@ -735,7 +835,7 @@ describe("runBackupCommand", () => {
tempDir = mkdtempSync(join(tmpdir(), "kb-backup-test-"));
fusionDir = join(tempDir, ".fusion");
await mkdir(fusionDir, { recursive: true });
writeFileSync(join(fusionDir, "fusion.db"), "dummy database content");
writeTestDb(join(fusionDir, "fusion.db"));
});
afterEach(async () => {

View File

@@ -2,6 +2,7 @@ import { describe, it, expect, beforeAll, beforeEach, afterEach, afterAll, vi }
import {
Database,
createDatabase,
quickCheckSqliteFile,
toJson,
toJsonNullable,
fromJson,
@@ -11,13 +12,13 @@ import {
} from "../db.js";
import { DEFAULT_PROJECT_SETTINGS } from "../types.js";
import { TaskStore } from "../store.js";
import { mkdtempSync, existsSync, readFileSync, rmSync, statSync } from "node:fs";
import { mkdtempSync, existsSync, readFileSync, rmSync, statSync, openSync, writeSync, closeSync } from "node:fs";
import { join, dirname } from "node:path";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
import { rm } from "node:fs/promises";
import { once } from "node:events";
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process";
import { ensureRoadmapSchema } from "../../../../plugins/fusion-plugin-roadmap/src/roadmap-schema.js";
import { createSharedTaskStoreTestHarness } from "./store-test-helpers.js";
@@ -2854,3 +2855,165 @@ describe("migration v67 drops orphan project auth tables", () => {
}
});
});
describe("Database operational-log retention and recovery-table cleanup", () => {
let tmpDir: string;
let fusionDir: string;
let db: Database;
beforeEach(() => {
tmpDir = makeTmpDir();
fusionDir = join(tmpDir, ".fusion");
db = new Database(fusionDir);
db.init();
});
afterEach(async () => {
try {
db.close();
} catch {
// already closed
}
await cleanupTmpDirsAsync();
});
function insertActivity(id: string, timestamp: string): void {
db.prepare(
"INSERT INTO activityLog (id, timestamp, type, details) VALUES (?, ?, 'test', '{}')",
).run(id, timestamp);
}
it("pruneOperationalLogs deletes rows older than the retention window", () => {
const old = new Date(Date.now() - 200 * 86_400_000).toISOString();
const recent = new Date(Date.now() - 1 * 86_400_000).toISOString();
insertActivity("old-1", old);
insertActivity("old-2", old);
insertActivity("recent-1", recent);
const result = db.pruneOperationalLogs(90 * 86_400_000);
expect(result.deletedTotal).toBe(2);
expect(result.deletedByTable.activityLog).toBe(2);
const remaining = db.prepare("SELECT id FROM activityLog ORDER BY id").all() as Array<{ id: string }>;
expect(remaining.map((r) => r.id)).toEqual(["recent-1"]);
});
it("pruneOperationalLogs is a no-op when retention is disabled (<= 0)", () => {
insertActivity("old-1", new Date(Date.now() - 200 * 86_400_000).toISOString());
const result = db.pruneOperationalLogs(0);
expect(result.deletedTotal).toBe(0);
expect(db.prepare("SELECT count(*) AS c FROM activityLog").get()).toMatchObject({ c: 1 });
});
it("dropOrphanRecoveryTables removes lost_and_found scratch tables", () => {
db.exec("CREATE TABLE lost_and_found (x)");
db.exec("CREATE TABLE lost_and_found_0 (x)");
db.exec("CREATE TABLE lost_and_found_2 (x)");
const dropped = db.dropOrphanRecoveryTables();
expect(dropped).toBe(3);
const tables = db
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lost_and_found%'")
.all();
expect(tables).toEqual([]);
});
it("init() drops pre-existing lost_and_found tables on open", () => {
db.exec("CREATE TABLE lost_and_found_0 (x)");
db.close();
const reopened = new Database(fusionDir);
reopened.init();
try {
const tables = reopened
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lost_and_found%'")
.all();
expect(tables).toEqual([]);
} finally {
reopened.close();
}
});
});
describe("Database.recoverIfCorrupt startup guard", () => {
let tmpDir: string;
let fusionDir: string;
const sqlite3Available = (() => {
const probe = spawnSync("sqlite3", ["--version"], { encoding: "utf-8" });
return !probe.error && probe.status === 0;
})();
beforeEach(() => {
tmpDir = makeTmpDir();
fusionDir = join(tmpDir, ".fusion");
});
afterEach(async () => {
await cleanupTmpDirsAsync();
});
it("returns 'absent' when no database exists", () => {
const result = Database.recoverIfCorrupt(fusionDir);
expect(result.status).toBe("absent");
});
it("returns 'healthy' for an intact database", () => {
if (!sqlite3Available) return;
const db = new Database(fusionDir);
db.init();
db.close();
const result = Database.recoverIfCorrupt(fusionDir);
expect(result.status).toBe("healthy");
});
it("rebuilds a malformed database and preserves the corrupt original", () => {
if (!sqlite3Available) return;
const dbPath = join(fusionDir, "fusion.db");
const db = new Database(fusionDir);
db.init();
// Span many pages so mid-file corruption lands on a B-tree page.
for (let i = 0; i < 3000; i++) {
db.prepare("INSERT INTO activityLog (id, timestamp, type, details) VALUES (?, ?, 'test', '{}')").run(
`row-${i}`,
new Date().toISOString(),
);
}
db.walCheckpoint("TRUNCATE");
db.close();
// Corrupt an interior region while leaving the header page intact.
const size = statSync(dbPath).size;
const fd = openSync(dbPath, "r+");
try {
const garbage = Buffer.alloc(16 * 1024, 0xab);
writeSync(fd, garbage, 0, garbage.length, Math.floor(size / 2));
} finally {
closeSync(fd);
}
// If the corruption didn't trip quick_check on this build, skip rather
// than assert flakily.
const pre = quickCheckSqliteFile(dbPath);
if (pre.ok) return;
const result = Database.recoverIfCorrupt(fusionDir);
expect(result.status).toBe("recovered");
expect(result.corruptBackupPath).toBeDefined();
expect(existsSync(result.corruptBackupPath!)).toBe(true);
// The swapped-in database must now be clean.
expect(quickCheckSqliteFile(dbPath).ok).toBe(true);
// Stale sidecars must not linger.
expect(existsSync(`${dbPath}-wal`)).toBe(false);
// And it must open and answer queries.
const reopened = new Database(fusionDir);
reopened.init();
try {
const row = reopened.prepare("SELECT count(*) AS c FROM activityLog").get() as { c: number };
expect(row.c).toBeGreaterThan(0);
} finally {
reopened.close();
}
});
});

View File

@@ -1,5 +1,6 @@
import { cp, mkdir, readdir, stat, unlink } from "node:fs/promises";
import { cp, mkdir, readdir, rename, stat, unlink } from "node:fs/promises";
import { existsSync } from "node:fs";
import { spawnSync } from "node:child_process";
import { join } from "node:path";
import { CronExpressionParser } from "cron-parser";
import { getDefaultCentralDbPath } from "./central-db.js";
@@ -34,6 +35,12 @@ export interface BackupOptions {
retention?: number;
centralDbPath?: string;
includeCentralDb?: boolean;
/**
* Verify each backup copy with `PRAGMA quick_check` and refuse to keep or
* rotate-in a corrupt copy. Defaults to true. Set false only where the
* source is intentionally not a real SQLite file (e.g. unit tests).
*/
verifyIntegrity?: boolean;
}
export class BackupManager {
@@ -42,6 +49,7 @@ export class BackupManager {
private retention: number;
private centralDbPath: string;
private includeCentralDb: boolean;
private verifyIntegrity: boolean;
constructor(fusionDir: string, options?: BackupOptions) {
this.fusionDir = fusionDir;
@@ -49,6 +57,7 @@ export class BackupManager {
this.retention = options?.retention ?? 7;
this.centralDbPath = options?.centralDbPath ?? join(this.fusionDir, "..", ".fusion", "fusion-central.db");
this.includeCentralDb = options?.includeCentralDb ?? true;
this.verifyIntegrity = options?.verifyIntegrity ?? true;
}
private getBackupDirPath(): string {
@@ -89,6 +98,21 @@ export class BackupManager {
await copyLiveDatabase(sourcePath, targetPath);
// Verify the freshly-written copy. A copy of a live WAL db can capture a
// torn/corrupt main file; refusing to keep a corrupt backup guarantees
// that every retained `fusion-*.db` is restorable and that a corrupt copy
// is never counted as the "last known-good" by cleanupOldBackups().
if (this.verifyIntegrity) {
const integrity = verifyDatabaseIntegrity(targetPath);
if (!integrity.ok) {
await quarantineCorruptBackup(targetPath);
throw new Error(
`Backup verification failed for ${filename}: ${integrity.error ?? "database disk image is malformed"}. ` +
"The source database may be corrupt; the unusable copy was quarantined as *.corrupt.",
);
}
}
const stats = await stat(targetPath);
const backup: BackupInfo = {
filename,
@@ -113,6 +137,16 @@ export class BackupManager {
try {
await copyLiveDatabase(this.centralDbPath, centralTargetPath);
if (this.verifyIntegrity) {
const centralIntegrity = verifyDatabaseIntegrity(centralTargetPath);
if (!centralIntegrity.ok) {
await quarantineCorruptBackup(centralTargetPath);
throw new Error(
`central DB verification failed: ${centralIntegrity.error ?? "database disk image is malformed"}`,
);
}
}
const centralStats = await stat(centralTargetPath);
backup.centralBackup = {
filename: centralFilename,
@@ -224,6 +258,28 @@ export class BackupManager {
});
const toDelete = sorted.slice(0, sorted.length - this.retention);
// Never rotate out the last known-good backup. The retained set is the
// newest `retention` files, but if every one of them fails verification
// (e.g. a run of corrupt copies from a flaky source db) we must protect the
// newest verifiably-good backup from deletion even though it falls outside
// the retention window. Verification is lazy: in the common case the newest
// kept backup is good and we run exactly one check.
if (this.verifyIntegrity) {
const kept = sorted.slice(sorted.length - this.retention);
const keptHasGood = kept
.slice()
.reverse()
.some((b) => verifyDatabaseIntegrity(b.path).ok);
if (!keptHasGood) {
for (let i = toDelete.length - 1; i >= 0; i--) {
if (verifyDatabaseIntegrity(toDelete[i].path).ok) {
toDelete.splice(i, 1);
break;
}
}
}
}
let deletedCount = 0;
for (const backup of toDelete) {
try {
@@ -352,6 +408,81 @@ async function copyLiveDatabase(sourcePath: string, targetPath: string): Promise
}
}
/**
* Result of an on-disk SQLite integrity verification.
*
* `verified` distinguishes "we ran the check" from "we couldn't run it". When
* the `sqlite3` CLI is unavailable (e.g. a packaged environment with no system
* binary on PATH) we return `{ ok: true, verified: false }` so verification
* degrades to a no-op rather than blocking backups or rotation.
*/
export interface DatabaseIntegrityResult {
ok: boolean;
verified: boolean;
error?: string;
}
/**
* Verify a SQLite database file with `PRAGMA quick_check`.
*
* Uses the `sqlite3` CLI (the same dependency the recovery path relies on) so
* we never open the file through the live `node:sqlite` connection — opening a
* WAL-mode copy through node:sqlite would replay/checkpoint pages and mutate
* the very backup we are trying to validate. `quick_check` is far cheaper than
* a full `integrity_check` but still detects the B-tree malformations
* ("rowid out of order", "2nd reference to page") that node:sqlite SIGSEGVs
* leave behind.
*/
export function verifyDatabaseIntegrity(dbPath: string): DatabaseIntegrityResult {
if (!existsSync(dbPath)) {
return { ok: false, verified: true, error: "file does not exist" };
}
const result = spawnSync("sqlite3", [dbPath, "PRAGMA quick_check;"], {
encoding: "utf-8",
maxBuffer: 8 * 1024 * 1024,
});
// ENOENT (or any spawn error) means the sqlite3 binary is unavailable — we
// cannot verify, so treat as a non-blocking pass.
if (result.error) {
return { ok: true, verified: false, error: result.error.message };
}
const stdout = (result.stdout ?? "").trim();
if (result.status !== 0) {
return {
ok: false,
verified: true,
error: stdout || (result.stderr ?? "").trim() || `sqlite3 exited ${result.status}`,
};
}
if (stdout.toLowerCase() === "ok") {
return { ok: true, verified: true };
}
return { ok: false, verified: true, error: stdout.split("\n").slice(0, 3).join(" | ") };
}
/** Move a verifiably-corrupt backup copy aside so it never masquerades as good. */
async function quarantineCorruptBackup(targetPath: string): Promise<void> {
for (const suffix of ["", "-wal", "-shm"]) {
const path = `${targetPath}${suffix}`;
if (!existsSync(path)) continue;
try {
await rename(path, `${path}.corrupt`);
} catch {
// Best effort — fall back to deleting so a corrupt copy is never listed.
try {
await unlink(path);
} catch {
// Ignore.
}
}
}
}
function sortBackupsNewestFirst(backups: BackupFileInfo[]): BackupFileInfo[] {
return backups.sort((a, b) => {
const timeCompare = b.createdAt.localeCompare(a.createdAt);

View File

@@ -10,7 +10,7 @@
import { DatabaseSync } from "./sqlite-adapter.js";
import { basename, isAbsolute, join } from "node:path";
import { mkdirSync, existsSync, statSync } from "node:fs";
import { mkdirSync, existsSync, statSync, renameSync, rmSync } from "node:fs";
import { spawnSync } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
import { DEFAULT_PROJECT_SETTINGS } from "./types.js";
@@ -1336,6 +1336,44 @@ export const SCHEMA_COMPAT_FINGERPRINT = createHash("sha1")
)
.digest("hex");
/** Compact UTC timestamp (YYYY-MM-DD-HHmmss) for recovery artifact filenames. */
function formatDbRecoveryTimestamp(date: Date): string {
const y = date.getUTCFullYear();
const m = String(date.getUTCMonth() + 1).padStart(2, "0");
const d = String(date.getUTCDate()).padStart(2, "0");
const hh = String(date.getUTCHours()).padStart(2, "0");
const mm = String(date.getUTCMinutes()).padStart(2, "0");
const ss = String(date.getUTCSeconds()).padStart(2, "0");
return `${y}-${m}-${d}-${hh}${mm}${ss}`;
}
/**
* Run `PRAGMA quick_check` against a SQLite file via the `sqlite3` CLI without
* opening a live connection (so we never replay/checkpoint a WAL onto it).
* `verified=false` means the check could not run (sqlite3 unavailable) and the
* caller should treat the result as non-blocking.
*/
export function quickCheckSqliteFile(dbPath: string): { ok: boolean; verified: boolean; errors?: string[] } {
if (!existsSync(dbPath)) {
return { ok: false, verified: true, errors: ["file does not exist"] };
}
const result = spawnSync("sqlite3", [dbPath, "PRAGMA quick_check;"], {
encoding: "utf-8",
maxBuffer: 16 * 1024 * 1024,
});
if (result.error) {
return { ok: true, verified: false };
}
const stdout = (result.stdout ?? "").trim();
if (result.status !== 0) {
return { ok: false, verified: true, errors: [stdout || (result.stderr ?? "").trim() || `sqlite3 exited ${result.status}`] };
}
if (stdout.toLowerCase() === "ok") {
return { ok: true, verified: true };
}
return { ok: false, verified: true, errors: stdout.split("\n").slice(0, 5) };
}
// ── Database Class ───────────────────────────────────────────────────
type SharedIntegrityCheckState = {
@@ -1581,9 +1619,9 @@ export class Database {
return false;
}
const recoveredSql = spawnSync("sqlite3", ["-cmd", ".recover main", this.dbPath], {
const recoveredSql = spawnSync("sqlite3", [this.dbPath, ".recover"], {
encoding: "utf-8",
maxBuffer: 50 * 1024 * 1024,
maxBuffer: 256 * 1024 * 1024,
});
if (recoveredSql.status !== 0 || !recoveredSql.stdout) {
return false;
@@ -1592,12 +1630,91 @@ export class Database {
const rebuilt = spawnSync("sqlite3", [outputPath], {
input: recoveredSql.stdout,
encoding: "utf-8",
maxBuffer: 50 * 1024 * 1024,
maxBuffer: 256 * 1024 * 1024,
});
return rebuilt.status === 0;
}
/**
* Startup guard: detect a malformed `fusion.db` and rebuild it via
* `sqlite3 .recover` BEFORE any connection is opened for normal use.
*
* This is the automated form of the manual recovery: a node:sqlite SIGSEGV
* mid-write can leave the B-tree malformed in a way that still *opens* and
* answers simple queries (so a sentinel SELECT won't catch it) — only an
* integrity/quick check does. When corruption is found we:
* 1. recover the readable data into a fresh file,
* 2. verify the rebuilt file passes quick_check,
* 3. preserve the corrupt original as `fusion.db.corrupt-<ts>`,
* 4. atomically swap the rebuilt file into place and drop stale -wal/-shm.
*
* Must run with no open connection to `fusion.db`. Returns a status describing
* what happened; on `failed` the original file is left untouched for manual
* inspection. `sqlite3` CLI absence yields `unverified` (non-blocking no-op).
*/
static recoverIfCorrupt(fusionDir: string): {
status: "absent" | "healthy" | "unverified" | "recovered" | "failed";
corruptBackupPath?: string;
recoveredPath?: string;
errors?: string[];
} {
const dbPath = join(fusionDir, "fusion.db");
if (!existsSync(dbPath)) {
return { status: "absent" };
}
const check = quickCheckSqliteFile(dbPath);
if (!check.verified) {
return { status: "unverified" };
}
if (check.ok) {
return { status: "healthy" };
}
// Corruption confirmed — attempt an offline rebuild.
const ts = formatDbRecoveryTimestamp(new Date());
const recoveredPath = `${dbPath}.recovered-${ts}`;
const recoveredSql = spawnSync("sqlite3", [dbPath, ".recover"], {
encoding: "utf-8",
maxBuffer: 256 * 1024 * 1024,
});
if (recoveredSql.status !== 0 || !recoveredSql.stdout) {
return { status: "failed", errors: check.errors };
}
const rebuilt = spawnSync("sqlite3", [recoveredPath], {
input: recoveredSql.stdout,
encoding: "utf-8",
maxBuffer: 256 * 1024 * 1024,
});
if (rebuilt.status !== 0) {
try { rmSync(recoveredPath, { force: true }); } catch { /* ignore */ }
return { status: "failed", errors: check.errors };
}
// Refuse to swap in a rebuild that is itself not clean.
const verifyRebuilt = quickCheckSqliteFile(recoveredPath);
if (verifyRebuilt.verified && !verifyRebuilt.ok) {
try { rmSync(recoveredPath, { force: true }); } catch { /* ignore */ }
return { status: "failed", errors: check.errors };
}
try {
const corruptBackupPath = `${dbPath}.corrupt-${ts}`;
renameSync(dbPath, corruptBackupPath);
// Stale WAL/SHM belong to the corrupt file; SQLite must not replay them
// onto the rebuilt database.
try { rmSync(`${dbPath}-wal`, { force: true }); } catch { /* ignore */ }
try { rmSync(`${dbPath}-shm`, { force: true }); } catch { /* ignore */ }
renameSync(recoveredPath, dbPath);
return { status: "recovered", corruptBackupPath, errors: check.errors };
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return { status: "failed", errors: [...(check.errors ?? []), message] };
}
}
/**
* Run WAL truncation + VACUUM and report compaction stats.
*
@@ -1642,6 +1759,88 @@ export class Database {
}
}
/**
* Drop scratch tables left behind by `sqlite3 .recover`.
*
* Recovery emits `lost_and_found` / `lost_and_found_N` tables holding orphaned
* rows it could not attribute to a real table. They are never part of the
* Fusion schema, but a recovered db that gets backed up and restored carries
* them forward indefinitely — on this database they had accumulated ~250K dead
* rows across prior recoveries, inflating file size and every integrity check.
* Returns the number of scratch tables dropped.
*/
dropOrphanRecoveryTables(): number {
if (this.inMemory) {
return 0;
}
const rows = this.db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'lost\\_and\\_found%' ESCAPE '\\'",
)
.all() as Array<{ name?: unknown }>;
let dropped = 0;
for (const row of rows) {
const name = typeof row.name === "string" ? row.name : null;
if (!name) continue;
try {
// Table names from sqlite_master are trusted identifiers; quote defensively.
this.db.exec(`DROP TABLE IF EXISTS "${name.replace(/"/g, '""')}"`);
dropped++;
} catch (error) {
console.warn(`[fusion:db] Failed to drop orphan recovery table ${name}`, error);
}
}
return dropped;
}
/**
* Append-only operational log tables that grow without bound. These are the
* primary driver of database bloat (activityLog alone accrues tens of
* thousands of rows per active day) and the bigger the file, the longer every
* checkpoint/VACUUM spends in the write path where a node:sqlite crash can
* corrupt it. Each entry has an ISO-8601 `timestamp` column.
*/
private static readonly OPERATIONAL_LOG_TABLES = [
"activityLog",
"agentLogEntries",
"runAuditEvents",
"agentHeartbeats",
] as const;
/**
* Delete operational-log rows older than `retentionMs`. No-ops (returns an
* empty result) when `retentionMs <= 0` so callers can treat 0 as "disabled".
* Each table is pruned independently; a failure on one (e.g. absent in an
* older schema) is logged and skipped rather than aborting the sweep.
*/
pruneOperationalLogs(retentionMs: number): { deletedByTable: Record<string, number>; deletedTotal: number } {
const deletedByTable: Record<string, number> = {};
if (this.inMemory || !Number.isFinite(retentionMs) || retentionMs <= 0) {
return { deletedByTable, deletedTotal: 0 };
}
const cutoffIso = new Date(Date.now() - retentionMs).toISOString();
let deletedTotal = 0;
for (const table of Database.OPERATIONAL_LOG_TABLES) {
if (!this.tableExists(table)) continue;
try {
const result = this.db
.prepare(`DELETE FROM "${table}" WHERE timestamp < ?`)
.run(cutoffIso);
const changes = typeof result.changes === "bigint" ? Number(result.changes) : result.changes;
deletedByTable[table] = changes;
deletedTotal += changes;
} catch (error) {
console.warn(`[fusion:db] Failed to prune operational log table ${table}`, error);
}
}
return { deletedByTable, deletedTotal };
}
/**
* Initialize the database: create tables if they don't exist
* and seed meta values.
@@ -1649,6 +1848,10 @@ export class Database {
init(): void {
this.db.exec(SCHEMA_SQL);
// Drop scratch tables from any prior `.recover` so they don't accumulate
// across backup/restore cycles. Idempotent and cheap when none exist.
this.dropOrphanRecoveryTables();
this.scheduleBackgroundIntegrityCheck();
// Seed schemaVersion and lastModified idempotently
@@ -3847,6 +4050,14 @@ export class Database {
return this.getTableColumns(table).has(column);
}
/** Check whether a table exists in the current schema. */
private tableExists(table: string): boolean {
const row = this.db
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1")
.get(table);
return row !== undefined && row !== null;
}
/**
* Add a column to a table if it does not already exist.
*/

View File

@@ -403,6 +403,7 @@ export const DEFAULT_PROJECT_SETTINGS = {
showQuickChatFAB: false,
chatAutoCleanupDays: 0,
mailAutoCleanupDays: 0,
operationalLogRetentionDays: 30,
chatRoomRecentVerbatimMessages: 25,
chatRoomCompactionFetchLimit: 200,
chatRoomSummaryMaxChars: 3_000,

View File

@@ -1345,6 +1345,33 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
// Initialize SQLite database
if (!this._db) {
// Startup corruption guard: before opening, detect a malformed fusion.db
// (a node:sqlite SIGSEGV mid-write can leave the B-tree corrupt in a way
// that still opens) and rebuild it via sqlite3 .recover, preserving the
// corrupt original. Disk-backed only; opt out with FUSION_DISABLE_DB_AUTORECOVER.
if (!this.inMemoryDb && process.env.FUSION_DISABLE_DB_AUTORECOVER !== "1") {
try {
const recovery = Database.recoverIfCorrupt(this.fusionDir);
if (recovery.status === "recovered") {
storeLog.warn("Recovered corrupt fusion.db on startup", {
phase: "init:db-autorecover",
corruptBackupPath: recovery.corruptBackupPath,
errors: recovery.errors?.slice(0, 5),
});
} else if (recovery.status === "failed") {
storeLog.error("fusion.db is corrupt and automatic recovery failed", {
phase: "init:db-autorecover",
errors: recovery.errors?.slice(0, 5),
});
}
} catch (error) {
storeLog.warn("Startup db corruption guard threw — continuing to open", {
phase: "init:db-autorecover",
error: error instanceof Error ? error.message : String(error),
});
}
}
const db = new Database(this.fusionDir, { inMemory: this.inMemoryDb });
try {
db.init();
@@ -10811,6 +10838,15 @@ ${stepsSection}`;
return this.db.walCheckpoint(mode);
}
/**
* Delete append-only operational-log rows older than `retentionMs`. Returns
* zeroed counts when retention is disabled (`<= 0`). This is the primary lever
* against unbounded database growth — see `Database.pruneOperationalLogs`.
*/
pruneOperationalLogs(retentionMs: number): { deletedByTable: Record<string, number>; deletedTotal: number } {
return this.db.pruneOperationalLogs(retentionMs);
}
getRootDir(): string {
return this.rootDir;
}

View File

@@ -3696,6 +3696,11 @@ export interface ProjectSettings {
/** Number of days of inactivity before old inbox/outbox messages are auto-pruned.
* Allowed values: 0 (off, default) or one of 7 | 14 | 30 | 60 | 90. Uses messages.updatedAt inactivity age. */
mailAutoCleanupDays?: number;
/** Number of days to retain append-only operational-log rows (activityLog,
* agentLogEntries, runAuditEvents, agentHeartbeats) before periodic maintenance
* prunes them. These tables are the main driver of unbounded database growth.
* Default: 30. Set 0 to disable pruning. Uses each row's `timestamp` column. */
operationalLogRetentionDays?: number;
/** Number of most-recent chat-room messages kept verbatim in the responder transcript.
* Older messages are compacted into a summary block. Default: 12. */
chatRoomRecentVerbatimMessages?: number;