Harden fusion.db against recurring corruption
Root cause: node:sqlite SIGSEGVs inside pager_write leave the B-tree malformed in a way that still opens but fails integrity checks; large operational-log tables widen the write window where the crash strikes. - backup: verify every copy with PRAGMA quick_check, quarantine corrupt copies as *.corrupt, and never rotate out the last verified-good backup - db: add Database.recoverIfCorrupt() startup guard (wired into TaskStore.init, disk-backed only, opt out via FUSION_DISABLE_DB_AUTORECOVER) that rebuilds a malformed db via sqlite3 .recover, preserving the corrupt original; also fixes the latent `.recover main` invalid-option bug that made recoverDatabase() always fail - db: drop lost_and_found* scratch tables on init; add pruneOperationalLogs() - settings: add operationalLogRetentionDays (default 30, 0 = off) and prune activityLog/agentLogEntries/runAuditEvents/agentHeartbeats during maintenance - dashboard: expose retention in Settings -> Backups -> Database Maintenance Tests: backup 59/59, db 135/135 (incl. real corrupt->recover->reopen), self-healing cleanup/corruption 10/10, settings 77/77, SettingsModal 460/460. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
@@ -1,5 +1,6 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
||||
import { mkdtempSync, writeFileSync, existsSync, readFileSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { rm, mkdir, writeFile, readdir } from "node:fs/promises";
|
||||
@@ -18,6 +19,21 @@ import { RoutineStore } from "../routine-store.js";
|
||||
import { TaskStore } from "../store.js";
|
||||
import type { ProjectSettings } from "../types.js";
|
||||
|
||||
/**
|
||||
* Write a real SQLite database file so the production backup path's
|
||||
* `PRAGMA quick_check` verification passes. Falls back to a placeholder string
|
||||
* when the `sqlite3` CLI is unavailable — in that environment verification also
|
||||
* no-ops, so the backup still succeeds and the assertions hold either way.
|
||||
*/
|
||||
function writeTestDb(path: string): void {
|
||||
const result = spawnSync("sqlite3", [path, "CREATE TABLE IF NOT EXISTS t(x); INSERT INTO t VALUES (1);"], {
|
||||
encoding: "utf-8",
|
||||
});
|
||||
if (result.error || result.status !== 0) {
|
||||
writeFileSync(path, "dummy database content");
|
||||
}
|
||||
}
|
||||
|
||||
describe("BackupManager", () => {
|
||||
let tempDir: string;
|
||||
let fusionDir: string;
|
||||
@@ -36,6 +52,10 @@ describe("BackupManager", () => {
|
||||
writeFileSync(join(fusionDir, "fusion-central.db"), "dummy central database content");
|
||||
backupManager = new BackupManager(fusionDir, {
|
||||
centralDbPath: join(fusionDir, "fusion-central.db"),
|
||||
// These tests use dummy (non-SQLite) files as the source db, so the
|
||||
// PRAGMA quick_check verification cannot run against them. Integrity
|
||||
// verification has its own dedicated tests with real SQLite databases.
|
||||
verifyIntegrity: false,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -75,6 +95,7 @@ describe("BackupManager", () => {
|
||||
const manager = new BackupManager(fusionDir, {
|
||||
backupDir: customBackupDir,
|
||||
centralDbPath: join(fusionDir, "fusion-central.db"),
|
||||
verifyIntegrity: false,
|
||||
});
|
||||
|
||||
const customBackupPath = join(tempDir, customBackupDir);
|
||||
@@ -96,6 +117,7 @@ describe("BackupManager", () => {
|
||||
it("skips central backup when central DB is missing", async () => {
|
||||
const manager = new BackupManager(fusionDir, {
|
||||
centralDbPath: join(fusionDir, "does-not-exist.db"),
|
||||
verifyIntegrity: false,
|
||||
});
|
||||
const backup = await manager.createBackup();
|
||||
expect(backup.centralBackup).toEqual({ skipped: "missing" });
|
||||
@@ -105,6 +127,7 @@ describe("BackupManager", () => {
|
||||
const manager = new BackupManager(fusionDir, {
|
||||
centralDbPath: join(fusionDir, "fusion-central.db"),
|
||||
includeCentralDb: false,
|
||||
verifyIntegrity: false,
|
||||
});
|
||||
const backup = await manager.createBackup();
|
||||
expect(backup.centralBackup).toEqual({ skipped: "disabled" });
|
||||
@@ -132,7 +155,7 @@ describe("BackupManager", () => {
|
||||
it("continues central copy when checkpoint open fails", async () => {
|
||||
const notSqlitePath = join(fusionDir, "not-sqlite.db");
|
||||
writeFileSync(notSqlitePath, "definitely not sqlite");
|
||||
const manager = new BackupManager(fusionDir, { centralDbPath: notSqlitePath });
|
||||
const manager = new BackupManager(fusionDir, { centralDbPath: notSqlitePath, verifyIntegrity: false });
|
||||
const backup = await manager.createBackup();
|
||||
expect(backup.centralBackup && "filename" in backup.centralBackup).toBe(true);
|
||||
if (backup.centralBackup && "filename" in backup.centralBackup) {
|
||||
@@ -141,7 +164,7 @@ describe("BackupManager", () => {
|
||||
});
|
||||
|
||||
it("keeps project backup when central copy fails", async () => {
|
||||
const manager = new BackupManager(fusionDir, { centralDbPath: fusionDir });
|
||||
const manager = new BackupManager(fusionDir, { centralDbPath: fusionDir, verifyIntegrity: false });
|
||||
const backup = await manager.createBackup();
|
||||
expect(existsSync(backup.path)).toBe(true);
|
||||
expect(backup.centralBackup && "failed" in backup.centralBackup).toBe(true);
|
||||
@@ -279,7 +302,7 @@ describe("BackupManager", () => {
|
||||
});
|
||||
|
||||
it("should delete oldest backups exceeding retention", async () => {
|
||||
const manager = new BackupManager(fusionDir, { retention: 2, centralDbPath: join(fusionDir, "fusion-central.db") });
|
||||
const manager = new BackupManager(fusionDir, { retention: 2, centralDbPath: join(fusionDir, "fusion-central.db"), verifyIntegrity: false });
|
||||
|
||||
// Create 4 backups by advancing time deterministically
|
||||
for (let i = 0; i < 4; i++) {
|
||||
@@ -295,7 +318,7 @@ describe("BackupManager", () => {
|
||||
});
|
||||
|
||||
it("should keep the newest backups after cleanup", async () => {
|
||||
const manager = new BackupManager(fusionDir, { retention: 2, centralDbPath: join(fusionDir, "fusion-central.db") });
|
||||
const manager = new BackupManager(fusionDir, { retention: 2, centralDbPath: join(fusionDir, "fusion-central.db"), verifyIntegrity: false });
|
||||
|
||||
// Create 4 backups and record their names by advancing time
|
||||
const backupNames: string[] = [];
|
||||
@@ -318,7 +341,7 @@ describe("BackupManager", () => {
|
||||
});
|
||||
|
||||
it("deletes sibling central backup when deleting project backup", async () => {
|
||||
const manager = new BackupManager(fusionDir, { retention: 1, centralDbPath: join(fusionDir, "fusion-central.db") });
|
||||
const manager = new BackupManager(fusionDir, { retention: 1, centralDbPath: join(fusionDir, "fusion-central.db"), verifyIntegrity: false });
|
||||
await manager.createBackup();
|
||||
vi.setSystemTime(new Date("2026-01-01T00:00:01.000Z"));
|
||||
await manager.createBackup();
|
||||
@@ -427,6 +450,83 @@ describe("BackupManager", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("backup integrity verification", () => {
|
||||
let tempDir: string;
|
||||
let fusionDir: string;
|
||||
let sqlite3Available: boolean;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = mkdtempSync(join(tmpdir(), "kb-backup-verify-"));
|
||||
fusionDir = join(tempDir, ".fusion");
|
||||
await mkdir(fusionDir, { recursive: true });
|
||||
// Detect sqlite3 once: verification (and the corruption assertions that
|
||||
// depend on it) only run meaningfully where the CLI exists.
|
||||
const probe = spawnSync("sqlite3", ["--version"], { encoding: "utf-8" });
|
||||
sqlite3Available = !probe.error && probe.status === 0;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("verifyDatabaseIntegrity returns ok for a real SQLite db", async () => {
|
||||
if (!sqlite3Available) return;
|
||||
const { verifyDatabaseIntegrity } = await import("../backup.js");
|
||||
const dbPath = join(fusionDir, "fusion.db");
|
||||
writeTestDb(dbPath);
|
||||
const result = verifyDatabaseIntegrity(dbPath);
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.verified).toBe(true);
|
||||
});
|
||||
|
||||
it("verifyDatabaseIntegrity flags a non-SQLite file as corrupt", async () => {
|
||||
if (!sqlite3Available) return;
|
||||
const { verifyDatabaseIntegrity } = await import("../backup.js");
|
||||
const dbPath = join(fusionDir, "fusion.db");
|
||||
writeFileSync(dbPath, "definitely not a sqlite database");
|
||||
const result = verifyDatabaseIntegrity(dbPath);
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.verified).toBe(true);
|
||||
});
|
||||
|
||||
it("createBackup refuses to keep a corrupt copy and quarantines it", async () => {
|
||||
if (!sqlite3Available) return;
|
||||
// Source db is not a valid SQLite file → verification must reject it.
|
||||
writeFileSync(join(fusionDir, "fusion.db"), "corrupt source");
|
||||
const manager = new BackupManager(fusionDir, {
|
||||
centralDbPath: join(fusionDir, "fusion-central.db"),
|
||||
includeCentralDb: false,
|
||||
});
|
||||
|
||||
await expect(manager.createBackup()).rejects.toThrow(/verification failed/i);
|
||||
|
||||
// No listed (good) backup remains; the corrupt copy is quarantined as *.corrupt.
|
||||
const backups = await manager.listBackups();
|
||||
expect(backups).toEqual([]);
|
||||
const files = await readdir(join(tempDir, ".fusion/backups"));
|
||||
expect(files.some((f) => f.endsWith(".corrupt"))).toBe(true);
|
||||
});
|
||||
|
||||
it("cleanupOldBackups never deletes the last verified-good backup", async () => {
|
||||
if (!sqlite3Available) return;
|
||||
const backupDir = join(tempDir, ".fusion/backups");
|
||||
await mkdir(backupDir, { recursive: true });
|
||||
|
||||
// One real (good) backup, older than two newer corrupt ones.
|
||||
writeTestDb(join(backupDir, "fusion-2026-01-01-000000.db"));
|
||||
writeFileSync(join(backupDir, "fusion-2026-01-02-000000.db"), "corrupt newer 1");
|
||||
writeFileSync(join(backupDir, "fusion-2026-01-03-000000.db"), "corrupt newer 2");
|
||||
|
||||
const manager = new BackupManager(fusionDir, { retention: 2, includeCentralDb: false });
|
||||
await manager.cleanupOldBackups();
|
||||
|
||||
// Retention=2 would normally delete the oldest, but it is the only good one,
|
||||
// so it must survive.
|
||||
const remaining = (await manager.listBackups()).map((b) => b.filename);
|
||||
expect(remaining).toContain("fusion-2026-01-01-000000.db");
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateBackupFilename", () => {
|
||||
it("should generate filename with correct pattern", () => {
|
||||
const filename = generateBackupFilename();
|
||||
@@ -520,7 +620,7 @@ describe("createBackupManager", () => {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "kb-backup-test-"));
|
||||
const fusionDir = join(tempDir, ".fusion");
|
||||
await mkdir(fusionDir, { recursive: true });
|
||||
writeFileSync(join(fusionDir, "fusion.db"), "test");
|
||||
writeTestDb(join(fusionDir, "fusion.db"));
|
||||
|
||||
const settings: Partial<ProjectSettings> = {
|
||||
autoBackupDir: "custom/backups",
|
||||
@@ -547,7 +647,7 @@ describe("createBackupManager", () => {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "kb-backup-test-"));
|
||||
const fusionDir = join(tempDir, ".fusion");
|
||||
await mkdir(fusionDir, { recursive: true });
|
||||
writeFileSync(join(fusionDir, "fusion.db"), "test");
|
||||
writeTestDb(join(fusionDir, "fusion.db"));
|
||||
|
||||
const settings: Partial<ProjectSettings> = {
|
||||
autoBackupDir: ".kb/backups", // Legacy value
|
||||
@@ -572,7 +672,7 @@ describe("createBackupManager", () => {
|
||||
const tempDir = mkdtempSync(join(tmpdir(), "kb-backup-test-"));
|
||||
const fusionDir = join(tempDir, ".fusion");
|
||||
await mkdir(fusionDir, { recursive: true });
|
||||
writeFileSync(join(fusionDir, "fusion.db"), "test");
|
||||
writeTestDb(join(fusionDir, "fusion.db"));
|
||||
|
||||
const settings: Partial<ProjectSettings> = {
|
||||
autoBackupDir: ".kb/my-custom-backups", // Custom path, not the legacy default
|
||||
@@ -735,7 +835,7 @@ describe("runBackupCommand", () => {
|
||||
tempDir = mkdtempSync(join(tmpdir(), "kb-backup-test-"));
|
||||
fusionDir = join(tempDir, ".fusion");
|
||||
await mkdir(fusionDir, { recursive: true });
|
||||
writeFileSync(join(fusionDir, "fusion.db"), "dummy database content");
|
||||
writeTestDb(join(fusionDir, "fusion.db"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, it, expect, beforeAll, beforeEach, afterEach, afterAll, vi }
|
||||
import {
|
||||
Database,
|
||||
createDatabase,
|
||||
quickCheckSqliteFile,
|
||||
toJson,
|
||||
toJsonNullable,
|
||||
fromJson,
|
||||
@@ -11,13 +12,13 @@ import {
|
||||
} from "../db.js";
|
||||
import { DEFAULT_PROJECT_SETTINGS } from "../types.js";
|
||||
import { TaskStore } from "../store.js";
|
||||
import { mkdtempSync, existsSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { mkdtempSync, existsSync, readFileSync, rmSync, statSync, openSync, writeSync, closeSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { once } from "node:events";
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { ensureRoadmapSchema } from "../../../../plugins/fusion-plugin-roadmap/src/roadmap-schema.js";
|
||||
import { createSharedTaskStoreTestHarness } from "./store-test-helpers.js";
|
||||
|
||||
@@ -2854,3 +2855,165 @@ describe("migration v67 drops orphan project auth tables", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("Database operational-log retention and recovery-table cleanup", () => {
|
||||
let tmpDir: string;
|
||||
let fusionDir: string;
|
||||
let db: Database;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = makeTmpDir();
|
||||
fusionDir = join(tmpDir, ".fusion");
|
||||
db = new Database(fusionDir);
|
||||
db.init();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
try {
|
||||
db.close();
|
||||
} catch {
|
||||
// already closed
|
||||
}
|
||||
await cleanupTmpDirsAsync();
|
||||
});
|
||||
|
||||
function insertActivity(id: string, timestamp: string): void {
|
||||
db.prepare(
|
||||
"INSERT INTO activityLog (id, timestamp, type, details) VALUES (?, ?, 'test', '{}')",
|
||||
).run(id, timestamp);
|
||||
}
|
||||
|
||||
it("pruneOperationalLogs deletes rows older than the retention window", () => {
|
||||
const old = new Date(Date.now() - 200 * 86_400_000).toISOString();
|
||||
const recent = new Date(Date.now() - 1 * 86_400_000).toISOString();
|
||||
insertActivity("old-1", old);
|
||||
insertActivity("old-2", old);
|
||||
insertActivity("recent-1", recent);
|
||||
|
||||
const result = db.pruneOperationalLogs(90 * 86_400_000);
|
||||
expect(result.deletedTotal).toBe(2);
|
||||
expect(result.deletedByTable.activityLog).toBe(2);
|
||||
|
||||
const remaining = db.prepare("SELECT id FROM activityLog ORDER BY id").all() as Array<{ id: string }>;
|
||||
expect(remaining.map((r) => r.id)).toEqual(["recent-1"]);
|
||||
});
|
||||
|
||||
it("pruneOperationalLogs is a no-op when retention is disabled (<= 0)", () => {
|
||||
insertActivity("old-1", new Date(Date.now() - 200 * 86_400_000).toISOString());
|
||||
const result = db.pruneOperationalLogs(0);
|
||||
expect(result.deletedTotal).toBe(0);
|
||||
expect(db.prepare("SELECT count(*) AS c FROM activityLog").get()).toMatchObject({ c: 1 });
|
||||
});
|
||||
|
||||
it("dropOrphanRecoveryTables removes lost_and_found scratch tables", () => {
|
||||
db.exec("CREATE TABLE lost_and_found (x)");
|
||||
db.exec("CREATE TABLE lost_and_found_0 (x)");
|
||||
db.exec("CREATE TABLE lost_and_found_2 (x)");
|
||||
|
||||
const dropped = db.dropOrphanRecoveryTables();
|
||||
expect(dropped).toBe(3);
|
||||
|
||||
const tables = db
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lost_and_found%'")
|
||||
.all();
|
||||
expect(tables).toEqual([]);
|
||||
});
|
||||
|
||||
it("init() drops pre-existing lost_and_found tables on open", () => {
|
||||
db.exec("CREATE TABLE lost_and_found_0 (x)");
|
||||
db.close();
|
||||
|
||||
const reopened = new Database(fusionDir);
|
||||
reopened.init();
|
||||
try {
|
||||
const tables = reopened
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lost_and_found%'")
|
||||
.all();
|
||||
expect(tables).toEqual([]);
|
||||
} finally {
|
||||
reopened.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("Database.recoverIfCorrupt startup guard", () => {
|
||||
let tmpDir: string;
|
||||
let fusionDir: string;
|
||||
const sqlite3Available = (() => {
|
||||
const probe = spawnSync("sqlite3", ["--version"], { encoding: "utf-8" });
|
||||
return !probe.error && probe.status === 0;
|
||||
})();
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = makeTmpDir();
|
||||
fusionDir = join(tmpDir, ".fusion");
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await cleanupTmpDirsAsync();
|
||||
});
|
||||
|
||||
it("returns 'absent' when no database exists", () => {
|
||||
const result = Database.recoverIfCorrupt(fusionDir);
|
||||
expect(result.status).toBe("absent");
|
||||
});
|
||||
|
||||
it("returns 'healthy' for an intact database", () => {
|
||||
if (!sqlite3Available) return;
|
||||
const db = new Database(fusionDir);
|
||||
db.init();
|
||||
db.close();
|
||||
const result = Database.recoverIfCorrupt(fusionDir);
|
||||
expect(result.status).toBe("healthy");
|
||||
});
|
||||
|
||||
it("rebuilds a malformed database and preserves the corrupt original", () => {
|
||||
if (!sqlite3Available) return;
|
||||
const dbPath = join(fusionDir, "fusion.db");
|
||||
const db = new Database(fusionDir);
|
||||
db.init();
|
||||
// Span many pages so mid-file corruption lands on a B-tree page.
|
||||
for (let i = 0; i < 3000; i++) {
|
||||
db.prepare("INSERT INTO activityLog (id, timestamp, type, details) VALUES (?, ?, 'test', '{}')").run(
|
||||
`row-${i}`,
|
||||
new Date().toISOString(),
|
||||
);
|
||||
}
|
||||
db.walCheckpoint("TRUNCATE");
|
||||
db.close();
|
||||
|
||||
// Corrupt an interior region while leaving the header page intact.
|
||||
const size = statSync(dbPath).size;
|
||||
const fd = openSync(dbPath, "r+");
|
||||
try {
|
||||
const garbage = Buffer.alloc(16 * 1024, 0xab);
|
||||
writeSync(fd, garbage, 0, garbage.length, Math.floor(size / 2));
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
|
||||
// If the corruption didn't trip quick_check on this build, skip rather
|
||||
// than assert flakily.
|
||||
const pre = quickCheckSqliteFile(dbPath);
|
||||
if (pre.ok) return;
|
||||
|
||||
const result = Database.recoverIfCorrupt(fusionDir);
|
||||
expect(result.status).toBe("recovered");
|
||||
expect(result.corruptBackupPath).toBeDefined();
|
||||
expect(existsSync(result.corruptBackupPath!)).toBe(true);
|
||||
// The swapped-in database must now be clean.
|
||||
expect(quickCheckSqliteFile(dbPath).ok).toBe(true);
|
||||
// Stale sidecars must not linger.
|
||||
expect(existsSync(`${dbPath}-wal`)).toBe(false);
|
||||
|
||||
// And it must open and answer queries.
|
||||
const reopened = new Database(fusionDir);
|
||||
reopened.init();
|
||||
try {
|
||||
const row = reopened.prepare("SELECT count(*) AS c FROM activityLog").get() as { c: number };
|
||||
expect(row.c).toBeGreaterThan(0);
|
||||
} finally {
|
||||
reopened.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { cp, mkdir, readdir, stat, unlink } from "node:fs/promises";
|
||||
import { cp, mkdir, readdir, rename, stat, unlink } from "node:fs/promises";
|
||||
import { existsSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { join } from "node:path";
|
||||
import { CronExpressionParser } from "cron-parser";
|
||||
import { getDefaultCentralDbPath } from "./central-db.js";
|
||||
@@ -34,6 +35,12 @@ export interface BackupOptions {
|
||||
retention?: number;
|
||||
centralDbPath?: string;
|
||||
includeCentralDb?: boolean;
|
||||
/**
|
||||
* Verify each backup copy with `PRAGMA quick_check` and refuse to keep or
|
||||
* rotate-in a corrupt copy. Defaults to true. Set false only where the
|
||||
* source is intentionally not a real SQLite file (e.g. unit tests).
|
||||
*/
|
||||
verifyIntegrity?: boolean;
|
||||
}
|
||||
|
||||
export class BackupManager {
|
||||
@@ -42,6 +49,7 @@ export class BackupManager {
|
||||
private retention: number;
|
||||
private centralDbPath: string;
|
||||
private includeCentralDb: boolean;
|
||||
private verifyIntegrity: boolean;
|
||||
|
||||
constructor(fusionDir: string, options?: BackupOptions) {
|
||||
this.fusionDir = fusionDir;
|
||||
@@ -49,6 +57,7 @@ export class BackupManager {
|
||||
this.retention = options?.retention ?? 7;
|
||||
this.centralDbPath = options?.centralDbPath ?? join(this.fusionDir, "..", ".fusion", "fusion-central.db");
|
||||
this.includeCentralDb = options?.includeCentralDb ?? true;
|
||||
this.verifyIntegrity = options?.verifyIntegrity ?? true;
|
||||
}
|
||||
|
||||
private getBackupDirPath(): string {
|
||||
@@ -89,6 +98,21 @@ export class BackupManager {
|
||||
|
||||
await copyLiveDatabase(sourcePath, targetPath);
|
||||
|
||||
// Verify the freshly-written copy. A copy of a live WAL db can capture a
|
||||
// torn/corrupt main file; refusing to keep a corrupt backup guarantees
|
||||
// that every retained `fusion-*.db` is restorable and that a corrupt copy
|
||||
// is never counted as the "last known-good" by cleanupOldBackups().
|
||||
if (this.verifyIntegrity) {
|
||||
const integrity = verifyDatabaseIntegrity(targetPath);
|
||||
if (!integrity.ok) {
|
||||
await quarantineCorruptBackup(targetPath);
|
||||
throw new Error(
|
||||
`Backup verification failed for ${filename}: ${integrity.error ?? "database disk image is malformed"}. ` +
|
||||
"The source database may be corrupt; the unusable copy was quarantined as *.corrupt.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const stats = await stat(targetPath);
|
||||
const backup: BackupInfo = {
|
||||
filename,
|
||||
@@ -113,6 +137,16 @@ export class BackupManager {
|
||||
try {
|
||||
await copyLiveDatabase(this.centralDbPath, centralTargetPath);
|
||||
|
||||
if (this.verifyIntegrity) {
|
||||
const centralIntegrity = verifyDatabaseIntegrity(centralTargetPath);
|
||||
if (!centralIntegrity.ok) {
|
||||
await quarantineCorruptBackup(centralTargetPath);
|
||||
throw new Error(
|
||||
`central DB verification failed: ${centralIntegrity.error ?? "database disk image is malformed"}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const centralStats = await stat(centralTargetPath);
|
||||
backup.centralBackup = {
|
||||
filename: centralFilename,
|
||||
@@ -224,6 +258,28 @@ export class BackupManager {
|
||||
});
|
||||
const toDelete = sorted.slice(0, sorted.length - this.retention);
|
||||
|
||||
// Never rotate out the last known-good backup. The retained set is the
|
||||
// newest `retention` files, but if every one of them fails verification
|
||||
// (e.g. a run of corrupt copies from a flaky source db) we must protect the
|
||||
// newest verifiably-good backup from deletion even though it falls outside
|
||||
// the retention window. Verification is lazy: in the common case the newest
|
||||
// kept backup is good and we run exactly one check.
|
||||
if (this.verifyIntegrity) {
|
||||
const kept = sorted.slice(sorted.length - this.retention);
|
||||
const keptHasGood = kept
|
||||
.slice()
|
||||
.reverse()
|
||||
.some((b) => verifyDatabaseIntegrity(b.path).ok);
|
||||
if (!keptHasGood) {
|
||||
for (let i = toDelete.length - 1; i >= 0; i--) {
|
||||
if (verifyDatabaseIntegrity(toDelete[i].path).ok) {
|
||||
toDelete.splice(i, 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let deletedCount = 0;
|
||||
for (const backup of toDelete) {
|
||||
try {
|
||||
@@ -352,6 +408,81 @@ async function copyLiveDatabase(sourcePath: string, targetPath: string): Promise
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of an on-disk SQLite integrity verification.
|
||||
*
|
||||
* `verified` distinguishes "we ran the check" from "we couldn't run it". When
|
||||
* the `sqlite3` CLI is unavailable (e.g. a packaged environment with no system
|
||||
* binary on PATH) we return `{ ok: true, verified: false }` so verification
|
||||
* degrades to a no-op rather than blocking backups or rotation.
|
||||
*/
|
||||
export interface DatabaseIntegrityResult {
|
||||
ok: boolean;
|
||||
verified: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a SQLite database file with `PRAGMA quick_check`.
|
||||
*
|
||||
* Uses the `sqlite3` CLI (the same dependency the recovery path relies on) so
|
||||
* we never open the file through the live `node:sqlite` connection — opening a
|
||||
* WAL-mode copy through node:sqlite would replay/checkpoint pages and mutate
|
||||
* the very backup we are trying to validate. `quick_check` is far cheaper than
|
||||
* a full `integrity_check` but still detects the B-tree malformations
|
||||
* ("rowid out of order", "2nd reference to page") that node:sqlite SIGSEGVs
|
||||
* leave behind.
|
||||
*/
|
||||
export function verifyDatabaseIntegrity(dbPath: string): DatabaseIntegrityResult {
|
||||
if (!existsSync(dbPath)) {
|
||||
return { ok: false, verified: true, error: "file does not exist" };
|
||||
}
|
||||
|
||||
const result = spawnSync("sqlite3", [dbPath, "PRAGMA quick_check;"], {
|
||||
encoding: "utf-8",
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
|
||||
// ENOENT (or any spawn error) means the sqlite3 binary is unavailable — we
|
||||
// cannot verify, so treat as a non-blocking pass.
|
||||
if (result.error) {
|
||||
return { ok: true, verified: false, error: result.error.message };
|
||||
}
|
||||
|
||||
const stdout = (result.stdout ?? "").trim();
|
||||
if (result.status !== 0) {
|
||||
return {
|
||||
ok: false,
|
||||
verified: true,
|
||||
error: stdout || (result.stderr ?? "").trim() || `sqlite3 exited ${result.status}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (stdout.toLowerCase() === "ok") {
|
||||
return { ok: true, verified: true };
|
||||
}
|
||||
|
||||
return { ok: false, verified: true, error: stdout.split("\n").slice(0, 3).join(" | ") };
|
||||
}
|
||||
|
||||
/** Move a verifiably-corrupt backup copy aside so it never masquerades as good. */
|
||||
async function quarantineCorruptBackup(targetPath: string): Promise<void> {
|
||||
for (const suffix of ["", "-wal", "-shm"]) {
|
||||
const path = `${targetPath}${suffix}`;
|
||||
if (!existsSync(path)) continue;
|
||||
try {
|
||||
await rename(path, `${path}.corrupt`);
|
||||
} catch {
|
||||
// Best effort — fall back to deleting so a corrupt copy is never listed.
|
||||
try {
|
||||
await unlink(path);
|
||||
} catch {
|
||||
// Ignore.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sortBackupsNewestFirst(backups: BackupFileInfo[]): BackupFileInfo[] {
|
||||
return backups.sort((a, b) => {
|
||||
const timeCompare = b.createdAt.localeCompare(a.createdAt);
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
import { DatabaseSync } from "./sqlite-adapter.js";
|
||||
import { basename, isAbsolute, join } from "node:path";
|
||||
import { mkdirSync, existsSync, statSync } from "node:fs";
|
||||
import { mkdirSync, existsSync, statSync, renameSync, rmSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { DEFAULT_PROJECT_SETTINGS } from "./types.js";
|
||||
@@ -1336,6 +1336,44 @@ export const SCHEMA_COMPAT_FINGERPRINT = createHash("sha1")
|
||||
)
|
||||
.digest("hex");
|
||||
|
||||
/** Compact UTC timestamp (YYYY-MM-DD-HHmmss) for recovery artifact filenames. */
|
||||
function formatDbRecoveryTimestamp(date: Date): string {
|
||||
const y = date.getUTCFullYear();
|
||||
const m = String(date.getUTCMonth() + 1).padStart(2, "0");
|
||||
const d = String(date.getUTCDate()).padStart(2, "0");
|
||||
const hh = String(date.getUTCHours()).padStart(2, "0");
|
||||
const mm = String(date.getUTCMinutes()).padStart(2, "0");
|
||||
const ss = String(date.getUTCSeconds()).padStart(2, "0");
|
||||
return `${y}-${m}-${d}-${hh}${mm}${ss}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `PRAGMA quick_check` against a SQLite file via the `sqlite3` CLI without
|
||||
* opening a live connection (so we never replay/checkpoint a WAL onto it).
|
||||
* `verified=false` means the check could not run (sqlite3 unavailable) and the
|
||||
* caller should treat the result as non-blocking.
|
||||
*/
|
||||
export function quickCheckSqliteFile(dbPath: string): { ok: boolean; verified: boolean; errors?: string[] } {
|
||||
if (!existsSync(dbPath)) {
|
||||
return { ok: false, verified: true, errors: ["file does not exist"] };
|
||||
}
|
||||
const result = spawnSync("sqlite3", [dbPath, "PRAGMA quick_check;"], {
|
||||
encoding: "utf-8",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
if (result.error) {
|
||||
return { ok: true, verified: false };
|
||||
}
|
||||
const stdout = (result.stdout ?? "").trim();
|
||||
if (result.status !== 0) {
|
||||
return { ok: false, verified: true, errors: [stdout || (result.stderr ?? "").trim() || `sqlite3 exited ${result.status}`] };
|
||||
}
|
||||
if (stdout.toLowerCase() === "ok") {
|
||||
return { ok: true, verified: true };
|
||||
}
|
||||
return { ok: false, verified: true, errors: stdout.split("\n").slice(0, 5) };
|
||||
}
|
||||
|
||||
// ── Database Class ───────────────────────────────────────────────────
|
||||
|
||||
type SharedIntegrityCheckState = {
|
||||
@@ -1581,9 +1619,9 @@ export class Database {
|
||||
return false;
|
||||
}
|
||||
|
||||
const recoveredSql = spawnSync("sqlite3", ["-cmd", ".recover main", this.dbPath], {
|
||||
const recoveredSql = spawnSync("sqlite3", [this.dbPath, ".recover"], {
|
||||
encoding: "utf-8",
|
||||
maxBuffer: 50 * 1024 * 1024,
|
||||
maxBuffer: 256 * 1024 * 1024,
|
||||
});
|
||||
if (recoveredSql.status !== 0 || !recoveredSql.stdout) {
|
||||
return false;
|
||||
@@ -1592,12 +1630,91 @@ export class Database {
|
||||
const rebuilt = spawnSync("sqlite3", [outputPath], {
|
||||
input: recoveredSql.stdout,
|
||||
encoding: "utf-8",
|
||||
maxBuffer: 50 * 1024 * 1024,
|
||||
maxBuffer: 256 * 1024 * 1024,
|
||||
});
|
||||
|
||||
return rebuilt.status === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Startup guard: detect a malformed `fusion.db` and rebuild it via
|
||||
* `sqlite3 .recover` BEFORE any connection is opened for normal use.
|
||||
*
|
||||
* This is the automated form of the manual recovery: a node:sqlite SIGSEGV
|
||||
* mid-write can leave the B-tree malformed in a way that still *opens* and
|
||||
* answers simple queries (so a sentinel SELECT won't catch it) — only an
|
||||
* integrity/quick check does. When corruption is found we:
|
||||
* 1. recover the readable data into a fresh file,
|
||||
* 2. verify the rebuilt file passes quick_check,
|
||||
* 3. preserve the corrupt original as `fusion.db.corrupt-<ts>`,
|
||||
* 4. atomically swap the rebuilt file into place and drop stale -wal/-shm.
|
||||
*
|
||||
* Must run with no open connection to `fusion.db`. Returns a status describing
|
||||
* what happened; on `failed` the original file is left untouched for manual
|
||||
* inspection. `sqlite3` CLI absence yields `unverified` (non-blocking no-op).
|
||||
*/
|
||||
static recoverIfCorrupt(fusionDir: string): {
|
||||
status: "absent" | "healthy" | "unverified" | "recovered" | "failed";
|
||||
corruptBackupPath?: string;
|
||||
recoveredPath?: string;
|
||||
errors?: string[];
|
||||
} {
|
||||
const dbPath = join(fusionDir, "fusion.db");
|
||||
if (!existsSync(dbPath)) {
|
||||
return { status: "absent" };
|
||||
}
|
||||
|
||||
const check = quickCheckSqliteFile(dbPath);
|
||||
if (!check.verified) {
|
||||
return { status: "unverified" };
|
||||
}
|
||||
if (check.ok) {
|
||||
return { status: "healthy" };
|
||||
}
|
||||
|
||||
// Corruption confirmed — attempt an offline rebuild.
|
||||
const ts = formatDbRecoveryTimestamp(new Date());
|
||||
const recoveredPath = `${dbPath}.recovered-${ts}`;
|
||||
|
||||
const recoveredSql = spawnSync("sqlite3", [dbPath, ".recover"], {
|
||||
encoding: "utf-8",
|
||||
maxBuffer: 256 * 1024 * 1024,
|
||||
});
|
||||
if (recoveredSql.status !== 0 || !recoveredSql.stdout) {
|
||||
return { status: "failed", errors: check.errors };
|
||||
}
|
||||
const rebuilt = spawnSync("sqlite3", [recoveredPath], {
|
||||
input: recoveredSql.stdout,
|
||||
encoding: "utf-8",
|
||||
maxBuffer: 256 * 1024 * 1024,
|
||||
});
|
||||
if (rebuilt.status !== 0) {
|
||||
try { rmSync(recoveredPath, { force: true }); } catch { /* ignore */ }
|
||||
return { status: "failed", errors: check.errors };
|
||||
}
|
||||
|
||||
// Refuse to swap in a rebuild that is itself not clean.
|
||||
const verifyRebuilt = quickCheckSqliteFile(recoveredPath);
|
||||
if (verifyRebuilt.verified && !verifyRebuilt.ok) {
|
||||
try { rmSync(recoveredPath, { force: true }); } catch { /* ignore */ }
|
||||
return { status: "failed", errors: check.errors };
|
||||
}
|
||||
|
||||
try {
|
||||
const corruptBackupPath = `${dbPath}.corrupt-${ts}`;
|
||||
renameSync(dbPath, corruptBackupPath);
|
||||
// Stale WAL/SHM belong to the corrupt file; SQLite must not replay them
|
||||
// onto the rebuilt database.
|
||||
try { rmSync(`${dbPath}-wal`, { force: true }); } catch { /* ignore */ }
|
||||
try { rmSync(`${dbPath}-shm`, { force: true }); } catch { /* ignore */ }
|
||||
renameSync(recoveredPath, dbPath);
|
||||
return { status: "recovered", corruptBackupPath, errors: check.errors };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return { status: "failed", errors: [...(check.errors ?? []), message] };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run WAL truncation + VACUUM and report compaction stats.
|
||||
*
|
||||
@@ -1642,6 +1759,88 @@ export class Database {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop scratch tables left behind by `sqlite3 .recover`.
|
||||
*
|
||||
* Recovery emits `lost_and_found` / `lost_and_found_N` tables holding orphaned
|
||||
* rows it could not attribute to a real table. They are never part of the
|
||||
* Fusion schema, but a recovered db that gets backed up and restored carries
|
||||
* them forward indefinitely — on this database they had accumulated ~250K dead
|
||||
* rows across prior recoveries, inflating file size and every integrity check.
|
||||
* Returns the number of scratch tables dropped.
|
||||
*/
|
||||
dropOrphanRecoveryTables(): number {
|
||||
if (this.inMemory) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'lost\\_and\\_found%' ESCAPE '\\'",
|
||||
)
|
||||
.all() as Array<{ name?: unknown }>;
|
||||
|
||||
let dropped = 0;
|
||||
for (const row of rows) {
|
||||
const name = typeof row.name === "string" ? row.name : null;
|
||||
if (!name) continue;
|
||||
try {
|
||||
// Table names from sqlite_master are trusted identifiers; quote defensively.
|
||||
this.db.exec(`DROP TABLE IF EXISTS "${name.replace(/"/g, '""')}"`);
|
||||
dropped++;
|
||||
} catch (error) {
|
||||
console.warn(`[fusion:db] Failed to drop orphan recovery table ${name}`, error);
|
||||
}
|
||||
}
|
||||
return dropped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append-only operational log tables that grow without bound. These are the
|
||||
* primary driver of database bloat (activityLog alone accrues tens of
|
||||
* thousands of rows per active day) and the bigger the file, the longer every
|
||||
* checkpoint/VACUUM spends in the write path where a node:sqlite crash can
|
||||
* corrupt it. Each entry has an ISO-8601 `timestamp` column.
|
||||
*/
|
||||
private static readonly OPERATIONAL_LOG_TABLES = [
|
||||
"activityLog",
|
||||
"agentLogEntries",
|
||||
"runAuditEvents",
|
||||
"agentHeartbeats",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Delete operational-log rows older than `retentionMs`. No-ops (returns an
|
||||
* empty result) when `retentionMs <= 0` so callers can treat 0 as "disabled".
|
||||
* Each table is pruned independently; a failure on one (e.g. absent in an
|
||||
* older schema) is logged and skipped rather than aborting the sweep.
|
||||
*/
|
||||
pruneOperationalLogs(retentionMs: number): { deletedByTable: Record<string, number>; deletedTotal: number } {
|
||||
const deletedByTable: Record<string, number> = {};
|
||||
if (this.inMemory || !Number.isFinite(retentionMs) || retentionMs <= 0) {
|
||||
return { deletedByTable, deletedTotal: 0 };
|
||||
}
|
||||
|
||||
const cutoffIso = new Date(Date.now() - retentionMs).toISOString();
|
||||
let deletedTotal = 0;
|
||||
|
||||
for (const table of Database.OPERATIONAL_LOG_TABLES) {
|
||||
if (!this.tableExists(table)) continue;
|
||||
try {
|
||||
const result = this.db
|
||||
.prepare(`DELETE FROM "${table}" WHERE timestamp < ?`)
|
||||
.run(cutoffIso);
|
||||
const changes = typeof result.changes === "bigint" ? Number(result.changes) : result.changes;
|
||||
deletedByTable[table] = changes;
|
||||
deletedTotal += changes;
|
||||
} catch (error) {
|
||||
console.warn(`[fusion:db] Failed to prune operational log table ${table}`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return { deletedByTable, deletedTotal };
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the database: create tables if they don't exist
|
||||
* and seed meta values.
|
||||
@@ -1649,6 +1848,10 @@ export class Database {
|
||||
init(): void {
|
||||
this.db.exec(SCHEMA_SQL);
|
||||
|
||||
// Drop scratch tables from any prior `.recover` so they don't accumulate
|
||||
// across backup/restore cycles. Idempotent and cheap when none exist.
|
||||
this.dropOrphanRecoveryTables();
|
||||
|
||||
this.scheduleBackgroundIntegrityCheck();
|
||||
|
||||
// Seed schemaVersion and lastModified idempotently
|
||||
@@ -3847,6 +4050,14 @@ export class Database {
|
||||
return this.getTableColumns(table).has(column);
|
||||
}
|
||||
|
||||
/** Check whether a table exists in the current schema. */
|
||||
private tableExists(table: string): boolean {
|
||||
const row = this.db
|
||||
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1")
|
||||
.get(table);
|
||||
return row !== undefined && row !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a column to a table if it does not already exist.
|
||||
*/
|
||||
|
||||
@@ -403,6 +403,7 @@ export const DEFAULT_PROJECT_SETTINGS = {
|
||||
showQuickChatFAB: false,
|
||||
chatAutoCleanupDays: 0,
|
||||
mailAutoCleanupDays: 0,
|
||||
operationalLogRetentionDays: 30,
|
||||
chatRoomRecentVerbatimMessages: 25,
|
||||
chatRoomCompactionFetchLimit: 200,
|
||||
chatRoomSummaryMaxChars: 3_000,
|
||||
|
||||
@@ -1345,6 +1345,33 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
|
||||
|
||||
// Initialize SQLite database
|
||||
if (!this._db) {
|
||||
// Startup corruption guard: before opening, detect a malformed fusion.db
|
||||
// (a node:sqlite SIGSEGV mid-write can leave the B-tree corrupt in a way
|
||||
// that still opens) and rebuild it via sqlite3 .recover, preserving the
|
||||
// corrupt original. Disk-backed only; opt out with FUSION_DISABLE_DB_AUTORECOVER.
|
||||
if (!this.inMemoryDb && process.env.FUSION_DISABLE_DB_AUTORECOVER !== "1") {
|
||||
try {
|
||||
const recovery = Database.recoverIfCorrupt(this.fusionDir);
|
||||
if (recovery.status === "recovered") {
|
||||
storeLog.warn("Recovered corrupt fusion.db on startup", {
|
||||
phase: "init:db-autorecover",
|
||||
corruptBackupPath: recovery.corruptBackupPath,
|
||||
errors: recovery.errors?.slice(0, 5),
|
||||
});
|
||||
} else if (recovery.status === "failed") {
|
||||
storeLog.error("fusion.db is corrupt and automatic recovery failed", {
|
||||
phase: "init:db-autorecover",
|
||||
errors: recovery.errors?.slice(0, 5),
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
storeLog.warn("Startup db corruption guard threw — continuing to open", {
|
||||
phase: "init:db-autorecover",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const db = new Database(this.fusionDir, { inMemory: this.inMemoryDb });
|
||||
try {
|
||||
db.init();
|
||||
@@ -10811,6 +10838,15 @@ ${stepsSection}`;
|
||||
return this.db.walCheckpoint(mode);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete append-only operational-log rows older than `retentionMs`. Returns
|
||||
* zeroed counts when retention is disabled (`<= 0`). This is the primary lever
|
||||
* against unbounded database growth — see `Database.pruneOperationalLogs`.
|
||||
*/
|
||||
pruneOperationalLogs(retentionMs: number): { deletedByTable: Record<string, number>; deletedTotal: number } {
|
||||
return this.db.pruneOperationalLogs(retentionMs);
|
||||
}
|
||||
|
||||
getRootDir(): string {
|
||||
return this.rootDir;
|
||||
}
|
||||
|
||||
@@ -3696,6 +3696,11 @@ export interface ProjectSettings {
|
||||
/** Number of days of inactivity before old inbox/outbox messages are auto-pruned.
|
||||
* Allowed values: 0 (off, default) or one of 7 | 14 | 30 | 60 | 90. Uses messages.updatedAt inactivity age. */
|
||||
mailAutoCleanupDays?: number;
|
||||
/** Number of days to retain append-only operational-log rows (activityLog,
|
||||
* agentLogEntries, runAuditEvents, agentHeartbeats) before periodic maintenance
|
||||
* prunes them. These tables are the main driver of unbounded database growth.
|
||||
* Default: 30. Set 0 to disable pruning. Uses each row's `timestamp` column. */
|
||||
operationalLogRetentionDays?: number;
|
||||
/** Number of most-recent chat-room messages kept verbatim in the responder transcript.
|
||||
* Older messages are compacted into a summary block. Default: 12. */
|
||||
chatRoomRecentVerbatimMessages?: number;
|
||||
|
||||
@@ -6106,6 +6106,31 @@ export function SettingsModal({
|
||||
)}
|
||||
</div>
|
||||
|
||||
<h4 className="settings-section-heading settings-section-heading--spaced">Database Maintenance</h4>
|
||||
<div className="form-group">
|
||||
<label htmlFor="operationalLogRetentionDays">Operational log retention</label>
|
||||
<select
|
||||
id="operationalLogRetentionDays"
|
||||
className="select"
|
||||
value={form.operationalLogRetentionDays ?? 0}
|
||||
onChange={(e) =>
|
||||
setForm((f) => ({ ...f, operationalLogRetentionDays: Number(e.target.value) || 0 }))
|
||||
}
|
||||
>
|
||||
<option value={0}>Off</option>
|
||||
<option value={30}>30 days</option>
|
||||
<option value={60}>60 days</option>
|
||||
<option value={90}>90 days</option>
|
||||
<option value={180}>180 days</option>
|
||||
<option value={365}>365 days</option>
|
||||
</select>
|
||||
<small>
|
||||
Prune append-only operational logs (activity log, agent logs, run audit, heartbeats) older than this
|
||||
many days during periodic maintenance. Keeps the database from growing without bound — large databases
|
||||
are slower to checkpoint and more prone to corruption. Default: 30 days.
|
||||
</small>
|
||||
</div>
|
||||
|
||||
<h4 className="settings-section-heading">Memory Backups</h4>
|
||||
<div className="form-group">
|
||||
<label htmlFor="memoryBackupEnabled" className="checkbox-label">
|
||||
|
||||
@@ -1626,6 +1626,22 @@ export class SelfHealingManager {
|
||||
log.log(`Maintenance batch 1 step "cleanup-old-mail" succeeded — messagesDeleted=${messagesDeleted}`);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "prune-operational-logs",
|
||||
fn: async () => {
|
||||
const days = Number(settings.operationalLogRetentionDays ?? 0);
|
||||
if (!Number.isFinite(days) || days <= 0) {
|
||||
log.log("Maintenance batch 1 step \"prune-operational-logs\" skipped — operationalLogRetentionDays is not enabled");
|
||||
return;
|
||||
}
|
||||
const { deletedTotal, deletedByTable } = this.store.pruneOperationalLogs(days * 86_400_000);
|
||||
const detail = Object.entries(deletedByTable)
|
||||
.filter(([, n]) => n > 0)
|
||||
.map(([t, n]) => `${t}=${n}`)
|
||||
.join(" ");
|
||||
log.log(`Maintenance batch 1 step "prune-operational-logs" succeeded — deleted=${deletedTotal}${detail ? ` (${detail})` : ""}`);
|
||||
},
|
||||
},
|
||||
{ name: "checkpoint-wal", fn: () => Promise.resolve(this.checkpointWal()) },
|
||||
{ name: "enforce-worktree-cap", fn: () => this.enforceWorktreeCap() },
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user