feat(FN-2895): add FTS5 corruption recovery path

- Add Database APIs to rebuild the tasks_fts index and detect FTS5 corruption signatures
- Retry task upserts once after rebuilding FTS5 when corruption errors are encountered
- Extend TaskStore health checks to include FTS5 integrity verification
- Add regression tests for rebuild/integrity behavior and upsert recovery, plus a patch changeset for @runfusion/fusion
This commit is contained in:
Fusion
2026-04-28 16:51:04 -07:00
committed by gsxdsm
parent 0e2480980b
commit cc9181db47
5 changed files with 248 additions and 4 deletions

View File

@@ -1,4 +1,4 @@
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { Database, createDatabase, toJson, toJsonNullable, fromJson, normalizeTaskComments } from "../db.js";
import { DEFAULT_PROJECT_SETTINGS } from "../types.js";
import { mkdtempSync, existsSync } from "node:fs";
@@ -1445,6 +1445,88 @@ describe("FTS5 full-text search", () => {
expect(ftsRow).toBeDefined();
expect(ftsRow.comments).toContain("xylophone");
});
it("rebuildFts5Index recreates and repopulates the FTS table", () => {
db.prepare(`
INSERT INTO tasks (id, title, description, "column", createdAt, updatedAt)
VALUES ('FN-FTS-REBUILD', 'Rebuild title', 'Rebuild description', 'todo', '2025-01-01T00:00:00.000Z', '2025-01-01T00:00:00.000Z')
`).run();
db.exec("DROP TRIGGER IF EXISTS tasks_fts_ai");
db.exec("DROP TRIGGER IF EXISTS tasks_fts_au");
db.exec("DROP TRIGGER IF EXISTS tasks_fts_ad");
db.exec("DROP TABLE IF EXISTS tasks_fts");
db.exec(`
CREATE VIRTUAL TABLE tasks_fts USING fts5(
id,
title,
description,
comments,
content='tasks',
content_rowid='rowid'
)
`);
const missingTrigger = db.prepare(
"SELECT name FROM sqlite_master WHERE type='trigger' AND name='tasks_fts_ai'"
).get() as { name: string } | undefined;
expect(missingTrigger).toBeUndefined();
expect(db.rebuildFts5Index()).toBe(true);
const searchRows = db.prepare(`
SELECT t.id FROM tasks t
JOIN tasks_fts fts ON t.rowid = fts.rowid
WHERE tasks_fts MATCH 'Rebuild'
`).all() as Array<{ id: string }>;
expect(searchRows.some((row) => row.id === "FN-FTS-REBUILD")).toBe(true);
});
it("checkFts5Integrity returns true for healthy index", () => {
expect(db.checkFts5Integrity()).toBe(true);
});
it("checkFts5Integrity returns false when integrity-check command fails", () => {
const execSpy = vi.spyOn((db as any).db, "exec");
execSpy.mockImplementation((sql: string) => {
if (sql.includes("integrity-check")) {
throw new Error("corruption found reading blob");
}
return undefined as any;
});
expect(db.checkFts5Integrity()).toBe(false);
});
it("isFts5CorruptionError detects known corruption signatures", () => {
expect(db.isFts5CorruptionError(new Error("database disk image is malformed"))).toBe(true);
expect(db.isFts5CorruptionError(new Error("FTS5 index corrupt at segment 4"))).toBe(true);
expect(db.isFts5CorruptionError(new Error("some other sqlite error"))).toBe(false);
});
});
describe("Database FTS5 guard behavior", () => {
it("rebuildFts5Index returns false when FTS5 is unavailable", async () => {
const prevEnv = process.env.FUSION_DISABLE_FTS5;
process.env.FUSION_DISABLE_FTS5 = "1";
const tmpDir = makeTmpDir();
const fusionDir = join(tmpDir, ".fusion");
const localDb = new Database(fusionDir);
try {
localDb.init();
expect(localDb.rebuildFts5Index()).toBe(false);
} finally {
localDb.close();
await rm(tmpDir, { recursive: true, force: true });
if (prevEnv === undefined) {
delete process.env.FUSION_DISABLE_FTS5;
} else {
process.env.FUSION_DISABLE_FTS5 = prevEnv;
}
}
});
});
describe("createDatabase factory", () => {

View File

@@ -10081,6 +10081,43 @@ describe("RunMutationContext", () => {
});
});
describe("FTS5 corruption recovery during upsert", () => {
it("rebuilds FTS5 and retries once when upsert fails with an FTS corruption error", async () => {
const db = store.getDatabase();
const rebuildSpy = vi.spyOn(db, "rebuildFts5Index").mockReturnValue(true);
const upsertSpy = vi.spyOn(store as any, "upsertTask");
const originalUpsert = upsertSpy.getMockImplementation();
upsertSpy
.mockImplementationOnce(() => {
throw new Error("SQLITE_CORRUPT: corruption found reading blob in fts5");
})
.mockImplementation((task: Task) => {
if (originalUpsert) {
return originalUpsert(task);
}
return (TaskStore.prototype as any).upsertTask.call(store, task);
});
const created = await store.createTask({ description: "Recover from FTS corruption" });
expect(created.id).toBeDefined();
expect(rebuildSpy).toHaveBeenCalledTimes(1);
expect(upsertSpy).toHaveBeenCalledTimes(2);
});
it("propagates non-FTS errors without rebuild", async () => {
const db = store.getDatabase();
const rebuildSpy = vi.spyOn(db, "rebuildFts5Index").mockReturnValue(true);
vi.spyOn(store as any, "upsertTask").mockImplementationOnce(() => {
throw new Error("constraint failed");
});
await expect(store.createTask({ description: "Should fail" })).rejects.toThrow("constraint failed");
expect(rebuildSpy).not.toHaveBeenCalled();
});
});
describe("clearStaleBaseBranchReferences (FN-2165)", () => {
it("nulls baseBranch on live tasks that reference a deleted branch", async () => {
const upstream = await store.createTask({ description: "Upstream" });

View File

@@ -704,6 +704,86 @@ export class Database {
return this._fts5Available;
}
/**
* Rebuild the task FTS5 index and maintenance triggers from scratch.
* Returns false when FTS5 is unavailable in this runtime.
*/
rebuildFts5Index(): boolean {
if (!this._fts5Available) {
return false;
}
try {
this.db.exec("DROP TRIGGER IF EXISTS tasks_fts_ai");
this.db.exec("DROP TRIGGER IF EXISTS tasks_fts_au");
this.db.exec("DROP TRIGGER IF EXISTS tasks_fts_ad");
this.db.exec("DROP TABLE IF EXISTS tasks_fts");
this.db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS tasks_fts USING fts5(
id,
title,
description,
comments,
content='tasks',
content_rowid='rowid'
)
`);
this.db.exec(`
CREATE TRIGGER IF NOT EXISTS tasks_fts_ai AFTER INSERT ON tasks BEGIN
INSERT INTO tasks_fts(rowid, id, title, description, comments)
VALUES (new.rowid, new.id, COALESCE(new.title, ''), new.description, COALESCE(new.comments, '[]'));
END
`);
const hasTaskTitle = this.hasColumn("tasks", "title");
const updateColumns = hasTaskTitle
? "id, title, description, comments"
: "id, description, comments";
const oldTitle = hasTaskTitle ? "COALESCE(old.title, '')" : "''";
const newTitle = hasTaskTitle ? "COALESCE(new.title, '')" : "''";
this.db.exec(`
CREATE TRIGGER IF NOT EXISTS tasks_fts_au AFTER UPDATE OF ${updateColumns} ON tasks BEGIN
INSERT INTO tasks_fts(tasks_fts, rowid, id, title, description, comments)
VALUES('delete', old.rowid, old.id, ${oldTitle}, old.description, COALESCE(old.comments, '[]'));
INSERT INTO tasks_fts(rowid, id, title, description, comments)
VALUES (new.rowid, new.id, ${newTitle}, new.description, COALESCE(new.comments, '[]'));
END
`);
this.db.exec(`
CREATE TRIGGER IF NOT EXISTS tasks_fts_ad AFTER DELETE ON tasks BEGIN
INSERT INTO tasks_fts(tasks_fts, rowid, id, title, description, comments)
VALUES('delete', old.rowid, old.id, COALESCE(old.title, ''), old.description, COALESCE(old.comments, '[]'));
END
`);
this.db.exec("INSERT INTO tasks_fts(tasks_fts) VALUES('rebuild')");
return true;
} catch (error) {
console.warn("[fusion:db] Failed to rebuild FTS5 index", error);
throw error;
}
}
/**
* Run FTS5 integrity check. Returns true when healthy or unavailable.
*/
checkFts5Integrity(): boolean {
if (!this._fts5Available) {
return true;
}
try {
this.db.exec("INSERT INTO tasks_fts(tasks_fts) VALUES('integrity-check')");
return true;
} catch {
return false;
}
}
/**
* Initialize the database: create tables if they don't exist
* and seed meta values.
@@ -1885,6 +1965,19 @@ export class Database {
return Boolean(row);
}
/**
* Check whether an error appears to be an FTS5 corruption/integrity failure.
*/
isFts5CorruptionError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error ?? "");
const lower = message.toLowerCase();
return (
lower.includes("corruption found reading blob") ||
lower.includes("database disk image is malformed") ||
(lower.includes("fts5") && lower.includes("corrupt"))
);
}
/**
* Check whether a table has a given column.
*/

View File

@@ -1079,6 +1079,33 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
this.db.bumpLastModified();
}
private upsertTaskWithFtsRecovery(task: Task): void {
try {
this.upsertTask(task);
return;
} catch (error) {
if (!this.db.isFts5CorruptionError(error)) {
throw error;
}
console.warn(`[fusion:store] FTS5 corruption detected during upsert for task ${task.id}; rebuilding index and retrying once`);
try {
this.db.rebuildFts5Index();
} catch (rebuildError) {
console.warn("[fusion:store] FTS5 rebuild failed; propagating original upsert error", rebuildError);
throw error;
}
try {
this.upsertTask(task);
} catch (retryError) {
console.warn("[fusion:store] Upsert retry after FTS5 rebuild failed; propagating original upsert error", retryError);
throw error;
}
}
}
/**
* Read a task from SQLite by ID.
*/
@@ -1330,7 +1357,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
* for backward compatibility and debugging.
*/
private async atomicWriteTaskJson(dir: string, task: Task): Promise<void> {
this.upsertTask(task);
this.upsertTaskWithFtsRecovery(task);
// Also write to disk for backward compatibility
await this.writeTaskJsonFile(dir, task);
}
@@ -1350,7 +1377,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
): Promise<void> {
this.db.transaction(() => {
// Upsert the task
this.upsertTask(task);
this.upsertTaskWithFtsRecovery(task);
// Optionally record the audit event in the same transaction
if (auditInput) {
@@ -5598,7 +5625,7 @@ ${stepsSection}`;
try {
// Simple query to verify database responsiveness
this.db.prepare("SELECT 1").get();
return true;
return this.db.checkFts5Integrity();
} catch {
return false;
}