FN-5975: extend archive FTS maintenance coverage
Add archive-database FTS maintenance and document the expanded compaction policy. - add archive FTS maintenance helpers for optimize, rebuild, size measurement, and row counts - extend self-healing maintenance to compact and rebuild archived_tasks_fts on a slower archive-specific cadence - cover archive FTS maintenance with new core and engine tests and update architecture/storage docs Files changed: docs/architecture.md | 2 +- docs/storage.md | 12 +- .../__tests__/archive-db-fts-maintenance.test.ts | 221 +++++++++++++++++++++ packages/core/src/archive-db.ts | 61 +++++- packages/core/src/db.ts | 21 +- packages/core/src/store.ts | 20 ++ .../src/__tests__/fts-maintenance-archive.test.ts | 207 +++++++++++++++++++ packages/engine/src/self-healing.ts | 80 ++++++++ 8 files changed, 608 insertions(+), 16 deletions(-) Fusion-Task-Id: FN-5975 Fusion-Task-Lineage: 13645c8c-3126-4d1b-af23-7cab1a8bb276
This commit is contained in:
gsxdsm
221
packages/core/src/__tests__/archive-db-fts-maintenance.test.ts
Normal file
221
packages/core/src/__tests__/archive-db-fts-maintenance.test.ts
Normal file
@@ -0,0 +1,221 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { ArchiveDatabase } from "../archive-db.js";
|
||||
import type { ArchivedTaskEntry } from "../types.js";
|
||||
|
||||
type ArchiveEntryOverrides = Partial<ArchivedTaskEntry> & { title?: string | null };
|
||||
|
||||
function makeTmpDir(prefix = "kb-archive-fts-"): string {
|
||||
return mkdtempSync(join(tmpdir(), prefix));
|
||||
}
|
||||
|
||||
function makeEntry(id: string, overrides: ArchiveEntryOverrides = {}): ArchivedTaskEntry {
|
||||
const timestamp = overrides.archivedAt ?? "2026-06-03T00:00:00.000Z";
|
||||
return {
|
||||
id,
|
||||
lineageId: overrides.lineageId ?? id,
|
||||
column: "archived",
|
||||
title: overrides.title === null ? undefined : overrides.title ?? `title ${id}`,
|
||||
description: overrides.description ?? `description ${id}`,
|
||||
comments: overrides.comments ?? [],
|
||||
dependencies: overrides.dependencies ?? [],
|
||||
steps: overrides.steps ?? [],
|
||||
currentStep: overrides.currentStep ?? 0,
|
||||
log: overrides.log ?? [],
|
||||
createdAt: overrides.createdAt ?? timestamp,
|
||||
updatedAt: overrides.updatedAt ?? timestamp,
|
||||
archivedAt: timestamp,
|
||||
columnMovedAt: overrides.columnMovedAt ?? timestamp,
|
||||
prompt: overrides.prompt,
|
||||
};
|
||||
}
|
||||
|
||||
describe("ArchiveDatabase FTS maintenance", () => {
|
||||
let prevDisableFts5: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
prevDisableFts5 = process.env.FUSION_DISABLE_FTS5;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (prevDisableFts5 === undefined) {
|
||||
delete process.env.FUSION_DISABLE_FTS5;
|
||||
} else {
|
||||
process.env.FUSION_DISABLE_FTS5 = prevDisableFts5;
|
||||
}
|
||||
});
|
||||
|
||||
it("rebuilds a churned disk-backed archive index down to a bounded size", async () => {
|
||||
const dir = makeTmpDir();
|
||||
const archive = new ArchiveDatabase(dir);
|
||||
|
||||
try {
|
||||
archive.init();
|
||||
if (!archive.fts5Available) {
|
||||
expect(archive.rebuildFts5Index()).toBe(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const payload = "alpha ".repeat(1200);
|
||||
for (let i = 0; i < 180; i++) {
|
||||
archive.upsert(makeEntry("FN-ARCHIVE-1", {
|
||||
archivedAt: new Date(1717372800000 + i * 1000).toISOString(),
|
||||
updatedAt: new Date(1717372800000 + i * 1000).toISOString(),
|
||||
title: `release-note-${i}`,
|
||||
description: `${payload}${i}`,
|
||||
comments: [{ id: `c-${i}`, text: `${payload}comment-${i}`, author: "tester", createdAt: new Date(1717372800000 + i * 1000).toISOString() }],
|
||||
}));
|
||||
}
|
||||
|
||||
const grownBytes = archive.getFtsIndexBytes();
|
||||
expect(grownBytes).not.toBeNull();
|
||||
expect(grownBytes!).toBeGreaterThan(0);
|
||||
expect(archive.getArchivedRowCount()).toBe(1);
|
||||
|
||||
expect(archive.rebuildFts5Index()).toBe(true);
|
||||
const rebuiltBytes = archive.getFtsIndexBytes();
|
||||
expect(rebuiltBytes).not.toBeNull();
|
||||
expect(rebuiltBytes!).toBeLessThan(grownBytes!);
|
||||
expect(rebuiltBytes!).toBeLessThan(1 * 1024 * 1024);
|
||||
expect(archive.search("release-note-179", 10).map((entry) => entry.id)).toContain("FN-ARCHIVE-1");
|
||||
} finally {
|
||||
archive.close();
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("supports optimize and merge compaction on disk-backed archives", async () => {
|
||||
const dir = makeTmpDir();
|
||||
const archive = new ArchiveDatabase(dir);
|
||||
|
||||
try {
|
||||
archive.init();
|
||||
if (!archive.fts5Available) {
|
||||
expect(archive.optimizeFts5("merge")).toBe(false);
|
||||
expect(archive.optimizeFts5("optimize")).toBe(false);
|
||||
return;
|
||||
}
|
||||
|
||||
archive.upsert(makeEntry("FN-ARCHIVE-2", {
|
||||
description: "optimize target alpha beta gamma",
|
||||
comments: [{ id: "c-1", text: "merge optimize searchable", author: "tester", createdAt: "2026-06-03T00:00:00.000Z" }],
|
||||
}));
|
||||
|
||||
expect(archive.optimizeFts5("merge")).toBe(true);
|
||||
expect(archive.optimizeFts5("optimize")).toBe(true);
|
||||
expect(archive.search("searchable", 10).map((entry) => entry.id)).toContain("FN-ARCHIVE-2");
|
||||
} finally {
|
||||
archive.close();
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps archive search results identical before and after compaction across null fields, hyphenated tokens, churn, and deletes", async () => {
|
||||
const dir = makeTmpDir();
|
||||
const archive = new ArchiveDatabase(dir);
|
||||
|
||||
try {
|
||||
archive.init();
|
||||
const rawDb = (archive as any).db;
|
||||
|
||||
archive.upsert(makeEntry("FN-ARCHIVE-3", {
|
||||
title: "release-note-guard",
|
||||
description: "archive special-char target",
|
||||
comments: [{ id: "c-2", text: "comment-needle", author: "tester", createdAt: "2026-06-03T00:00:00.000Z" }],
|
||||
}));
|
||||
archive.upsert(makeEntry("FN-ARCHIVE-4", {
|
||||
title: null,
|
||||
description: "null title searchable phrase",
|
||||
comments: [],
|
||||
}));
|
||||
rawDb.prepare("UPDATE archived_tasks SET comments = NULL WHERE id = ?").run("FN-ARCHIVE-4");
|
||||
archive.upsert(makeEntry("FN-ARCHIVE-5", {
|
||||
title: "delete-me",
|
||||
description: "deleted archive needle",
|
||||
}));
|
||||
archive.delete("FN-ARCHIVE-5");
|
||||
|
||||
for (let i = 0; i < 60; i++) {
|
||||
archive.upsert(makeEntry("FN-ARCHIVE-3", {
|
||||
archivedAt: new Date(1717372800000 + i * 1000).toISOString(),
|
||||
updatedAt: new Date(1717372800000 + i * 1000).toISOString(),
|
||||
title: `release-note-guard ${i}`,
|
||||
description: `archive special-char target marker-${i}`,
|
||||
comments: [{ id: `c-${i}`, text: `comment-needle marker-${i}`, author: "tester", createdAt: new Date(1717372800000 + i * 1000).toISOString() }],
|
||||
}));
|
||||
}
|
||||
|
||||
const queryResultsBefore = {
|
||||
hyphen: archive.search("release-note-guard", 10).map((entry) => entry.id).sort(),
|
||||
nullTitle: archive.search("searchable phrase", 10).map((entry) => entry.id).sort(),
|
||||
comment: archive.search("comment-needle", 10).map((entry) => entry.id).sort(),
|
||||
special: archive.search("test + special (chars)", 10).map((entry) => entry.id).sort(),
|
||||
deleted: archive.search("deleted archive needle", 10).map((entry) => entry.id).sort(),
|
||||
};
|
||||
|
||||
expect(queryResultsBefore.hyphen).toContain("FN-ARCHIVE-3");
|
||||
expect(queryResultsBefore.nullTitle).toContain("FN-ARCHIVE-4");
|
||||
expect(queryResultsBefore.comment).toContain("FN-ARCHIVE-3");
|
||||
expect(queryResultsBefore.deleted).not.toContain("FN-ARCHIVE-5");
|
||||
|
||||
expect(archive.optimizeFts5("optimize")).toBe(archive.fts5Available);
|
||||
expect(archive.rebuildFts5Index()).toBe(archive.fts5Available);
|
||||
|
||||
const queryResultsAfter = {
|
||||
hyphen: archive.search("release-note-guard", 10).map((entry) => entry.id).sort(),
|
||||
nullTitle: archive.search("searchable phrase", 10).map((entry) => entry.id).sort(),
|
||||
comment: archive.search("comment-needle", 10).map((entry) => entry.id).sort(),
|
||||
special: archive.search("test + special (chars)", 10).map((entry) => entry.id).sort(),
|
||||
deleted: archive.search("deleted archive needle", 10).map((entry) => entry.id).sort(),
|
||||
};
|
||||
|
||||
expect(queryResultsAfter).toEqual(queryResultsBefore);
|
||||
} finally {
|
||||
archive.close();
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("treats maintenance seams as safe no-ops when FTS5 is disabled or in-memory", async () => {
|
||||
process.env.FUSION_DISABLE_FTS5 = "1";
|
||||
const disabledDir = makeTmpDir("kb-archive-fts-disabled-");
|
||||
const disabledArchive = new ArchiveDatabase(disabledDir);
|
||||
|
||||
try {
|
||||
disabledArchive.init();
|
||||
disabledArchive.upsert(makeEntry("FN-ARCHIVE-6", { title: null, description: "fallback-like alpha-beta" }));
|
||||
expect(disabledArchive.fts5Available).toBe(false);
|
||||
expect(disabledArchive.getFtsIndexBytes()).toBeNull();
|
||||
expect(disabledArchive.optimizeFts5("merge")).toBe(false);
|
||||
expect(disabledArchive.optimizeFts5("optimize")).toBe(false);
|
||||
expect(disabledArchive.rebuildFts5Index()).toBe(false);
|
||||
expect(disabledArchive.search("alpha-beta", 10).map((entry) => entry.id)).toEqual(["FN-ARCHIVE-6"]);
|
||||
} finally {
|
||||
disabledArchive.close();
|
||||
await rm(disabledDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
delete process.env.FUSION_DISABLE_FTS5;
|
||||
const memoryArchive = new ArchiveDatabase("/tmp/fusion-archive-memory-test", { inMemory: true });
|
||||
try {
|
||||
memoryArchive.init();
|
||||
memoryArchive.upsert(makeEntry("FN-ARCHIVE-7", { description: "memory archive search" }));
|
||||
expect(() => memoryArchive.getArchivedRowCount()).not.toThrow();
|
||||
expect(memoryArchive.search("memory archive", 10).map((entry) => entry.id)).toContain("FN-ARCHIVE-7");
|
||||
if (memoryArchive.fts5Available) {
|
||||
expect(memoryArchive.optimizeFts5("merge")).toBe(true);
|
||||
expect(memoryArchive.rebuildFts5Index()).toBe(true);
|
||||
} else {
|
||||
expect(memoryArchive.optimizeFts5("merge")).toBe(false);
|
||||
expect(memoryArchive.rebuildFts5Index()).toBe(false);
|
||||
}
|
||||
} finally {
|
||||
memoryArchive.close();
|
||||
rmSync("/tmp/fusion-archive-memory-test", { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -2,9 +2,11 @@ import { DatabaseSync } from "./sqlite-adapter.js";
|
||||
import { existsSync, mkdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { ArchivedTaskEntry } from "./types.js";
|
||||
import { probeFts5 } from "./db.js";
|
||||
import { isFts5CorruptionError, probeFts5 } from "./db.js";
|
||||
import { hasTitleIdDrift, normalizeTitleForTaskId } from "./task-title-id-drift.js";
|
||||
|
||||
const ARCHIVED_TASKS_FTS_MERGE_PAGES = 16;
|
||||
|
||||
const BASE_SCHEMA_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS archived_tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
@@ -157,6 +159,63 @@ export class ArchiveDatabase {
|
||||
this.db.prepare("DELETE FROM archived_tasks WHERE id = ?").run(id);
|
||||
}
|
||||
|
||||
rebuildFts5Index(): boolean {
|
||||
if (!this._fts5Available) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
this.db.exec("INSERT INTO archived_tasks_fts(archived_tasks_fts) VALUES('rebuild')");
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn("[fusion:archive-db] Failed to rebuild archive FTS5 index", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
optimizeFts5(mode: "optimize" | "merge" = "optimize"): boolean {
|
||||
if (!this._fts5Available) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
if (mode === "merge") {
|
||||
this.db.exec(
|
||||
`INSERT INTO archived_tasks_fts(archived_tasks_fts, rank) VALUES('merge', ${ARCHIVED_TASKS_FTS_MERGE_PAGES})`,
|
||||
);
|
||||
} else {
|
||||
this.db.exec("INSERT INTO archived_tasks_fts(archived_tasks_fts) VALUES('optimize')");
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isFts5CorruptionError(error)) {
|
||||
return this.rebuildFts5Index();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate archive FTS index bytes using the shadow-table block payload.
|
||||
* Prefer this over `dbstat` because node:sqlite builds do not guarantee
|
||||
* `SQLITE_ENABLE_DBSTAT_VTAB`, while `archived_tasks_fts_data` exists anywhere FTS5 does.
|
||||
*/
|
||||
getFtsIndexBytes(): number | null {
|
||||
if (!this._fts5Available) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const row = this.db.prepare("SELECT COALESCE(SUM(LENGTH(block)), 0) AS bytes FROM archived_tasks_fts_data").get() as
|
||||
| { bytes?: number }
|
||||
| undefined;
|
||||
return typeof row?.bytes === "number" ? row.bytes : 0;
|
||||
}
|
||||
|
||||
getArchivedRowCount(): number {
|
||||
const row = this.db.prepare("SELECT COUNT(*) AS count FROM archived_tasks").get() as { count?: number } | undefined;
|
||||
return typeof row?.count === "number" ? row.count : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-text search over archived tasks. Accepts a raw user query and routes
|
||||
* through FTS5 when available, or a LIKE-based scan when not.
|
||||
|
||||
@@ -147,6 +147,19 @@ export function probeFts5(db: DatabaseSync): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether an error appears to be an FTS5 corruption/integrity failure.
|
||||
*/
|
||||
export function isFts5CorruptionError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error ?? "");
|
||||
const lower = message.toLowerCase();
|
||||
return (
|
||||
lower.includes("corruption found reading blob") ||
|
||||
lower.includes("database disk image is malformed") ||
|
||||
(lower.includes("fts5") && lower.includes("corrupt"))
|
||||
);
|
||||
}
|
||||
|
||||
// ── Schema Definition ────────────────────────────────────────────────
|
||||
|
||||
const SCHEMA_VERSION = 114;
|
||||
@@ -4703,13 +4716,7 @@ export class Database {
|
||||
* Check whether an error appears to be an FTS5 corruption/integrity failure.
|
||||
*/
|
||||
isFts5CorruptionError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error ?? "");
|
||||
const lower = message.toLowerCase();
|
||||
return (
|
||||
lower.includes("corruption found reading blob") ||
|
||||
lower.includes("database disk image is malformed") ||
|
||||
(lower.includes("fts5") && lower.includes("corrupt"))
|
||||
);
|
||||
return isFts5CorruptionError(error);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -14660,18 +14660,38 @@ ${stepsSection}`;
|
||||
return this.db.fts5Available;
|
||||
}
|
||||
|
||||
get archiveFts5Available(): boolean {
|
||||
return this.archiveDb.fts5Available;
|
||||
}
|
||||
|
||||
optimizeFts5(mode?: "optimize" | "merge"): boolean {
|
||||
return this.db.optimizeFts5(mode);
|
||||
}
|
||||
|
||||
optimizeArchiveFts5(mode?: "optimize" | "merge"): boolean {
|
||||
return this.archiveDb.optimizeFts5(mode);
|
||||
}
|
||||
|
||||
getFtsIndexBytes(): number | null {
|
||||
return this.db.getFtsIndexBytes();
|
||||
}
|
||||
|
||||
getArchiveFtsIndexBytes(): number | null {
|
||||
return this.archiveDb.getFtsIndexBytes();
|
||||
}
|
||||
|
||||
getTaskRowCount(): number {
|
||||
return this.db.getTaskRowCount();
|
||||
}
|
||||
|
||||
getArchivedRowCount(): number {
|
||||
return this.archiveDb.getArchivedRowCount();
|
||||
}
|
||||
|
||||
rebuildArchiveFts5Index(): boolean {
|
||||
return this.archiveDb.rebuildFts5Index();
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a WAL checkpoint and return checkpoint stats.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user