diff --git a/docs/architecture.md b/docs/architecture.md index 7d7e8d8949..7b9e8dccf8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -669,6 +669,7 @@ Runtime action-gate flow (v1): - `GridlockDetector` (`gridlock-detector.ts`) — detects all-blocked todo pipelines and emits notification events (plus explicit clear signals when gridlock resolves) - `TransientErrorDetector` (`transient-error-detector.ts`) — retriable error classification - `SelfHealingManager` (`self-healing.ts`) — auto-unpause/maintenance recovery actions + - Batch 1 maintenance now includes `fts-maintenance` for the live task search index. When `fts5Available === true`, every maintenance tick runs an incremental `merge`, every 4th tick escalates to `optimize`, and any index larger than `32 MiB` or `1 MiB × live task count` is fully rebuilt. Each pass emits `task:fts-maintenance` run-audit telemetry with before/after byte counts. - `recoverGhostReviewTasks()` is a fallback only for idle, non-terminal `in-review` states. Terminal/actionable states (notably `status: "failed"`) are preserved and **not** auto-kicked back to `todo`. - Mission validation has a dedicated stale-run reaper: startup recovery and Batch 2 maintenance call `reapStaleMissionValidatorRuns()` when wired by the runtime, using `VALIDATOR_RUN_STALE_MAX_AGE_MS` (currently 6 hours). The sweep terminates ownerless `mission_validator_runs.status='running'` rows as `error`, writes the reap reason into `summary`, leaves `lastValidatorRunId` pointing at the now-terminal run, and emits run-audit telemetry with `mutationType: "mission:validator-run-reaped"` plus `runId`/`featureId`/`missionId`/`triggerType`/`elapsedMs` metadata. Active mission features move to `loopState="needs_fix"` + `lastValidatorStatus="error"` unless their parent mission is already `complete`/`archived`. diff --git a/docs/storage.md b/docs/storage.md index 3bda2e7eeb..8c890c1bab 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -136,6 +136,20 @@ Important execution nuance: - done tasks: prefer `mergeDetails.landedFiles` - in-progress/in-review (or legacy pre-FN-4646 tasks): fall back to `task.modifiedFiles` +## FTS5 task-index maintenance (FN-5943) + +- Live task search uses the `tasks_fts` external-content FTS5 table in `fusion.db`; the archive log uses a separate `archived_tasks_fts` table in `archive.db`. +- `tasks_fts_au` is value-aware: even though hot task writes still upsert full rows, the trigger only fires when indexed text actually changes (`id`, `title`, `description`, `comments`, `deletedAt`). Status/step/worktree churn no longer rewrites the FTS row on every update. +- `Database.getFtsIndexBytes()` measures index size via `SELECT SUM(LENGTH(block)) FROM tasks_fts_data`. Fusion intentionally does **not** rely on `dbstat`, because node:sqlite builds do not guarantee `SQLITE_ENABLE_DBSTAT_VTAB`. +- `SelfHealingManager` Batch 1 now runs `fts-maintenance` when `fts5Available === true`: + - every maintenance tick: incremental `merge` compaction + - every 4th maintenance tick: heavier `optimize` + - immediate full `rebuild` when `tasks_fts` exceeds either `32 MiB` absolute or `1 MiB × live task count` +- Each maintenance pass emits run-audit telemetry with `mutationType: "task:fts-maintenance"` and `metadata` including `mode`, `bytesBefore`, `bytesAfter`, `taskCount`, `rebuilt`, and the threshold values. +- `rebuildFts5Index()` and migration 103 also set conservative FTS5 merge policy (`automerge=8`, `crisismerge=16`) so legitimate text edits merge segments sooner without forcing the heaviest optimize path on every write. +- `archived_tasks_fts` is intentionally **not** compacted by this task. The archive DB is effectively append-only for completed tasks, so it does not see the same constant-update churn as the live board; archive FTS compaction is deferred to a follow-up if archive bloat is observed. +- Separate attached-DB recommendation: **defer** moving `tasks_fts*` into a dedicated attached SQLite file. It would isolate FTS bloat/corruption from the main DB, but today it would complicate cross-DB joins in `searchTasks`, widen transaction/backup/checkpoint coordination, and add new multi-instance/polling failure modes on a path that is now bounded by guarded triggers + maintenance. Revisit only if live-main-DB FTS size or corruption remains operationally significant after FN-5943. + ## SQLite write-path lock recovery (FN-4042 / FN-4083) - Every disk-backed SQLite connection that Fusion opens for project storage (`fusion.db`), the central registry (`fusion-central.db`), archives (`archive.db`), and worktree hydration explicitly sets `PRAGMA busy_timeout = 5000` and `PRAGMA journal_mode = WAL` at connection open time before write work begins. diff --git a/packages/core/src/__tests__/db-migrate.test.ts b/packages/core/src/__tests__/db-migrate.test.ts index 269c840a19..fd69f5c1b0 100644 --- a/packages/core/src/__tests__/db-migrate.test.ts +++ b/packages/core/src/__tests__/db-migrate.test.ts @@ -715,8 +715,8 @@ describe("schema migration", () => { const row = db.prepare("SELECT deletedAt FROM tasks WHERE id = 'FN-legacy'").get() as { deletedAt: string | null }; expect(row.deletedAt).toBeNull(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -749,8 +749,8 @@ describe("schema migration", () => { { id: "WS-001", mode: "prompt", gateMode: "advisory" }, { id: "WS-002", mode: "script", gateMode: "advisory" }, ]); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -800,8 +800,8 @@ describe("schema migration", () => { reviewerContextRetryCount: 0, reviewerFallbackRetryCount: 0, }); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -830,8 +830,8 @@ describe("schema migration", () => { const columns = db.prepare("PRAGMA table_info(milestones)").all() as Array<{ name: string }>; expect(columns.map((column) => column.name)).toContain("acceptanceCriteria"); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -872,8 +872,8 @@ describe("schema migration", () => { const missionColumns = db.prepare("PRAGMA table_info(missions)").all() as Array<{ name: string }>; expect(missionColumns.map((column) => column.name)).toContain("autoMerge"); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -907,8 +907,8 @@ describe("schema migration", () => { { id: "WS-002", mode: "script", enabled: 1, gateMode: "advisory" }, { id: "WS-003", mode: "prompt", enabled: 0, gateMode: "advisory" }, ]); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -945,8 +945,8 @@ describe("schema migration", () => { const indexes = db.prepare("PRAGMA index_list(mission_goals)").all() as Array<{ name: string }>; expect(indexes.some((index) => index.name === "idxMissionGoalsGoalId")).toBe(true); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -1007,7 +1007,7 @@ describe("schema migration", () => { expect(customFieldsColumn).toBeDefined(); expect(customFieldsColumn?.dflt_value).toBe("'{}'"); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -1045,7 +1045,7 @@ describe("schema migration", () => { const indexes = db.prepare("PRAGMA index_list(workflow_settings)").all() as Array<{ name: string }>; expect(indexes.some((index) => index.name === "idx_workflow_settings_project")).toBe(true); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -1127,7 +1127,7 @@ describe("schema migration", () => { expect(indexNames).toContain("idx_cli_sessions_chatSessionId"); expect(indexNames).toContain("idx_cli_sessions_project_state"); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -1159,7 +1159,7 @@ describe("schema migration", () => { .all() as Array<{ name: string }>; expect(columns.map((column) => column.name)).toContain("cliExecutorAdapterId"); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -1169,7 +1169,7 @@ describe("schema migration", () => { const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as Array<{ name: string }>; expect(tables.map((row) => row.name)).toContain("cli_sessions"); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -1226,23 +1226,23 @@ describe("schema migration", () => { .get() as { migrated_fragment_id: string | null }; expect(stepRow.migrated_fragment_id).toBeNull(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); it("migration 109 is idempotent on re-init", () => { const db = new Database(fusionDir); db.init(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); db.close(); // Re-open the same on-disk DB: already at 109, the 109 block must be a no-op. const reopened = new Database(fusionDir); reopened.init(); - expect(reopened.getSchemaVersion()).toBe(113); - expect(reopened.getSchemaVersion()).toBe(113); + expect(reopened.getSchemaVersion()).toBe(114); + expect(reopened.getSchemaVersion()).toBe(114); const workflowColumns = reopened.prepare("PRAGMA table_info(workflows)").all() as Array<{ name: string }>; expect(workflowColumns.filter((c) => c.name === "kind")).toHaveLength(1); const stepColumns = reopened.prepare("PRAGMA table_info(workflow_steps)").all() as Array<{ name: string }>; diff --git a/packages/core/src/__tests__/db.test.ts b/packages/core/src/__tests__/db.test.ts index 30b379e99c..23d1f377b7 100644 --- a/packages/core/src/__tests__/db.test.ts +++ b/packages/core/src/__tests__/db.test.ts @@ -334,8 +334,8 @@ describe("Database", () => { }); it("seeds schema version", () => { - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); }); it("includes tokenUsageCacheWriteTokens on freshly initialized tasks table", () => { @@ -394,8 +394,8 @@ describe("Database", () => { it("is idempotent - calling init() twice does not fail", () => { expect(() => db.init()).not.toThrow(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); }); it("does not overwrite existing config on re-init", () => { // Update the config @@ -1465,8 +1465,8 @@ describe("schema migrations", () => { db.init(); // Verify version bumped to 29 (includes v1→v2 through v26→v29) - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); // Verify new columns exist and existing data is intact const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; @@ -1491,16 +1491,16 @@ describe("schema migrations", () => { const db = new Database(fusionDir); db.init(); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); // Re-init should not fail db.init(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); // Re-init should not fail db.init(); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); db.close(); }); @@ -1535,8 +1535,8 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; expect(cols.map((col) => col.name)).toContain("priority"); @@ -1577,8 +1577,8 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; const colNames = cols.map((col) => col.name); @@ -1650,8 +1650,8 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; const colNames = cols.map((col) => col.name); @@ -1891,8 +1891,8 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); const cols = db.prepare("PRAGMA table_info(chat_messages)").all() as Array<{ name: string }>; expect(cols.map((col) => col.name)).toContain("attachments"); @@ -1966,8 +1966,8 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>; expect(tables).toEqual([{ name: "agentRatings" }]); @@ -1991,8 +1991,8 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>; expect(tables).toEqual([{ name: "mission_events" }]); @@ -2096,8 +2096,8 @@ describe("schema migrations", () => { db.init(); // Verify version bumped to 29 - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); // Verify new columns exist and existing data is intact const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; @@ -2316,8 +2316,8 @@ describe("schema migrations", () => { localDb.init(); - expect(localDb.getSchemaVersion()).toBe(113); - expect(localDb.getSchemaVersion()).toBe(113); + expect(localDb.getSchemaVersion()).toBe(114); + expect(localDb.getSchemaVersion()).toBe(114); const columns = localDb.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; expect(columns.map((column) => column.name)).toContain("tokenUsageCacheWriteTokens"); @@ -2628,8 +2628,8 @@ describe("createDatabase factory", () => { const db = createDatabase(fusionDir); db.init(); - expect(db.getSchemaVersion()).toBe(113); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); + expect(db.getSchemaVersion()).toBe(114); expect(db.getLastModified()).toBeGreaterThan(0); db.close(); @@ -2783,8 +2783,8 @@ describe("migration v77 task token budget columns", () => { migrated = new Database(fusion); migrated.init(); - expect(migrated.getSchemaVersion()).toBe(113); - expect(migrated.getSchemaVersion()).toBe(113); + expect(migrated.getSchemaVersion()).toBe(114); + expect(migrated.getSchemaVersion()).toBe(114); const rows = migrated.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; const names = new Set(rows.map((row) => row.name)); expect(names.has("tokenBudgetSoftAlertedAt")).toBe(true); @@ -2815,8 +2815,8 @@ describe("migration v106 adds tasks.transitionPending (FN-1417)", () => { const fresh = new Database(fusion); try { fresh.init(); - expect(fresh.getSchemaVersion()).toBe(113); - expect(fresh.getSchemaVersion()).toBe(113); + expect(fresh.getSchemaVersion()).toBe(114); + expect(fresh.getSchemaVersion()).toBe(114); const names = new Set( (fresh.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>).map((r) => r.name), ); @@ -2844,8 +2844,8 @@ describe("migration v106 adds tasks.transitionPending (FN-1417)", () => { migrated = new Database(fusion); migrated.init(); - expect(migrated.getSchemaVersion()).toBe(113); - expect(migrated.getSchemaVersion()).toBe(113); + expect(migrated.getSchemaVersion()).toBe(114); + expect(migrated.getSchemaVersion()).toBe(114); const names = new Set( (migrated.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>).map((r) => r.name), ); @@ -2871,8 +2871,8 @@ describe("migration v107 adds workflow_run_branches + index (FN-1417)", () => { const fresh = new Database(fusion); try { fresh.init(); - expect(fresh.getSchemaVersion()).toBe(113); - expect(fresh.getSchemaVersion()).toBe(113); + expect(fresh.getSchemaVersion()).toBe(114); + expect(fresh.getSchemaVersion()).toBe(114); const table = fresh .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'workflow_run_branches'") .get() as { name: string } | undefined; @@ -2906,8 +2906,8 @@ describe("migration v107 adds workflow_run_branches + index (FN-1417)", () => { migrated = new Database(fusion); migrated.init(); - expect(migrated.getSchemaVersion()).toBe(113); - expect(migrated.getSchemaVersion()).toBe(113); + expect(migrated.getSchemaVersion()).toBe(114); + expect(migrated.getSchemaVersion()).toBe(114); const table = migrated .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'workflow_run_branches'") .get() as { name: string } | undefined; @@ -2948,8 +2948,8 @@ describe("migration v67 drops orphan project auth tables", () => { migrated = new Database(fusion); migrated.init(); - expect(migrated.getSchemaVersion()).toBe(113); - expect(migrated.getSchemaVersion()).toBe(113); + expect(migrated.getSchemaVersion()).toBe(114); + expect(migrated.getSchemaVersion()).toBe(114); const tables = migrated .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'project_auth_%'") .all() as Array<{ name: string }>; @@ -2976,8 +2976,8 @@ describe("migration v67 drops orphan project auth tables", () => { try { fresh.init(); - expect(fresh.getSchemaVersion()).toBe(113); - expect(fresh.getSchemaVersion()).toBe(113); + expect(fresh.getSchemaVersion()).toBe(114); + expect(fresh.getSchemaVersion()).toBe(114); const tables = fresh .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'project_auth_%'") .all() as Array<{ name: string }>; diff --git a/packages/core/src/__tests__/goals-schema.test.ts b/packages/core/src/__tests__/goals-schema.test.ts index c60effed3f..d1859c8df3 100644 --- a/packages/core/src/__tests__/goals-schema.test.ts +++ b/packages/core/src/__tests__/goals-schema.test.ts @@ -91,6 +91,6 @@ describe("goals schema", () => { }); it("reports schema version 101", () => { - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); }); }); diff --git a/packages/core/src/__tests__/insight-store.test.ts b/packages/core/src/__tests__/insight-store.test.ts index a5c1b3ce1c..cc42db9ec3 100644 --- a/packages/core/src/__tests__/insight-store.test.ts +++ b/packages/core/src/__tests__/insight-store.test.ts @@ -1000,7 +1000,7 @@ describe("Migration: pre-33 DB upgrade", () => { // Step 1: Create a fresh database at v33 (runs all migrations up to 33) const db1 = createDatabase(legacyDir); db1.init(); - expect(db1.getSchemaVersion()).toBe(113); + expect(db1.getSchemaVersion()).toBe(114); db1.close(); // Step 2: Manually downgrade to version 32 and drop insight tables @@ -1035,7 +1035,7 @@ describe("Migration: pre-33 DB upgrade", () => { expect(tableNamesBefore).not.toContain("project_insight_runs"); // Now run init — this triggers the v32→v33 migration db3.init(); - expect(db3.getSchemaVersion()).toBe(113); + expect(db3.getSchemaVersion()).toBe(114); // Step 4: Verify insight tables exist after migration const tablesAfter = db3.prepare( @@ -1066,12 +1066,12 @@ describe("Migration: pre-33 DB upgrade", () => { try { const db1 = createDatabase(testDir); db1.init(); - expect(db1.getSchemaVersion()).toBe(113); + expect(db1.getSchemaVersion()).toBe(114); db1.close(); const db2 = createDatabase(testDir); expect(() => db2.init()).not.toThrow(); - expect(db2.getSchemaVersion()).toBe(113); + expect(db2.getSchemaVersion()).toBe(114); db2.close(); } finally { rmSync(testDir, { recursive: true, force: true }); @@ -1085,7 +1085,7 @@ describe("Migration: pre-33 DB upgrade", () => { // Step 1: Create a fresh DB and run migrations const db1 = createDatabase(compatDir); db1.init(); - expect(db1.getSchemaVersion()).toBe(113); + expect(db1.getSchemaVersion()).toBe(114); // Step 2: Strip lifecycle and cancelledAt columns by recreating the // table without them. This simulates a DB that was created before the diff --git a/packages/core/src/__tests__/merge-request-record.test.ts b/packages/core/src/__tests__/merge-request-record.test.ts index d1b0b9e428..364dbdeac8 100644 --- a/packages/core/src/__tests__/merge-request-record.test.ts +++ b/packages/core/src/__tests__/merge-request-record.test.ts @@ -38,7 +38,7 @@ describe("TaskStore merge request record + completion handoff marker", () => { .all() as Array<{ name: string }>; expect(tableRows).toEqual([{ name: "completion_handoff_markers" }, { name: "merge_requests" }]); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); }); it("upserts merge request records", async () => { diff --git a/packages/core/src/__tests__/mission-store.test.ts b/packages/core/src/__tests__/mission-store.test.ts index 49b629e06a..52fdebcac2 100644 --- a/packages/core/src/__tests__/mission-store.test.ts +++ b/packages/core/src/__tests__/mission-store.test.ts @@ -3746,7 +3746,7 @@ describe("MissionStore", () => { describe("Loop State & Validator Run Schema (v31)", () => { it("schema version is 101 after migration", () => { - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); }); it("mission_features table has loop state columns", () => { diff --git a/packages/core/src/__tests__/run-audit.test.ts b/packages/core/src/__tests__/run-audit.test.ts index 017b5c1fe6..ce0035b235 100644 --- a/packages/core/src/__tests__/run-audit.test.ts +++ b/packages/core/src/__tests__/run-audit.test.ts @@ -584,7 +584,7 @@ describe("Run Audit", () => { }); it("schema version is bumped to 40", () => { - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); }); }); }); diff --git a/packages/core/src/__tests__/store-archive-search.test.ts b/packages/core/src/__tests__/store-archive-search.test.ts index d9331eeeaf..0ee83b39bb 100644 --- a/packages/core/src/__tests__/store-archive-search.test.ts +++ b/packages/core/src/__tests__/store-archive-search.test.ts @@ -981,6 +981,39 @@ describe("searchTasks", () => { expect(results.length).toBeGreaterThanOrEqual(0); // Should not throw }); + + it("keeps search correctness across hyphenated tokens, null text fields, soft delete, restore, and compaction", async () => { + const hyphenTask = await store.createTask({ title: "release-note-guard", description: "hyphenated task target" }); + const nullTitleTask = await store.createTask({ description: "null title searchable phrase" }); + await store.addComment(nullTitleTask.id, "comment-needle text", "tester"); + (store as any).db.prepare("UPDATE tasks SET comments = NULL WHERE id = ?").run(nullTitleTask.id); + + const hyphenBefore = await store.searchTasks("release-note-guard"); + expect(hyphenBefore.map((entry) => entry.id)).toContain(hyphenTask.id); + + const nullFieldResults = await store.searchTasks("searchable phrase"); + expect(nullFieldResults.map((entry) => entry.id)).toContain(nullTitleTask.id); + + await store.deleteTask(hyphenTask.id, { allowResurrection: true }); + expect((await store.searchTasks("release-note-guard")).map((entry) => entry.id)).not.toContain(hyphenTask.id); + + await store.createTaskWithReservedId( + { + title: "release-note-guard restored", + description: "hyphenated task target", + forceResurrect: true, + }, + { taskId: hyphenTask.id }, + ); + + const beforeOptimize = (await store.searchTasks("release-note-guard")).map((entry) => entry.id).sort(); + expect(beforeOptimize).toContain(hyphenTask.id); + + expect(store.optimizeFts5("optimize")).toBe(store.fts5Available); + + const afterOptimize = (await store.searchTasks("release-note-guard")).map((entry) => entry.id).sort(); + expect(afterOptimize).toEqual(beforeOptimize); + }); }); diff --git a/packages/core/src/__tests__/store-merge-queue.test.ts b/packages/core/src/__tests__/store-merge-queue.test.ts index d1aee05555..6f5fd1551c 100644 --- a/packages/core/src/__tests__/store-merge-queue.test.ts +++ b/packages/core/src/__tests__/store-merge-queue.test.ts @@ -60,7 +60,7 @@ describe("TaskStore merge queue", () => { expect.arrayContaining(["idx_mergeQueue_lease_ready", "idx_mergeQueue_leaseExpiresAt"]), ); - expect(store.getDatabase().getSchemaVersion()).toBe(113); + expect(store.getDatabase().getSchemaVersion()).toBe(114); }); it("migrates a legacy v88 database and preserves task rows", async () => { diff --git a/packages/core/src/__tests__/task-documents.test.ts b/packages/core/src/__tests__/task-documents.test.ts index 4cbf2e1de3..b7b71047c0 100644 --- a/packages/core/src/__tests__/task-documents.test.ts +++ b/packages/core/src/__tests__/task-documents.test.ts @@ -51,7 +51,7 @@ describe("TaskStore task documents", () => { expect(tableNames.has("task_documents")).toBe(true); expect(tableNames.has("task_document_revisions")).toBe(true); - expect(db.getSchemaVersion()).toBe(113); + expect(db.getSchemaVersion()).toBe(114); const index = db .prepare( diff --git a/packages/core/src/archive-db.ts b/packages/core/src/archive-db.ts index eb709e2fcf..799f5758ce 100644 --- a/packages/core/src/archive-db.ts +++ b/packages/core/src/archive-db.ts @@ -38,7 +38,11 @@ CREATE TRIGGER IF NOT EXISTS archived_tasks_fts_ai AFTER INSERT ON archived_task VALUES (new.rowid, new.id, COALESCE(new.title, ''), new.description, COALESCE(new.comments, '[]')); END; -CREATE TRIGGER IF NOT EXISTS archived_tasks_fts_au AFTER UPDATE OF id, title, description, comments ON archived_tasks BEGIN +CREATE TRIGGER IF NOT EXISTS archived_tasks_fts_au AFTER UPDATE OF id, title, description, comments ON archived_tasks +WHEN ( + old.id IS NOT new.id OR old.title IS NOT new.title + OR old.description IS NOT new.description OR old.comments IS NOT new.comments +) BEGIN INSERT INTO archived_tasks_fts(archived_tasks_fts, rowid, id, title, description, comments) VALUES('delete', old.rowid, old.id, COALESCE(old.title, ''), old.description, COALESCE(old.comments, '[]')); INSERT INTO archived_tasks_fts(rowid, id, title, description, comments) diff --git a/packages/core/src/db.ts b/packages/core/src/db.ts index 3b15ccc379..ae83d9179a 100644 --- a/packages/core/src/db.ts +++ b/packages/core/src/db.ts @@ -149,7 +149,11 @@ export function probeFts5(db: DatabaseSync): boolean { // ── Schema Definition ──────────────────────────────────────────────── -const SCHEMA_VERSION = 113; +const SCHEMA_VERSION = 114; + +const TASKS_FTS_AUTOMERGE = 8; +const TASKS_FTS_CRISISMERGE = 16; +const TASKS_FTS_MERGE_PAGES = 16; export { SCHEMA_VERSION }; @@ -1655,6 +1659,48 @@ export class Database { return this._fts5Available; } + private getTaskFtsTriggerParts(): { + updateColumns: string; + oldTitle: string; + newTitle: string; + whenClause: string; + reinsertWhere: string; + } { + const hasTaskTitle = this.hasColumn("tasks", "title"); + const hasDeletedAt = this.hasColumn("tasks", "deletedAt"); + const updateColumns = hasTaskTitle + ? hasDeletedAt ? "id, title, description, comments, deletedAt" : "id, title, description, comments" + : hasDeletedAt ? "id, description, comments, deletedAt" : "id, description, comments"; + const oldTitle = hasTaskTitle ? "COALESCE(old.title, '')" : "''"; + const newTitle = hasTaskTitle ? "COALESCE(new.title, '')" : "''"; + const whenChecks = [ + "old.id IS NOT new.id", + hasTaskTitle ? "old.title IS NOT new.title" : "0", + "old.description IS NOT new.description", + "old.comments IS NOT new.comments", + hasDeletedAt ? "old.deletedAt IS NOT new.deletedAt" : "0", + ].join(" OR\n "); + + return { + updateColumns, + oldTitle, + newTitle, + whenClause: `WHEN (\n ${whenChecks}\n ) `, + reinsertWhere: hasDeletedAt ? "new.deletedAt IS NULL" : "1 = 1", + }; + } + + private configureTaskFts5(): void { + if (!this.tableExists("tasks_fts")) { + return; + } + // Per https://www.sqlite.org/fts5.html, lower automerge/crisismerge + // bounds keep segment counts from ballooning under legitimate text edits + // without forcing every write onto the heaviest optimize path. + this.db.exec(`INSERT INTO tasks_fts(tasks_fts, rank) VALUES('automerge', ${TASKS_FTS_AUTOMERGE})`); + this.db.exec(`INSERT INTO tasks_fts(tasks_fts, rank) VALUES('crisismerge', ${TASKS_FTS_CRISISMERGE})`); + } + /** * Rebuild the task FTS5 index and maintenance triggers from scratch. * Returns false when FTS5 is unavailable in this runtime. @@ -1681,8 +1727,8 @@ export class Database { ) `); - const hasTaskTitle = this.hasColumn("tasks", "title"); const hasDeletedAt = this.hasColumn("tasks", "deletedAt"); + const { updateColumns, oldTitle, newTitle, whenClause, reinsertWhere } = this.getTaskFtsTriggerParts(); this.db.exec(` CREATE TRIGGER IF NOT EXISTS tasks_fts_ai AFTER INSERT ON tasks @@ -1692,19 +1738,14 @@ export class Database { END `); - const updateColumns = hasTaskTitle - ? hasDeletedAt ? "id, title, description, comments, deletedAt" : "id, title, description, comments" - : hasDeletedAt ? "id, description, comments, deletedAt" : "id, description, comments"; - const oldTitle = hasTaskTitle ? "COALESCE(old.title, '')" : "''"; - const newTitle = hasTaskTitle ? "COALESCE(new.title, '')" : "''"; - this.db.exec(` - CREATE TRIGGER IF NOT EXISTS tasks_fts_au AFTER UPDATE OF ${updateColumns} ON tasks BEGIN + CREATE TRIGGER IF NOT EXISTS tasks_fts_au AFTER UPDATE OF ${updateColumns} ON tasks + ${whenClause}BEGIN INSERT INTO tasks_fts(tasks_fts, rowid, id, title, description, comments) VALUES('delete', old.rowid, old.id, ${oldTitle}, old.description, COALESCE(old.comments, '[]')); INSERT INTO tasks_fts(rowid, id, title, description, comments) SELECT new.rowid, new.id, ${newTitle}, new.description, COALESCE(new.comments, '[]') - WHERE ${hasDeletedAt ? "new.deletedAt IS NULL" : "1 = 1"}; + WHERE ${reinsertWhere}; END `); @@ -1716,6 +1757,7 @@ export class Database { END `); + this.configureTaskFts5(); this.db.exec("INSERT INTO tasks_fts(tasks_fts) VALUES('rebuild')"); return true; } catch (error) { @@ -1724,6 +1766,51 @@ export class Database { } } + /** + * Run incremental or full FTS5 compaction. + * Returns false when FTS5 is unavailable in this runtime. + */ + optimizeFts5(mode: "optimize" | "merge" = "optimize"): boolean { + if (!this._fts5Available) { + return false; + } + + try { + if (mode === "merge") { + this.db.exec(`INSERT INTO tasks_fts(tasks_fts, rank) VALUES('merge', ${TASKS_FTS_MERGE_PAGES})`); + } else { + this.db.exec("INSERT INTO tasks_fts(tasks_fts) VALUES('optimize')"); + } + return true; + } catch (error) { + if (this.isFts5CorruptionError(error)) { + return this.rebuildFts5Index(); + } + throw error; + } + } + + /** + * Estimate FTS index bytes using the aggregate size of `tasks_fts_data.block`. + * Prefer this over `dbstat` because node:sqlite builds do not guarantee + * `SQLITE_ENABLE_DBSTAT_VTAB`, while the shadow table exists anywhere FTS5 does. + */ + getFtsIndexBytes(): number | null { + if (!this._fts5Available) { + return null; + } + + const row = this.db.prepare("SELECT COALESCE(SUM(LENGTH(block)), 0) AS bytes FROM tasks_fts_data").get() as + | { bytes?: number } + | undefined; + return typeof row?.bytes === "number" ? row.bytes : 0; + } + + getTaskRowCount(): number { + const row = this.db.prepare("SELECT COUNT(*) AS count FROM tasks").get() as { count?: number } | undefined; + return typeof row?.count === "number" ? row.count : 0; + } + /** * Run FTS5 integrity check. Returns true when healthy or unavailable. */ @@ -2593,7 +2680,6 @@ export class Database { } // AFTER INSERT trigger - index new tasks - const hasTaskTitle = this.hasColumn("tasks", "title"); const hasDeletedAt = this.hasColumn("tasks", "deletedAt"); this.db.exec(` CREATE TRIGGER IF NOT EXISTS tasks_fts_ai AFTER INSERT ON tasks @@ -2603,22 +2689,20 @@ export class Database { END `); - const updateColumns = hasTaskTitle - ? hasDeletedAt ? "id, title, description, comments, deletedAt" : "id, title, description, comments" - : hasDeletedAt ? "id, description, comments, deletedAt" : "id, description, comments"; - const oldTitle = hasTaskTitle ? "COALESCE(old.title, '')" : "''"; - const newTitle = hasTaskTitle ? "COALESCE(new.title, '')" : "''"; + const { updateColumns, oldTitle, newTitle, whenClause, reinsertWhere } = this.getTaskFtsTriggerParts(); // AFTER UPDATE trigger - reindex updated tasks (delete old + insert new). // Restrict this to searchable columns so log/status churn does not bloat - // the FTS index during long-running executor activity. + // the FTS index during long-running executor activity, then add a + // value-aware WHEN guard so no-op `SET title = title` upserts do not churn. this.db.exec(` - CREATE TRIGGER IF NOT EXISTS tasks_fts_au AFTER UPDATE OF ${updateColumns} ON tasks BEGIN + CREATE TRIGGER IF NOT EXISTS tasks_fts_au AFTER UPDATE OF ${updateColumns} ON tasks + ${whenClause}BEGIN INSERT INTO tasks_fts(tasks_fts, rowid, id, title, description, comments) VALUES('delete', old.rowid, old.id, ${oldTitle}, old.description, COALESCE(old.comments, '[]')); INSERT INTO tasks_fts(rowid, id, title, description, comments) SELECT new.rowid, new.id, ${newTitle}, new.description, COALESCE(new.comments, '[]') - WHERE ${hasDeletedAt ? "new.deletedAt IS NULL" : "1 = 1"}; + WHERE ${reinsertWhere}; END `); @@ -2630,6 +2714,8 @@ export class Database { VALUES('delete', old.rowid, old.id, COALESCE(old.title, ''), old.description, COALESCE(old.comments, '[]')); END `); + + this.configureTaskFts5(); }); } @@ -3053,24 +3139,22 @@ export class Database { return; } const hasTaskTitle = this.hasColumn("tasks", "title"); - const hasDeletedAt = this.hasColumn("tasks", "deletedAt"); - const updateColumns = hasTaskTitle - ? hasDeletedAt ? "id, title, description, comments, deletedAt" : "id, title, description, comments" - : hasDeletedAt ? "id, description, comments, deletedAt" : "id, description, comments"; - const oldTitle = hasTaskTitle ? "COALESCE(old.title, '')" : "''"; - const newTitle = hasTaskTitle ? "COALESCE(new.title, '')" : "''"; + const { updateColumns, oldTitle, newTitle, whenClause, reinsertWhere } = this.getTaskFtsTriggerParts(); this.db.exec(` DROP TRIGGER IF EXISTS tasks_fts_au; - CREATE TRIGGER tasks_fts_au AFTER UPDATE OF ${updateColumns} ON tasks BEGIN + CREATE TRIGGER tasks_fts_au AFTER UPDATE OF ${updateColumns} ON tasks + ${whenClause}BEGIN INSERT INTO tasks_fts(tasks_fts, rowid, id, title, description, comments) VALUES('delete', old.rowid, old.id, ${oldTitle}, old.description, COALESCE(old.comments, '[]')); INSERT INTO tasks_fts(rowid, id, title, description, comments) SELECT new.rowid, new.id, ${newTitle}, new.description, COALESCE(new.comments, '[]') - WHERE ${hasDeletedAt ? "new.deletedAt IS NULL" : "1 = 1"}; + WHERE ${reinsertWhere}; END; `); + this.configureTaskFts5(); + if (hasTaskTitle) { this.db.exec("INSERT INTO tasks_fts(tasks_fts) VALUES('rebuild')"); } @@ -4513,6 +4597,30 @@ export class Database { }); } + // Migration 114: FTS5 task index maintenance. Rebuilds the task-update + // trigger so no-op searchable-field updates do not rewrite index rows, and + // reapplies maintenance tuning on migrated databases. + if (version < 114) { + this.applyMigration(114, () => { + if (!this._fts5Available) { + return; + } + const { updateColumns, oldTitle, newTitle, whenClause, reinsertWhere } = this.getTaskFtsTriggerParts(); + this.db.exec(` + DROP TRIGGER IF EXISTS tasks_fts_au; + CREATE TRIGGER tasks_fts_au AFTER UPDATE OF ${updateColumns} ON tasks + ${whenClause}BEGIN + INSERT INTO tasks_fts(tasks_fts, rowid, id, title, description, comments) + VALUES('delete', old.rowid, old.id, ${oldTitle}, old.description, COALESCE(old.comments, '[]')); + INSERT INTO tasks_fts(rowid, id, title, description, comments) + SELECT new.rowid, new.id, ${newTitle}, new.description, COALESCE(new.comments, '[]') + WHERE ${reinsertWhere}; + END; + `); + this.configureTaskFts5(); + }); + } + } /** diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 4d568f33e3..a44600b437 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -14430,6 +14430,22 @@ ${stepsSection}`; this.secretsStore = null; } + get fts5Available(): boolean { + return this.db.fts5Available; + } + + optimizeFts5(mode?: "optimize" | "merge"): boolean { + return this.db.optimizeFts5(mode); + } + + getFtsIndexBytes(): number | null { + return this.db.getFtsIndexBytes(); + } + + getTaskRowCount(): number { + return this.db.getTaskRowCount(); + } + /** * Run a WAL checkpoint and return checkpoint stats. * diff --git a/packages/engine/src/__tests__/fts-maintenance.test.ts b/packages/engine/src/__tests__/fts-maintenance.test.ts new file mode 100644 index 0000000000..d284aa7e7b --- /dev/null +++ b/packages/engine/src/__tests__/fts-maintenance.test.ts @@ -0,0 +1,268 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { EventEmitter } from "node:events"; +import { mkdtempSync, rmSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { TaskStore, type Settings, type TaskStore as TaskStoreType } from "@fusion/core"; + +import { SelfHealingManager } from "../self-healing.js"; + +function createMockStore(overrides: Record = {}): TaskStoreType & EventEmitter { + const emitter = new EventEmitter(); + return Object.assign(emitter, { + getSettings: vi.fn().mockResolvedValue({ + maintenanceIntervalMs: 0, + globalPause: false, + enginePaused: false, + } as unknown as Settings), + listTasks: vi.fn().mockResolvedValue([]), + walCheckpoint: vi.fn().mockReturnValue({ busy: 0, log: 0, checkpointed: 0 }), + pruneOperationalLogs: vi.fn().mockReturnValue({ deletedByTable: {}, deletedTotal: 0 }), + pruneAgentLogFiles: vi.fn().mockReturnValue({ prunedFiles: 0, prunedEntries: 0, freedBytes: 0 }), + recordRunAuditEvent: vi.fn().mockResolvedValue(undefined), + fts5Available: true, + getFtsIndexBytes: vi.fn().mockReturnValue(1024), + getTaskRowCount: vi.fn().mockReturnValue(4), + optimizeFts5: vi.fn().mockReturnValue(true), + getDatabase: vi.fn().mockReturnValue({ rebuildFts5Index: vi.fn().mockReturnValue(true) }), + ...overrides, + }) as unknown as TaskStoreType & EventEmitter; +} + +function makeTmpDir(prefix: string): string { + return mkdtempSync(join(tmpdir(), prefix)); +} + +const createdDirs = new Set(); + +function trackDir(path: string): string { + createdDirs.add(path); + return path; +} + +async function createStore(options?: { disableFts5?: boolean; inMemoryDb?: boolean }) { + const prevEnv = process.env.FUSION_DISABLE_FTS5; + if (options?.disableFts5) { + process.env.FUSION_DISABLE_FTS5 = "1"; + } else if (prevEnv === "1") { + delete process.env.FUSION_DISABLE_FTS5; + } + + const rootDir = trackDir(makeTmpDir("kb-engine-fts-root-")); + const globalDir = trackDir(makeTmpDir("kb-engine-fts-global-")); + const store = new TaskStore(rootDir, globalDir, { inMemoryDb: options?.inMemoryDb === true }); + await store.init(); + const manager = new SelfHealingManager(store, { rootDir }); + + return { + rootDir, + globalDir, + store, + manager, + restoreEnv() { + if (prevEnv === undefined) { + delete process.env.FUSION_DISABLE_FTS5; + } else { + process.env.FUSION_DISABLE_FTS5 = prevEnv; + } + }, + }; +} + +async function cleanupStore(context: Awaited> | undefined) { + if (!context) return; + context.manager.stop(); + context.store.close(); + context.restoreEnv(); + await rm(context.rootDir, { recursive: true, force: true }); + await rm(context.globalDir, { recursive: true, force: true }); + createdDirs.delete(context.rootDir); + createdDirs.delete(context.globalDir); +} + +afterEach(async () => { + vi.restoreAllMocks(); + for (const dir of Array.from(createdDirs)) { + try { + await rm(dir, { recursive: true, force: true }); + } catch { + rmSync(dir, { recursive: true, force: true }); + } finally { + createdDirs.delete(dir); + } + } +}); + +describe("SelfHealingManager FTS maintenance", () => { + it("runs incremental merge on ordinary maintenance ticks and records audit telemetry", async () => { + const store = createMockStore({ + getFtsIndexBytes: vi.fn().mockReturnValueOnce(2048).mockReturnValueOnce(1024), + }); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (manager as any).maintenanceTickCounter = 1; + + await (manager as any).maintainTaskFts(); + + expect(store.optimizeFts5).toHaveBeenCalledWith("merge"); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + domain: "database", + mutationType: "task:fts-maintenance", + target: "tasks_fts", + metadata: expect.objectContaining({ + mode: "merge", + bytesBefore: 2048, + bytesAfter: 1024, + rebuilt: false, + taskCount: 4, + }), + })); + }); + + it("runs optimize on the configured cadence", async () => { + const store = createMockStore(); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (manager as any).maintenanceTickCounter = 4; + + await (manager as any).maintainTaskFts(); + + expect(store.optimizeFts5).toHaveBeenCalledWith("optimize"); + }); + + it("rebuilds when the index exceeds the absolute threshold", async () => { + const rebuildFts5Index = vi.fn().mockReturnValue(true); + const store = createMockStore({ + getFtsIndexBytes: vi.fn().mockReturnValueOnce(40 * 1024 * 1024).mockReturnValueOnce(128 * 1024), + getDatabase: vi.fn().mockReturnValue({ rebuildFts5Index }), + getTaskRowCount: vi.fn().mockReturnValue(2), + }); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (manager as any).maintenanceTickCounter = 2; + + await (manager as any).maintainTaskFts(); + + expect(rebuildFts5Index).toHaveBeenCalledTimes(1); + expect(store.optimizeFts5).not.toHaveBeenCalled(); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + metadata: expect.objectContaining({ mode: "rebuild", rebuilt: true }), + })); + }); + + it("rebuilds when the per-task ratio exceeds the relative threshold", async () => { + const rebuildFts5Index = vi.fn().mockReturnValue(true); + const store = createMockStore({ + getFtsIndexBytes: vi.fn().mockReturnValueOnce(2 * 1024 * 1024).mockReturnValueOnce(64 * 1024), + getDatabase: vi.fn().mockReturnValue({ rebuildFts5Index }), + getTaskRowCount: vi.fn().mockReturnValue(1), + }); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + (manager as any).maintenanceTickCounter = 2; + + await (manager as any).maintainTaskFts(); + + expect(rebuildFts5Index).toHaveBeenCalledTimes(1); + expect(store.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + metadata: expect.objectContaining({ + mode: "rebuild", + relativeThresholdBytes: 1024 * 1024, + }), + })); + }); + + it("skips cleanly when FTS5 is unavailable", async () => { + const store = createMockStore({ fts5Available: false }); + const manager = new SelfHealingManager(store, { rootDir: "/tmp/test-project" }); + + await expect((manager as any).maintainTaskFts()).resolves.toBeUndefined(); + expect(store.optimizeFts5).not.toHaveBeenCalled(); + expect(store.recordRunAuditEvent).not.toHaveBeenCalled(); + }); + + it("compacts a real disk-backed index and keeps archive search working", async () => { + let ctx: Awaited> | undefined; + try { + ctx = await createStore(); + const { store, manager } = ctx; + if (!store.fts5Available) { + expect(store.fts5Available).toBe(false); + return; + } + + const churnTask = await store.createTask({ title: "fts seed", description: "fts seed description", column: "todo" }); + const softDeleted = await store.createTask({ title: "soft delete target", description: "soft-delete-needle", column: "todo" }); + const archived = await store.createTask({ title: "archive target", description: "archive-needle", column: "done" }); + await store.archiveTask(archived.id); + + const before = store.getFtsIndexBytes(); + const update = store.getDatabase().prepare(` + UPDATE tasks + SET title = ?, description = ?, comments = ?, updatedAt = ? + WHERE id = ? + `); + for (let i = 0; i < 220; i++) { + const marker = `marker-${i}`; + const payload = `${"alpha ".repeat(600)}${marker}`; + update.run( + `fts ${marker}`, + payload, + JSON.stringify([{ id: `c-${i}`, text: `${payload} comment` }]), + `2026-06-03T00:${String(i % 60).padStart(2, "0")}:00.000Z`, + churnTask.id, + ); + } + const grown = store.getFtsIndexBytes(); + expect(before).not.toBeNull(); + expect(grown).not.toBeNull(); + expect(grown!).toBeGreaterThan(before!); + + await store.deleteTask(softDeleted.id); + (manager as any).maintenanceTickCounter = 2; + await (manager as any).maintainTaskFts(); + + const after = store.getFtsIndexBytes(); + expect(after).not.toBeNull(); + expect(after!).toBeLessThan(grown!); + expect(after!).toBeLessThan(store.getTaskRowCount() * 1024 * 1024); + + const searchResults = await store.searchTasks("marker-219"); + expect(searchResults.map((task) => task.id)).toContain(churnTask.id); + expect((await store.searchTasks("soft-delete-needle")).map((task) => task.id)).not.toContain(softDeleted.id); + + const archiveResults = (store as any).archiveDb.search("archive-needle", 10) as Array<{ id: string }>; + expect(archiveResults.map((task) => task.id)).toContain(archived.id); + } finally { + await cleanupStore(ctx); + } + }); + + it("real disk-backed maintenance is a no-op when FTS5 is disabled", async () => { + let ctx: Awaited> | undefined; + try { + ctx = await createStore({ disableFts5: true }); + const { store, manager } = ctx; + expect(store.fts5Available).toBe(false); + await expect((manager as any).maintainTaskFts()).resolves.toBeUndefined(); + } finally { + await cleanupStore(ctx); + } + }); + + it("does not throw for in-memory stores", async () => { + let ctx: Awaited> | undefined; + try { + ctx = await createStore({ inMemoryDb: true }); + const { store, manager } = ctx; + if (!store.fts5Available) { + expect(store.fts5Available).toBe(false); + return; + } + + await store.createTask({ title: "memory fts", description: "memory fts payload", column: "todo" }); + (manager as any).maintenanceTickCounter = 4; + await expect((manager as any).maintainTaskFts()).resolves.toBeUndefined(); + } finally { + await cleanupStore(ctx); + } + }); +}); diff --git a/packages/engine/src/self-healing.ts b/packages/engine/src/self-healing.ts index ec806944f4..f506332a9a 100644 --- a/packages/engine/src/self-healing.ts +++ b/packages/engine/src/self-healing.ts @@ -71,6 +71,13 @@ const yieldEventLoop = (): Promise => new Promise((resolve) => setImmediat const DONE_TASK_INTEGRITY_SWEEP_LIMIT = 50; const BOARD_STALL_NOTIFICATION_COOLDOWN_MS = 60 * 60_000; const DB_CORRUPTION_NOTIFICATION_COOLDOWN_MS = 60 * 60 * 1000; +const FTS_MAINTENANCE_MERGE_CADENCE_TICKS = 1; +const FTS_MAINTENANCE_OPTIMIZE_CADENCE_TICKS = 4; +// Live pathology peaked around 775 KB/task (~96 MB for ~120 tasks), while a +// rebuilt healthy index was ~0.1 MB. Keep the steady-state budget generous but +// bounded so sustained text churn heals before segment growth becomes material. +const FTS_REBUILD_THRESHOLD_BYTES = 32 * 1024 * 1024; +const FTS_REBUILD_BYTES_PER_TASK = 1 * 1024 * 1024; export const STALE_ACTIVE_BRANCH_EXECUTION_GRACE_MS = 10 * 60_000; export const COMPLETION_HANDOFF_LIMBO_GRACE_MS = 5 * 60_000; export const MAX_COMPLETION_HANDOFF_LIMBO_RECOVERIES = 3; @@ -1716,6 +1723,7 @@ export class SelfHealingManager { log.log(`Maintenance batch 1 step "prune-agent-log-files" succeeded — files=${prunedFiles} entries=${prunedEntries} bytes=${freedBytes}`); }, }, + { name: "fts-maintenance", fn: () => this.maintainTaskFts() }, { name: "checkpoint-wal", fn: () => Promise.resolve(this.checkpointWal()) }, { name: "enforce-worktree-cap", fn: () => this.enforceWorktreeCap() }, ]; @@ -8637,6 +8645,68 @@ export class SelfHealingManager { } } + private async maintainTaskFts(): Promise { + if (!this.store.fts5Available) { + log.log('Maintenance batch 1 step "fts-maintenance" skipped — FTS5 unavailable'); + return; + } + + const bytesBefore = this.store.getFtsIndexBytes(); + if (bytesBefore === null) { + log.log('Maintenance batch 1 step "fts-maintenance" skipped — FTS shadow tables unavailable'); + return; + } + + const taskCount = this.store.getTaskRowCount(); + const relativeThresholdBytes = taskCount > 0 ? taskCount * FTS_REBUILD_BYTES_PER_TASK : null; + const shouldRebuild = bytesBefore >= FTS_REBUILD_THRESHOLD_BYTES + || (relativeThresholdBytes !== null && bytesBefore > relativeThresholdBytes); + const shouldOptimize = !shouldRebuild + && FTS_MAINTENANCE_OPTIMIZE_CADENCE_TICKS > 0 + && this.maintenanceTickCounter % FTS_MAINTENANCE_OPTIMIZE_CADENCE_TICKS === 0; + const mode = shouldRebuild ? "rebuild" : shouldOptimize ? "optimize" : "merge"; + + if (mode === "merge" + && FTS_MAINTENANCE_MERGE_CADENCE_TICKS > 1 + && this.maintenanceTickCounter % FTS_MAINTENANCE_MERGE_CADENCE_TICKS !== 0) { + log.log('Maintenance batch 1 step "fts-maintenance" skipped — merge cadence not due'); + return; + } + + let rebuilt = false; + if (mode === "rebuild") { + rebuilt = this.store.getDatabase().rebuildFts5Index(); + } else { + this.store.optimizeFts5(mode); + } + + const bytesAfter = this.store.getFtsIndexBytes(); + log.log(`Maintenance batch 1 step "fts-maintenance" ${mode}: ${bytesBefore} → ${bytesAfter ?? "unknown"} bytes (tasks=${taskCount})`); + + try { + await createRunAuditor(this.store, { + runId: generateSyntheticRunId("self-heal-fts-maintenance", "tasks_fts"), + agentId: "self-healing", + phase: "maintenance-fts", + }).database({ + type: "task:fts-maintenance" as DatabaseMutationType, + target: "tasks_fts", + metadata: { + mode, + bytesBefore, + bytesAfter, + taskCount, + rebuilt, + absoluteThresholdBytes: FTS_REBUILD_THRESHOLD_BYTES, + relativeThresholdBytes, + }, + }); + } catch (err: unknown) { + const errorMessage = err instanceof Error ? err.message : String(err); + log.warn(`Failed to write task:fts-maintenance run-audit event: ${errorMessage}`); + } + } + /** Run a best-effort passive WAL checkpoint without forcing live writers to truncate. */ private checkpointWal(): void { try { diff --git a/plugins/fusion-plugin-roadmap/src/store/__tests__/roadmap-store.test.ts b/plugins/fusion-plugin-roadmap/src/store/__tests__/roadmap-store.test.ts index 8b31b7e7b0..f248fa0b2b 100644 --- a/plugins/fusion-plugin-roadmap/src/store/__tests__/roadmap-store.test.ts +++ b/plugins/fusion-plugin-roadmap/src/store/__tests__/roadmap-store.test.ts @@ -743,10 +743,10 @@ describe("RoadmapStore", () => { }); describe("schema version", () => { - it("schema version is 112 after init", () => { + it("schema version is 114 after init", () => { // Tracks @fusion/core's SCHEMA_VERSION (the roadmap store layers on core's // Database). Bump this in lockstep when core adds a migration. - expect(db.getSchemaVersion()).toBe(112); + expect(db.getSchemaVersion()).toBe(114); }); });