root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: guard SQLite FTS5 at runtime and fall back to LIKE search

Browse Source 
On Node builds whose bundled node:sqlite was compiled without
SQLITE_ENABLE_FTS5 (older 22.x LTS), `fn dashboard` crashed on first
run with `Error: no such module: fts5` during schema migration 21.

Database and ArchiveDatabase now probe FTS5 at startup via a disposable
virtual table. When unavailable, migrations 21 and 35 skip the tasks_fts
DDL, ArchiveDatabase skips the archived_tasks_fts block, and
TaskStore.searchTasks / ArchiveDatabase.search fall back to LIKE scans
over id/title/description/comments with ESCAPE-aware patterns.

Set FUSION_DISABLE_FTS5=1 to force the fallback on runtimes where FTS5
is available but undesirable (e.g. reproducing fresh-install behavior
in tests).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
gsxdsm
2026-04-23 12:05:35 -07:00
parent 01e24dfbdb
commit bbdd11aab3
5 changed files with 453 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
packages/core/src/archive-db.ts
View File

@@ -2,8 +2,9 @@ import { DatabaseSync } from "node:sqlite";
import { existsSync, mkdirSync } from "node:fs";
import { join } from "node:path";
import type { ArchivedTaskEntry } from "./types.js";
import { probeFts5 } from "./db.js";
const ARCHIVE_SCHEMA_SQL = `
const BASE_SCHEMA_SQL = `
CREATE TABLE IF NOT EXISTS archived_tasks (
id TEXT PRIMARY KEY,
taskJson TEXT NOT NULL,
@@ -19,7 +20,9 @@ CREATE TABLE IF NOT EXISTS archived_tasks (
CREATE INDEX IF NOT EXISTS idxArchivedTasksArchivedAt ON archived_tasks(archivedAt);
CREATE INDEX IF NOT EXISTS idxArchivedTasksCreatedAt ON archived_tasks(createdAt);
`;
const FTS5_SCHEMA_SQL = `
CREATE VIRTUAL TABLE IF NOT EXISTS archived_tasks_fts USING fts5(
id,
title,
@@ -49,18 +52,28 @@ END;
export class ArchiveDatabase {
private db: DatabaseSync;
private readonly _fts5Available: boolean;
constructor(private readonly kbDir: string) {
constructor(kbDir: string) {
if (!existsSync(kbDir)) {
mkdirSync(kbDir, { recursive: true });
}
this.db = new DatabaseSync(join(kbDir, "archive.db"));
this.db.exec("PRAGMA journal_mode = WAL");
this.db.exec("PRAGMA busy_timeout = 5000");
this._fts5Available = probeFts5(this.db);
}
/** True when this SQLite build has FTS5. See db.ts#probeFts5. */
get fts5Available(): boolean {
return this._fts5Available;
}
init(): void {
this.db.exec(ARCHIVE_SCHEMA_SQL);
this.db.exec(BASE_SCHEMA_SQL);
if (this._fts5Available) {
this.db.exec(FTS5_SCHEMA_SQL);
}
this.addColumnIfMissing("archived_tasks", "prompt", "TEXT");
}
@@ -109,15 +122,60 @@ export class ArchiveDatabase {
this.db.prepare("DELETE FROM archived_tasks WHERE id = ?").run(id);
}
/**
* Full-text search over archived tasks. Accepts a raw user query and routes
* through FTS5 when available, or a LIKE-based scan when not.
*/
search(query: string, limit: number): ArchivedTaskEntry[] {
const trimmed = query?.trim();
if (!trimmed) return [];
const tokens = trimmed
.split(/\s+/)
.filter((t) => t.length > 0)
.map((t) => t.replace(/["{}:*^+()]/g, ""))
.filter((t) => t.length > 0);
if (tokens.length === 0) return [];
if (this._fts5Available) {
const ftsQuery = tokens
.map((token) => {
if (/[":(){}*^+-]/.test(token)) {
return `"${token.replace(/"/g, '\\"')}"`;
}
return token;
})
.join(" OR ");
const rows = this.db.prepare(`
SELECT a.taskJson
FROM archived_tasks a
JOIN archived_tasks_fts fts ON a.rowid = fts.rowid
WHERE archived_tasks_fts MATCH ?
ORDER BY rank
LIMIT ?
`).all(ftsQuery, limit) as Array<{ taskJson: string }>;
return rows.map((row) => JSON.parse(row.taskJson) as ArchivedTaskEntry);
}
// LIKE fallback
const searchColumns = ["id", "title", "description", "comments"];
const perTokenClause = `(${searchColumns
.map((c) => `"${c}" LIKE ? ESCAPE '\\'`)
.join(" OR ")})`;
const whereTokens = tokens.map(() => perTokenClause).join(" OR ");
const params: (string | number)[] = [];
for (const token of tokens) {
const pattern = `%${token.replace(/[\\%_]/g, "\\$&")}%`;
for (let i = 0; i < searchColumns.length; i++) params.push(pattern);
}
params.push(limit);
const rows = this.db.prepare(`
SELECT a.taskJson
FROM archived_tasks a
JOIN archived_tasks_fts fts ON a.rowid = fts.rowid
WHERE archived_tasks_fts MATCH ?
ORDER BY rank
SELECT taskJson
FROM archived_tasks
WHERE ${whereTokens}
ORDER BY archivedAt DESC
LIMIT ?
`).all(query, limit) as Array<{ taskJson: string }>;
`).all(...params) as Array<{ taskJson: string }>;
return rows.map((row) => JSON.parse(row.taskJson) as ArchivedTaskEntry);
}

51
packages/core/src/db.ts
View File

@@ -57,6 +57,33 @@ export function fromJson<T>(json: string | null | undefined): T | undefined {
}
}
// ── Runtime capability probes ────────────────────────────────────────
/**
* Probe whether this SQLite build supports the FTS5 extension.
*
* Node's built-in `node:sqlite` only exposes FTS5 when the bundled SQLite was
* compiled with `SQLITE_ENABLE_FTS5`. Newer Node builds (≥ 22.13, 24, 25) have
* it on; some older 22.x LTS builds do not, and attempting to
* `CREATE VIRTUAL TABLE … USING fts5(…)` on those throws `no such module: fts5`.
*
* The probe creates and drops a disposable virtual table. Set
* `FUSION_DISABLE_FTS5=1` to force the LIKE fallback path in environments where
* FTS5 is available at probe time but undesirable at runtime (e.g. tests).
*/
export function probeFts5(db: DatabaseSync): boolean {
if (process.env.FUSION_DISABLE_FTS5 === "1" || process.env.FUSION_DISABLE_FTS5 === "true") {
return false;
}
try {
db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS __fusion_fts5_probe USING fts5(x)");
db.exec("DROP TABLE IF EXISTS __fusion_fts5_probe");
return true;
} catch {
return false;
}
}
// ── Schema Definition ────────────────────────────────────────────────
const SCHEMA_VERSION = 42;
@@ -589,6 +616,7 @@ export class Database {
private readonly dbPath: string;
/** Tracks transaction nesting depth for savepoint-based nested transactions. */
private transactionDepth = 0;
private readonly _fts5Available: boolean;
constructor(kbDir: string) {
this.dbPath = join(kbDir, "fusion.db");
@@ -610,6 +638,18 @@ export class Database {
this.db.exec("PRAGMA busy_timeout = 5000");
// Enable foreign key enforcement
this.db.exec("PRAGMA foreign_keys = ON");
this._fts5Available = probeFts5(this.db);
}
/**
* True when the underlying SQLite build has FTS5 (`CREATE VIRTUAL TABLE … USING fts5`).
* Node's bundled SQLite only exposes FTS5 when built with `SQLITE_ENABLE_FTS5`;
* older Node 22.x LTS builds do not. Consumers must fall back to LIKE-based scans
* when this is false. Override with `FUSION_DISABLE_FTS5=1` to force the fallback path.
*/
get fts5Available(): boolean {
return this._fts5Available;
}
/**
@@ -972,6 +1012,12 @@ export class Database {
// up comment text, IDs, timestamps, and author names. This is acceptable for v1.
if (version < 21) {
this.applyMigration(21, () => {
if (!this._fts5Available) {
// FTS5 unavailable (older node:sqlite build). Bump the migration
// version so we don't retry forever, and fall back to LIKE-based
// search in TaskStore.searchTasks / ArchiveDatabase.search.
return;
}
// Create FTS5 virtual table for full-text search
// Note: Column names must match the tasks table for external content mode to work
this.db.exec(`
@@ -1501,6 +1547,11 @@ export class Database {
// log-only executor updates should not churn or bloat the FTS index.
if (version < 35) {
this.applyMigration(35, () => {
if (!this._fts5Available) {
// tasks_fts does not exist when FTS5 is unavailable; nothing to
// rebuild or re-trigger.
return;
}
const hasTaskTitle = this.hasColumn("tasks", "title");
const updateColumns = hasTaskTitle
? "id, title, description, comments"

283
packages/core/src/fts5-guard.test.ts Normal file
View File

@@ -0,0 +1,283 @@
/**
* Regression tests for the FTS5 runtime guard.
*
* On Node builds whose bundled SQLite lacks FTS5 (older 22.x LTS),
* `CREATE VIRTUAL TABLE … USING fts5(…)` throws `no such module: fts5`
* and the dashboard crashes on first-run DB migration. These tests lock in
* the fallback path: init() must succeed, and search() must route through
* LIKE-based SQL.
*
* The `FUSION_DISABLE_FTS5=1` env var forces the probe to report FTS5 as
* unavailable even on runtimes that support it — so the CI machine can
* exercise the same code path a fresh install on an old Node would hit.
*/
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { mkdtempSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { rm } from "node:fs/promises";
import { Database } from "./db.js";
import { ArchiveDatabase } from "./archive-db.js";
import { TaskStore } from "./store.js";
function makeTmpDir(): string {
return mkdtempSync(join(tmpdir(), "kb-fts5-guard-test-"));
}
describe("FTS5 runtime guard", () => {
let prevEnv: string | undefined;
beforeEach(() => {
prevEnv = process.env.FUSION_DISABLE_FTS5;
process.env.FUSION_DISABLE_FTS5 = "1";
});
afterEach(() => {
if (prevEnv === undefined) {
delete process.env.FUSION_DISABLE_FTS5;
} else {
process.env.FUSION_DISABLE_FTS5 = prevEnv;
}
});
describe("Database", () => {
let tmpDir: string;
let kbDir: string;
let db: Database;
beforeEach(() => {
tmpDir = makeTmpDir();
kbDir = join(tmpDir, ".fusion");
db = new Database(kbDir);
});
afterEach(async () => {
try { db.close(); } catch { /* already closed */ }
await rm(tmpDir, { recursive: true, force: true });
});
it("reports fts5Available=false when FUSION_DISABLE_FTS5 is set", () => {
expect(db.fts5Available).toBe(false);
});
it("init() does not throw when FTS5 is unavailable", () => {
expect(() => db.init()).not.toThrow();
});
it("skips creating tasks_fts virtual table", () => {
db.init();
const row = db.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='tasks_fts'"
).get() as { name: string } | undefined;
expect(row).toBeUndefined();
});
it("skips creating FTS5 triggers", () => {
db.init();
const triggers = db.prepare(
"SELECT name FROM sqlite_master WHERE type='trigger'"
).all() as { name: string }[];
const ftsTriggers = triggers.filter((t) => t.name.startsWith("tasks_fts_"));
expect(ftsTriggers).toHaveLength(0);
});
it("still advances the schemaVersion so migrations don't retry", () => {
db.init();
const row = db.prepare(
"SELECT value FROM __meta WHERE key = 'schemaVersion'"
).get() as { value: string };
// Migration 21 guards FTS5; 35 also guards. The final version is
// the full SCHEMA_VERSION regardless of FTS5 availability.
expect(Number(row.value)).toBeGreaterThanOrEqual(35);
});
});
describe("TaskStore.searchTasks LIKE fallback", () => {
let rootDir: string;
let globalDir: string;
let store: TaskStore;
beforeEach(async () => {
rootDir = makeTmpDir();
globalDir = makeTmpDir();
store = new TaskStore(rootDir, globalDir);
await store.init();
});
afterEach(async () => {
store.close();
await rm(rootDir, { recursive: true, force: true });
await rm(globalDir, { recursive: true, force: true });
});
it("finds tasks by exact id match", async () => {
await store.createTask({ description: "First task" });
await store.createTask({ description: "Second task" });
const results = await store.searchTasks("FN-001");
expect(results).toHaveLength(1);
expect(results[0].id).toBe("FN-001");
});
it("finds tasks by title substring", async () => {
await store.createTask({ title: "Fix login bug", description: "Login issue" });
await store.createTask({ title: "Add dashboard feature", description: "New UI" });
const results = await store.searchTasks("dashboard");
expect(results).toHaveLength(1);
expect(results[0].title).toBe("Add dashboard feature");
});
it("finds tasks by description substring", async () => {
await store.createTask({ description: "Fix the login button on the homepage" });
await store.createTask({ description: "Update the settings page layout" });
const results = await store.searchTasks("homepage");
expect(results).toHaveLength(1);
expect(results[0].description).toContain("homepage");
});
it("finds tasks by comment text", async () => {
const task = await store.createTask({ description: "A task" });
await store.addComment(task.id, "Need to prioritize the xylophone implementation", "tester");
const results = await store.searchTasks("xylophone");
expect(results).toHaveLength(1);
expect(results[0].id).toBe(task.id);
});
it("is case insensitive (LIKE on SQLite is ASCII-case-insensitive)", async () => {
await store.createTask({ title: "UPPERCASE SEARCH TEST", description: "x" });
const results = await store.searchTasks("uppercase");
expect(results).toHaveLength(1);
});
it("uses OR semantics across tokens", async () => {
await store.createTask({ title: "Fix login", description: "Button issues" });
await store.createTask({ title: "Add dashboard", description: "New features" });
const results = await store.searchTasks("login dashboard");
expect(results).toHaveLength(2);
});
it("returns empty array for non-matching query", async () => {
await store.createTask({ description: "Regular task description" });
const results = await store.searchTasks("xyznonexistent12345");
expect(results).toHaveLength(0);
});
it("escapes LIKE metacharacters in user input", async () => {
await store.createTask({ description: "this has 100% coverage" });
await store.createTask({ description: "the word percent does not have a literal" });
// "100%" with a literal percent should match only the first task,
// not every task via wildcard.
const results = await store.searchTasks("100%");
expect(results).toHaveLength(1);
expect(results[0].description).toContain("100%");
});
it("respects limit option", async () => {
await store.createTask({ title: "widget alpha", description: "x" });
await store.createTask({ title: "widget beta", description: "x" });
await store.createTask({ title: "widget gamma", description: "x" });
const results = await store.searchTasks("widget", { limit: 2 });
expect(results).toHaveLength(2);
});
it("excludes archived tasks when includeArchived is false", async () => {
const uniqueTerm = `archguardterm${Date.now()}`;
const task = await store.createTask({ description: `archived ${uniqueTerm}` });
await store.moveTask(task.id, "todo");
await store.moveTask(task.id, "in-progress");
await store.moveTask(task.id, "in-review");
await store.moveTask(task.id, "done");
await store.archiveTask(task.id);
const withArchived = await store.searchTasks(uniqueTerm);
const withoutArchived = await store.searchTasks(uniqueTerm, { includeArchived: false });
expect(withArchived.some((r) => r.id === task.id)).toBe(true);
expect(withoutArchived.some((r) => r.id === task.id)).toBe(false);
});
});
describe("ArchiveDatabase.search LIKE fallback", () => {
let tmpDir: string;
let kbDir: string;
let archive: ArchiveDatabase;
beforeEach(() => {
tmpDir = makeTmpDir();
kbDir = join(tmpDir, ".fusion");
archive = new ArchiveDatabase(kbDir);
archive.init();
});
afterEach(async () => {
try { archive.close(); } catch { /* already closed */ }
await rm(tmpDir, { recursive: true, force: true });
});
it("reports fts5Available=false under the env override", () => {
expect(archive.fts5Available).toBe(false);
});
it("init() does not throw when FTS5 is unavailable", () => {
// init was called in beforeEach; re-running should still work
expect(() => archive.init()).not.toThrow();
});
it("skips creating archived_tasks_fts virtual table", () => {
// Direct probe via sqlite_master — exposed through Database's prepared
// statement interface isn't available here, so we test via a known
// side effect: search() must still return results.
archive.upsert({
id: "FN-ARCH-001",
archivedAt: "2026-01-01T00:00:00.000Z",
createdAt: "2025-12-01T00:00:00.000Z",
updatedAt: "2025-12-02T00:00:00.000Z",
title: "archived widget alpha",
description: "this is an archived task about widgets",
comments: [],
} as any);
const results = archive.search("widget", 10);
expect(results).toHaveLength(1);
expect(results[0].id).toBe("FN-ARCH-001");
});
it("finds archived tasks via LIKE across id, title, description, comments", () => {
archive.upsert({
id: "FN-ARCH-002",
archivedAt: "2026-01-02T00:00:00.000Z",
createdAt: "2025-12-01T00:00:00.000Z",
updatedAt: "2025-12-02T00:00:00.000Z",
title: "unrelated",
description: "task mentions xylophone in the body",
comments: [],
} as any);
archive.upsert({
id: "FN-ARCH-003",
archivedAt: "2026-01-03T00:00:00.000Z",
createdAt: "2025-12-03T00:00:00.000Z",
updatedAt: "2025-12-03T00:00:00.000Z",
title: "unrelated",
description: "no match here",
comments: [],
} as any);
const results = archive.search("xylophone", 10);
expect(results.map((r) => r.id)).toEqual(["FN-ARCH-002"]);
});
it("returns empty array for empty or whitespace-only query", () => {
expect(archive.search("", 10)).toEqual([]);
expect(archive.search(" ", 10)).toEqual([]);
});
});
});

72
packages/core/src/store.ts
View File

@@ -1991,7 +1991,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
return this.listTasks(options);
}
// Sanitize query for FTS5 safety: strip dangerous operators but preserve alphanumeric
// Sanitize query: strip FTS5 operators so both code paths see the same token set
const sanitizedTokens = trimmedQuery
.split(/\s+/)
.filter((token) => token.length > 0)
@@ -2002,34 +2002,56 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
return this.listTasks(options);
}
// For FTS5 MATCH, quote tokens that contain special characters like hyphens
// to prevent them from being interpreted as operators
const ftsQuery = sanitizedTokens
.map((token) => {
// If token contains FTS5 special chars, wrap in double quotes
if (/[":(){}*^+-]/.test(token)) {
return `"${token.replace(/"/g, '\\"')}"`;
}
return token;
})
.join(" OR ");
// Execute FTS query with ranking
const limit = options?.limit ?? -1;
const offset = options?.offset ?? 0;
const offsetClause = offset > 0 ? ` OFFSET ${offset}` : "";
const includeArchived = options?.includeArchived ?? true;
const whereClause = includeArchived ? "" : ` AND t."column" != 'archived'`;
const selectClause = this.getTaskSelectClause(options?.slim ?? false, "t");
const slim = options?.slim ?? false;
const selectClause = this.getTaskSelectClause(slim, "t");
const rows = this.db.prepare(`
SELECT ${selectClause} FROM tasks t
JOIN tasks_fts fts ON t.rowid = fts.rowid
WHERE tasks_fts MATCH ?
${whereClause}
ORDER BY rank
LIMIT ${limit >= 0 ? limit : -1}${offsetClause}
`).all(ftsQuery) as any[];
let rows: any[];
if (this.db.fts5Available) {
// For FTS5 MATCH, quote tokens that contain special characters like hyphens
// to prevent them from being interpreted as operators
const ftsQuery = sanitizedTokens
.map((token) => {
if (/[":(){}*^+-]/.test(token)) {
return `"${token.replace(/"/g, '\\"')}"`;
}
return token;
})
.join(" OR ");
const whereClause = includeArchived ? "" : ` AND t."column" != 'archived'`;
rows = this.db.prepare(`
SELECT ${selectClause} FROM tasks t
JOIN tasks_fts fts ON t.rowid = fts.rowid
WHERE tasks_fts MATCH ?
${whereClause}
ORDER BY rank
LIMIT ${limit >= 0 ? limit : -1}${offsetClause}
`).all(ftsQuery) as any[];
} else {
// LIKE fallback: any token matching any searchable column counts as a hit.
// Tokens are OR'd; per token we OR across id/title/description/comments.
// ESCAPE '\\' lets us include user input containing % or _ literally.
const searchColumns = ["id", "title", "description", "comments"];
const perTokenClause = `(${searchColumns
.map((c) => `t."${c}" LIKE ? ESCAPE '\\'`)
.join(" OR ")})`;
const whereTokens = sanitizedTokens.map(() => perTokenClause).join(" OR ");
const params: string[] = [];
for (const token of sanitizedTokens) {
const pattern = `%${token.replace(/[\\%_]/g, "\\$&")}%`;
for (let i = 0; i < searchColumns.length; i++) params.push(pattern);
}
const archivedClause = includeArchived ? "" : ` AND t."column" != 'archived'`;
rows = this.db.prepare(`
SELECT ${selectClause} FROM tasks t
WHERE (${whereTokens})${archivedClause}
ORDER BY t.createdAt ASC
LIMIT ${limit >= 0 ? limit : -1}${offsetClause}
`).all(...params) as any[];
}
const activeMatches = await Promise.all(rows.map(async (row) => {
const task = this.rowToTask(row);
@@ -2041,7 +2063,7 @@ export class TaskStore extends EventEmitter<TaskStoreEvents> {
return steps.length > 0 ? { ...task, steps } : task;
}));
const archiveMatches = includeArchived
? this.archiveDb.search(ftsQuery, limit >= 0 ? limit : 100).map((entry) => this.archiveEntryToTask(entry, options?.slim ?? false))
? this.archiveDb.search(trimmedQuery, limit >= 0 ? limit : 100).map((entry) => this.archiveEntryToTask(entry, slim))
: [];
const matches = [...activeMatches, ...archiveMatches];