root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-4094): cache schema compatibility checks with fingerprint fast path

Browse Source 
Adds schema fingerprinting to cache schema compatibility checks, enabling a fast path that skips unnecessary re-initialization work on startup, with performance tests covering the new code paths and updated storage documentation.

Fusion-Task-Id: FN-4094
This commit is contained in:
Fusion
2026-05-12 07:15:20 -07:00
committed by gsxdsm
parent 2d46a882f0
commit db1bbda369
6 changed files with 286 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/core/src/__tests__/architecture-schema-compat.test.ts
View File

@@ -11,8 +11,8 @@ function readDbSource(): string {
describe("architecture schema compatibility", () => {
it("invokes ensureSchemaCompatibility() from init()", () => {
const source = readDbSource();
expect(source).toMatch(/private ensureSchemaCompatibility\(\): void/);
expect(source).toMatch(/this\.migrate\(\);\s*[\s\S]*?this\.ensureSchemaCompatibility\(\);/);
expect(source).toMatch(/private ensureSchemaCompatibility\(options: SchemaCompatibilityOptions = \{\}\): void/);
expect(source).toMatch(/this\.migrate\(\);\s*[\s\S]*?this\.ensureSchemaCompatibility\([^)]*\);\s*[\s\S]*?this\.ensureRoutinesSchemaCompatibility\([^)]*\);\s*[\s\S]*?this\.ensureInsightRunsSchemaCompatibility\([^)]*\);\s*[\s\S]*?this\.ensureEvalTaskResultsSchemaCompatibility\([^)]*\);/);
});
it("restores missing declared columns for SCHEMA_SQL tables", () => {

120
packages/core/src/__tests__/db-init-perf.test.ts Normal file
View File

@@ -0,0 +1,120 @@
import { describe, expect, it, vi } from "vitest";
import { Database, SCHEMA_COMPAT_FINGERPRINT } from "../db.js";
function createInMemoryDatabase(): Database {
return new Database("/tmp/fn-db-init-perf", { inMemory: true });
}
function getMetaValue(db: Database, key: string): string | null {
const row = db.prepare("SELECT value FROM __meta WHERE key = ?").get(key) as { value: string } | undefined;
return row?.value ?? null;
}
function getColumnNames(db: Database, table: string): string[] {
return (db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map((column) => column.name);
}
function median(values: number[]): number {
const sorted = [...values].sort((left, right) => left - right);
const middle = Math.floor(sorted.length / 2);
return sorted.length % 2 === 0
? (sorted[middle - 1] + sorted[middle]) / 2
: sorted[middle];
}
describe("Database.init() schema compatibility performance", () => {
it("writes schemaCompatFingerprint to __meta for a fresh database", () => {
const db = createInMemoryDatabase();
try {
db.init();
expect(getMetaValue(db, "schemaCompatFingerprint")).toBe(SCHEMA_COMPAT_FINGERPRINT);
} finally {
db.close();
}
});
it("skips ALTER TABLE work and keeps PRAGMA table_info calls under a strict ceiling on unchanged-schema re-init", () => {
const db = createInMemoryDatabase();
try {
db.init();
const execSpy = vi.spyOn((db as any).db, "exec");
const prepareSpy = vi.spyOn((db as any).db, "prepare");
db.init();
const alterTableStatements = execSpy.mock.calls.filter(([sql]) => sql.includes("ALTER TABLE"));
expect(alterTableStatements).toHaveLength(0);
const pragmaTableInfoCalls = prepareSpy.mock.calls.filter(([sql]) => sql.includes("PRAGMA table_info("));
// Current-schema re-init still probes tasks twice from migrate()'s legacy guard;
// the fingerprint hit should prevent the broader schema-compatibility sweep.
expect(pragmaTableInfoCalls.length).toBeLessThanOrEqual(2);
} finally {
db.close();
}
});
it("restores a missing declared column when the fingerprint is absent", () => {
const db = createInMemoryDatabase();
try {
db.init();
db.exec("ALTER TABLE tasks DROP COLUMN modifiedFiles");
db.exec("DELETE FROM __meta WHERE key = 'schemaCompatFingerprint'");
expect(getColumnNames(db, "tasks")).not.toContain("modifiedFiles");
db.init();
expect(getColumnNames(db, "tasks")).toContain("modifiedFiles");
expect(getMetaValue(db, "schemaCompatFingerprint")).toBe(SCHEMA_COMPAT_FINGERPRINT);
} finally {
db.close();
}
});
it("restores a missing declared column when the fingerprint is stale", () => {
const db = createInMemoryDatabase();
try {
db.init();
db.exec("ALTER TABLE tasks DROP COLUMN modifiedFiles");
db.exec("INSERT OR REPLACE INTO __meta (key, value) VALUES ('schemaCompatFingerprint', 'stale-fingerprint')");
expect(getColumnNames(db, "tasks")).not.toContain("modifiedFiles");
db.init();
expect(getColumnNames(db, "tasks")).toContain("modifiedFiles");
expect(getMetaValue(db, "schemaCompatFingerprint")).toBe(SCHEMA_COMPAT_FINGERPRINT);
} finally {
db.close();
}
});
it("keeps repeated unchanged-schema init() calls comfortably below the coarse perf guard", () => {
const db = createInMemoryDatabase();
try {
db.init();
const durationsMs: number[] = [];
for (let index = 0; index < 50; index += 1) {
const startedAt = process.hrtime.bigint();
db.init();
const endedAt = process.hrtime.bigint();
durationsMs.push(Number(endedAt - startedAt) / 1_000_000);
}
// Coarse local/CI-safe guard: unchanged-schema re-init should stay well below
// tens of milliseconds once the fingerprint short-circuits reconciliation.
expect(median(durationsMs)).toBeLessThan(50);
} finally {
db.close();
}
});
});

1
packages/core/src/__tests__/insight-store.test.ts
View File

@@ -985,6 +985,7 @@ describe("Migration: pre-33 DB upgrade", () => {
`);
db1.exec("DROP TABLE project_insight_runs");
db1.exec("ALTER TABLE project_insight_runs_legacy RENAME TO project_insight_runs");
db1.exec("DELETE FROM __meta WHERE key = 'schemaCompatFingerprint'");
// Verify lifecycle column is gone
const colsBefore = db1.prepare("PRAGMA table_info(project_insight_runs)").all() as Array<{ name: string }>;

183
packages/core/src/db.ts
View File

@@ -12,7 +12,7 @@ import { DatabaseSync } from "./sqlite-adapter.js";
import { isAbsolute, join } from "node:path";
import { mkdirSync, existsSync, statSync } from "node:fs";
import { spawnSync } from "node:child_process";
import { randomUUID } from "node:crypto";
import { createHash, randomUUID } from "node:crypto";
import { DEFAULT_PROJECT_SETTINGS } from "./types.js";
import type { PluginOnSchemaInit } from "./plugin-types.js";
import type { SteeringComment, TaskComment } from "./types.js";
@@ -34,6 +34,12 @@ const DEFAULT_SQLITE_LOCK_RECOVERY_WINDOW_MS = 1_000;
const DEFAULT_SQLITE_LOCK_RECOVERY_DELAY_MS = 50;
type TransactionMode = "deferred" | "immediate";
type TableColumnsCache = Map<string, Set<string>>;
type SchemaCompatibilityOptions = {
tableColumnsCache?: TableColumnsCache;
skipColumnReconciliation?: boolean;
};
// ── JSON Helpers ─────────────────────────────────────────────────────
@@ -903,6 +909,19 @@ export function getSchemaCompatibilityTableSchemas(): Map<string, Map<string, st
return tables;
}
function canonicalizeSchemaTables(tables: Map<string, Map<string, string>>): Record<string, Record<string, string>> {
return Object.fromEntries(
[...tables.entries()]
.sort(([left], [right]) => left.localeCompare(right))
.map(([tableName, columns]) => [
tableName,
Object.fromEntries(
[...columns.entries()].sort(([left], [right]) => left.localeCompare(right)),
),
]),
);
}
export const MIGRATION_ONLY_TABLE_SCHEMAS: Record<string, Record<string, string>> = {
ai_sessions: {
id: "TEXT PRIMARY KEY",
@@ -1099,6 +1118,33 @@ export const MIGRATION_ONLY_TABLE_SCHEMAS: Record<string, Record<string, string>
},
};
/**
* Process-local fingerprint of the additive schema compatibility contract.
*
* The hash covers the current schema version plus the canonicalized column
* declarations from both SCHEMA_SQL and MIGRATION_ONLY_TABLE_SCHEMAS, so any
* schema edit that changes the compatibility surface automatically invalidates
* the persisted __meta cache on next init().
*/
export const SCHEMA_COMPAT_FINGERPRINT = createHash("sha1")
.update(
JSON.stringify({
schemaVersion: SCHEMA_VERSION,
schemaSqlTables: canonicalizeSchemaTables(SCHEMA_TABLE_SCHEMAS),
migrationOnlyTableSchemas: Object.fromEntries(
Object.entries(MIGRATION_ONLY_TABLE_SCHEMAS)
.sort(([left], [right]) => left.localeCompare(right))
.map(([tableName, columns]) => [
tableName,
Object.fromEntries(
Object.entries(columns).sort(([left], [right]) => left.localeCompare(right)),
),
]),
),
}),
)
.digest("hex");
// ── Database Class ───────────────────────────────────────────────────
type SharedIntegrityCheckState = {
@@ -1397,11 +1443,23 @@ export class Database {
// Run schema migrations
this.migrate();
const schemaCompatFingerprint = this.getMetaValue("schemaCompatFingerprint");
const skipColumnReconciliation = schemaCompatFingerprint === SCHEMA_COMPAT_FINGERPRINT;
const tableColumnsCache = skipColumnReconciliation ? undefined : new Map<string, Set<string>>();
const compatibilityOptions: SchemaCompatibilityOptions = {
tableColumnsCache,
skipColumnReconciliation,
};
// Compatibility backfills that must run even when schemaVersion is current.
this.ensureSchemaCompatibility();
this.ensureRoutinesSchemaCompatibility();
this.ensureInsightRunsSchemaCompatibility();
this.ensureEvalTaskResultsSchemaCompatibility();
this.ensureSchemaCompatibility(compatibilityOptions);
this.ensureRoutinesSchemaCompatibility(compatibilityOptions);
this.ensureInsightRunsSchemaCompatibility(compatibilityOptions);
this.ensureEvalTaskResultsSchemaCompatibility(compatibilityOptions);
if (!skipColumnReconciliation) {
this.setMetaValue("schemaCompatFingerprint", SCHEMA_COMPAT_FINGERPRINT);
}
// Seed config row idempotently with default settings
const configNow = new Date().toISOString();
@@ -1422,21 +1480,29 @@ export class Database {
* re-run even if a previous migration partially applied.
*/
/**
* Applies unconditional column reconciliation for all known project DB tables.
* Reconciles additive columns for every known project DB table unless the
* persisted `schemaCompatFingerprint` already matches SCHEMA_COMPAT_FINGERPRINT.
*
* FN-3879 introduced a tasks checkout-column self-heal, FN-3898 formalized it,
* and FN-3887 generalized the guardrail so migration-version drift no longer
* determines whether additive columns exist. Invariant: every column declared
* in SCHEMA_SQL or MIGRATION_ONLY_TABLE_SCHEMAS exists on any live table after
* this method returns, regardless of the persisted schemaVersion.
* The fingerprint is invalidated automatically by SCHEMA_VERSION changes and by
* edits to the canonicalized column declarations from SCHEMA_SQL or
* MIGRATION_ONLY_TABLE_SCHEMAS. When it is absent or stale, this method runs the
* full FN-3879/FN-3887/FN-3898 safety pass so every declared column exists on
* every live table after init() returns.
*/
private ensureSchemaCompatibility(): void {
private ensureSchemaCompatibility(options: SchemaCompatibilityOptions = {}): void {
if (options.skipColumnReconciliation) {
return;
}
const knownTableSchemas = getSchemaCompatibilityTableSchemas();
const tableColumnsCache = options.tableColumnsCache;
for (const [tableName, columns] of knownTableSchemas) {
if (!this.hasTable(tableName)) continue;
const cachedColumns = this.getTableColumns(tableName, true, tableColumnsCache);
for (const [columnName, columnDefinition] of columns) {
this.addColumnIfMissing(tableName, columnName, columnDefinition);
if (cachedColumns.has(columnName)) continue;
this.addColumnIfMissingCached(tableName, columnName, columnDefinition, tableColumnsCache);
}
}
}
@@ -1448,11 +1514,16 @@ export class Database {
* agent IDs from earlier table definitions. `RoutineStore.rowToRoutine()` and
* backup routine sync expect a safe string value, so normalize to ''.
*/
private ensureRoutinesSchemaCompatibility(): void {
private ensureRoutinesSchemaCompatibility(options: SchemaCompatibilityOptions = {}): void {
if (!this.hasTable("routines")) {
return;
}
if (!options.skipColumnReconciliation) {
this.addColumnIfMissingCached("routines", "agentId", "TEXT DEFAULT ''", options.tableColumnsCache);
this.addColumnIfMissingCached("routines", "scope", "TEXT DEFAULT 'project'", options.tableColumnsCache);
}
this.db.exec("UPDATE routines SET agentId = '' WHERE agentId IS NULL");
this.db.exec("UPDATE routines SET scope = 'project' WHERE scope IS NULL OR TRIM(scope) = ''");
@@ -1468,15 +1539,20 @@ export class Database {
* remains focused on index creation that should run after the generic column
* backfill pass.
*/
private ensureInsightRunsSchemaCompatibility(): void {
private ensureInsightRunsSchemaCompatibility(options: SchemaCompatibilityOptions = {}): void {
if (!this.hasTable("project_insight_runs")) {
return;
}
if (!options.skipColumnReconciliation) {
this.addColumnIfMissingCached("project_insight_runs", "lifecycle", "TEXT", options.tableColumnsCache);
this.addColumnIfMissingCached("project_insight_runs", "cancelledAt", "TEXT", options.tableColumnsCache);
}
this.db.exec(`CREATE INDEX IF NOT EXISTS idxInsightRunsProjectTriggerStatus ON project_insight_runs(projectId, trigger, status)`);
}
private ensureEvalTaskResultsSchemaCompatibility(): void {
private ensureEvalTaskResultsSchemaCompatibility(_options: SchemaCompatibilityOptions = {}): void {
if (!this.hasTable("eval_task_results")) {
return;
}
@@ -3108,14 +3184,30 @@ export class Database {
);
}
/**
* Read the declared columns for a table.
*/
private getTableColumns(table: string, useCache = false, cache?: TableColumnsCache): Set<string> {
if (useCache && cache?.has(table)) {
return cache.get(table) ?? new Set<string>();
}
const columns = new Set(
(this.db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map((column) => column.name),
);
if (useCache && cache) {
cache.set(table, columns);
}
return columns;
}
/**
* Check whether a table has a given column.
*/
private hasColumn(table: string, column: string): boolean {
const cols = this.db
.prepare(`PRAGMA table_info(${table})`)
.all() as Array<{ name: string }>;
return cols.some((c) => c.name === column);
return this.getTableColumns(table).has(column);
}
/**
@@ -3127,6 +3219,27 @@ export class Database {
}
}
/**
* Add a column using a per-init table-info cache when available.
*/
private addColumnIfMissingCached(
table: string,
column: string,
definition: string,
cache?: TableColumnsCache,
): void {
const columns = this.getTableColumns(table, Boolean(cache), cache);
if (columns.has(column)) {
return;
}
this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
columns.add(column);
if (cache) {
cache.set(table, columns);
}
}
/**
* Normalize legacy steering comments into the unified comments field exactly once.
*
@@ -3381,16 +3494,28 @@ export class Database {
this.db.exec(sql);
}
private getMetaValue(key: string): string | undefined {
const row = this.db.prepare("SELECT value FROM __meta WHERE key = ?").get(key) as
| { value: string }
| undefined;
return row?.value;
}
/**
* Persist a __meta value idempotently.
*/
private setMetaValue(key: string, value: string): void {
this.db.prepare("INSERT OR REPLACE INTO __meta (key, value) VALUES (?, ?)").run(key, value);
}
/**
* Get the last modification timestamp (epoch ms).
* Returns 0 if the value is not set.
*/
getLastModified(): number {
const row = this.db.prepare("SELECT value FROM __meta WHERE key = 'lastModified'").get() as
| { value: string }
| undefined;
if (!row) return 0;
return parseInt(row.value, 10) || 0;
const value = this.getMetaValue("lastModified");
if (!value) return 0;
return parseInt(value, 10) || 0;
}
/**
@@ -3411,11 +3536,9 @@ export class Database {
* Get the schema version number.
*/
getSchemaVersion(): number {
const row = this.db.prepare("SELECT value FROM __meta WHERE key = 'schemaVersion'").get() as
| { value: string }
| undefined;
if (!row) return 0;
return parseInt(row.value, 10) || 0;
const value = this.getMetaValue("schemaVersion");
if (!value) return 0;
return parseInt(value, 10) || 0;
}
/**