feat(FN-4094): cache schema compatibility checks with fingerprint fast path
Adds schema fingerprinting to cache schema compatibility checks, enabling a fast path that skips unnecessary re-initialization work on startup, with performance tests covering the new code paths and updated storage documentation. Fusion-Task-Id: FN-4094
This commit is contained in:
Fusion
@@ -12,7 +12,7 @@ import { DatabaseSync } from "./sqlite-adapter.js";
|
||||
import { isAbsolute, join } from "node:path";
|
||||
import { mkdirSync, existsSync, statSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { DEFAULT_PROJECT_SETTINGS } from "./types.js";
|
||||
import type { PluginOnSchemaInit } from "./plugin-types.js";
|
||||
import type { SteeringComment, TaskComment } from "./types.js";
|
||||
@@ -34,6 +34,12 @@ const DEFAULT_SQLITE_LOCK_RECOVERY_WINDOW_MS = 1_000;
|
||||
const DEFAULT_SQLITE_LOCK_RECOVERY_DELAY_MS = 50;
|
||||
|
||||
type TransactionMode = "deferred" | "immediate";
|
||||
type TableColumnsCache = Map<string, Set<string>>;
|
||||
|
||||
type SchemaCompatibilityOptions = {
|
||||
tableColumnsCache?: TableColumnsCache;
|
||||
skipColumnReconciliation?: boolean;
|
||||
};
|
||||
|
||||
// ── JSON Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
@@ -903,6 +909,19 @@ export function getSchemaCompatibilityTableSchemas(): Map<string, Map<string, st
|
||||
return tables;
|
||||
}
|
||||
|
||||
function canonicalizeSchemaTables(tables: Map<string, Map<string, string>>): Record<string, Record<string, string>> {
|
||||
return Object.fromEntries(
|
||||
[...tables.entries()]
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([tableName, columns]) => [
|
||||
tableName,
|
||||
Object.fromEntries(
|
||||
[...columns.entries()].sort(([left], [right]) => left.localeCompare(right)),
|
||||
),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
export const MIGRATION_ONLY_TABLE_SCHEMAS: Record<string, Record<string, string>> = {
|
||||
ai_sessions: {
|
||||
id: "TEXT PRIMARY KEY",
|
||||
@@ -1099,6 +1118,33 @@ export const MIGRATION_ONLY_TABLE_SCHEMAS: Record<string, Record<string, string>
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Process-local fingerprint of the additive schema compatibility contract.
|
||||
*
|
||||
* The hash covers the current schema version plus the canonicalized column
|
||||
* declarations from both SCHEMA_SQL and MIGRATION_ONLY_TABLE_SCHEMAS, so any
|
||||
* schema edit that changes the compatibility surface automatically invalidates
|
||||
* the persisted __meta cache on next init().
|
||||
*/
|
||||
export const SCHEMA_COMPAT_FINGERPRINT = createHash("sha1")
|
||||
.update(
|
||||
JSON.stringify({
|
||||
schemaVersion: SCHEMA_VERSION,
|
||||
schemaSqlTables: canonicalizeSchemaTables(SCHEMA_TABLE_SCHEMAS),
|
||||
migrationOnlyTableSchemas: Object.fromEntries(
|
||||
Object.entries(MIGRATION_ONLY_TABLE_SCHEMAS)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([tableName, columns]) => [
|
||||
tableName,
|
||||
Object.fromEntries(
|
||||
Object.entries(columns).sort(([left], [right]) => left.localeCompare(right)),
|
||||
),
|
||||
]),
|
||||
),
|
||||
}),
|
||||
)
|
||||
.digest("hex");
|
||||
|
||||
// ── Database Class ───────────────────────────────────────────────────
|
||||
|
||||
type SharedIntegrityCheckState = {
|
||||
@@ -1397,11 +1443,23 @@ export class Database {
|
||||
// Run schema migrations
|
||||
this.migrate();
|
||||
|
||||
const schemaCompatFingerprint = this.getMetaValue("schemaCompatFingerprint");
|
||||
const skipColumnReconciliation = schemaCompatFingerprint === SCHEMA_COMPAT_FINGERPRINT;
|
||||
const tableColumnsCache = skipColumnReconciliation ? undefined : new Map<string, Set<string>>();
|
||||
const compatibilityOptions: SchemaCompatibilityOptions = {
|
||||
tableColumnsCache,
|
||||
skipColumnReconciliation,
|
||||
};
|
||||
|
||||
// Compatibility backfills that must run even when schemaVersion is current.
|
||||
this.ensureSchemaCompatibility();
|
||||
this.ensureRoutinesSchemaCompatibility();
|
||||
this.ensureInsightRunsSchemaCompatibility();
|
||||
this.ensureEvalTaskResultsSchemaCompatibility();
|
||||
this.ensureSchemaCompatibility(compatibilityOptions);
|
||||
this.ensureRoutinesSchemaCompatibility(compatibilityOptions);
|
||||
this.ensureInsightRunsSchemaCompatibility(compatibilityOptions);
|
||||
this.ensureEvalTaskResultsSchemaCompatibility(compatibilityOptions);
|
||||
|
||||
if (!skipColumnReconciliation) {
|
||||
this.setMetaValue("schemaCompatFingerprint", SCHEMA_COMPAT_FINGERPRINT);
|
||||
}
|
||||
|
||||
// Seed config row idempotently with default settings
|
||||
const configNow = new Date().toISOString();
|
||||
@@ -1422,21 +1480,29 @@ export class Database {
|
||||
* re-run even if a previous migration partially applied.
|
||||
*/
|
||||
/**
|
||||
* Applies unconditional column reconciliation for all known project DB tables.
|
||||
* Reconciles additive columns for every known project DB table unless the
|
||||
* persisted `schemaCompatFingerprint` already matches SCHEMA_COMPAT_FINGERPRINT.
|
||||
*
|
||||
* FN-3879 introduced a tasks checkout-column self-heal, FN-3898 formalized it,
|
||||
* and FN-3887 generalized the guardrail so migration-version drift no longer
|
||||
* determines whether additive columns exist. Invariant: every column declared
|
||||
* in SCHEMA_SQL or MIGRATION_ONLY_TABLE_SCHEMAS exists on any live table after
|
||||
* this method returns, regardless of the persisted schemaVersion.
|
||||
* The fingerprint is invalidated automatically by SCHEMA_VERSION changes and by
|
||||
* edits to the canonicalized column declarations from SCHEMA_SQL or
|
||||
* MIGRATION_ONLY_TABLE_SCHEMAS. When it is absent or stale, this method runs the
|
||||
* full FN-3879/FN-3887/FN-3898 safety pass so every declared column exists on
|
||||
* every live table after init() returns.
|
||||
*/
|
||||
private ensureSchemaCompatibility(): void {
|
||||
private ensureSchemaCompatibility(options: SchemaCompatibilityOptions = {}): void {
|
||||
if (options.skipColumnReconciliation) {
|
||||
return;
|
||||
}
|
||||
|
||||
const knownTableSchemas = getSchemaCompatibilityTableSchemas();
|
||||
const tableColumnsCache = options.tableColumnsCache;
|
||||
|
||||
for (const [tableName, columns] of knownTableSchemas) {
|
||||
if (!this.hasTable(tableName)) continue;
|
||||
const cachedColumns = this.getTableColumns(tableName, true, tableColumnsCache);
|
||||
for (const [columnName, columnDefinition] of columns) {
|
||||
this.addColumnIfMissing(tableName, columnName, columnDefinition);
|
||||
if (cachedColumns.has(columnName)) continue;
|
||||
this.addColumnIfMissingCached(tableName, columnName, columnDefinition, tableColumnsCache);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1448,11 +1514,16 @@ export class Database {
|
||||
* agent IDs from earlier table definitions. `RoutineStore.rowToRoutine()` and
|
||||
* backup routine sync expect a safe string value, so normalize to ''.
|
||||
*/
|
||||
private ensureRoutinesSchemaCompatibility(): void {
|
||||
private ensureRoutinesSchemaCompatibility(options: SchemaCompatibilityOptions = {}): void {
|
||||
if (!this.hasTable("routines")) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!options.skipColumnReconciliation) {
|
||||
this.addColumnIfMissingCached("routines", "agentId", "TEXT DEFAULT ''", options.tableColumnsCache);
|
||||
this.addColumnIfMissingCached("routines", "scope", "TEXT DEFAULT 'project'", options.tableColumnsCache);
|
||||
}
|
||||
|
||||
this.db.exec("UPDATE routines SET agentId = '' WHERE agentId IS NULL");
|
||||
this.db.exec("UPDATE routines SET scope = 'project' WHERE scope IS NULL OR TRIM(scope) = ''");
|
||||
|
||||
@@ -1468,15 +1539,20 @@ export class Database {
|
||||
* remains focused on index creation that should run after the generic column
|
||||
* backfill pass.
|
||||
*/
|
||||
private ensureInsightRunsSchemaCompatibility(): void {
|
||||
private ensureInsightRunsSchemaCompatibility(options: SchemaCompatibilityOptions = {}): void {
|
||||
if (!this.hasTable("project_insight_runs")) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!options.skipColumnReconciliation) {
|
||||
this.addColumnIfMissingCached("project_insight_runs", "lifecycle", "TEXT", options.tableColumnsCache);
|
||||
this.addColumnIfMissingCached("project_insight_runs", "cancelledAt", "TEXT", options.tableColumnsCache);
|
||||
}
|
||||
|
||||
this.db.exec(`CREATE INDEX IF NOT EXISTS idxInsightRunsProjectTriggerStatus ON project_insight_runs(projectId, trigger, status)`);
|
||||
}
|
||||
|
||||
private ensureEvalTaskResultsSchemaCompatibility(): void {
|
||||
private ensureEvalTaskResultsSchemaCompatibility(_options: SchemaCompatibilityOptions = {}): void {
|
||||
if (!this.hasTable("eval_task_results")) {
|
||||
return;
|
||||
}
|
||||
@@ -3108,14 +3184,30 @@ export class Database {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the declared columns for a table.
|
||||
*/
|
||||
private getTableColumns(table: string, useCache = false, cache?: TableColumnsCache): Set<string> {
|
||||
if (useCache && cache?.has(table)) {
|
||||
return cache.get(table) ?? new Set<string>();
|
||||
}
|
||||
|
||||
const columns = new Set(
|
||||
(this.db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map((column) => column.name),
|
||||
);
|
||||
|
||||
if (useCache && cache) {
|
||||
cache.set(table, columns);
|
||||
}
|
||||
|
||||
return columns;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a table has a given column.
|
||||
*/
|
||||
private hasColumn(table: string, column: string): boolean {
|
||||
const cols = this.db
|
||||
.prepare(`PRAGMA table_info(${table})`)
|
||||
.all() as Array<{ name: string }>;
|
||||
return cols.some((c) => c.name === column);
|
||||
return this.getTableColumns(table).has(column);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -3127,6 +3219,27 @@ export class Database {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a column using a per-init table-info cache when available.
|
||||
*/
|
||||
private addColumnIfMissingCached(
|
||||
table: string,
|
||||
column: string,
|
||||
definition: string,
|
||||
cache?: TableColumnsCache,
|
||||
): void {
|
||||
const columns = this.getTableColumns(table, Boolean(cache), cache);
|
||||
if (columns.has(column)) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
||||
columns.add(column);
|
||||
if (cache) {
|
||||
cache.set(table, columns);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize legacy steering comments into the unified comments field exactly once.
|
||||
*
|
||||
@@ -3381,16 +3494,28 @@ export class Database {
|
||||
this.db.exec(sql);
|
||||
}
|
||||
|
||||
private getMetaValue(key: string): string | undefined {
|
||||
const row = this.db.prepare("SELECT value FROM __meta WHERE key = ?").get(key) as
|
||||
| { value: string }
|
||||
| undefined;
|
||||
return row?.value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a __meta value idempotently.
|
||||
*/
|
||||
private setMetaValue(key: string, value: string): void {
|
||||
this.db.prepare("INSERT OR REPLACE INTO __meta (key, value) VALUES (?, ?)").run(key, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the last modification timestamp (epoch ms).
|
||||
* Returns 0 if the value is not set.
|
||||
*/
|
||||
getLastModified(): number {
|
||||
const row = this.db.prepare("SELECT value FROM __meta WHERE key = 'lastModified'").get() as
|
||||
| { value: string }
|
||||
| undefined;
|
||||
if (!row) return 0;
|
||||
return parseInt(row.value, 10) || 0;
|
||||
const value = this.getMetaValue("lastModified");
|
||||
if (!value) return 0;
|
||||
return parseInt(value, 10) || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -3411,11 +3536,9 @@ export class Database {
|
||||
* Get the schema version number.
|
||||
*/
|
||||
getSchemaVersion(): number {
|
||||
const row = this.db.prepare("SELECT value FROM __meta WHERE key = 'schemaVersion'").get() as
|
||||
| { value: string }
|
||||
| undefined;
|
||||
if (!row) return 0;
|
||||
return parseInt(row.value, 10) || 0;
|
||||
const value = this.getMetaValue("schemaVersion");
|
||||
if (!value) return 0;
|
||||
return parseInt(value, 10) || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user