feat(FN-4094): cache schema compatibility checks with fingerprint fast path

Adds schema fingerprinting to cache schema compatibility checks, enabling a fast path that skips unnecessary re-initialization work on startup, with performance tests covering the new code paths and updated storage documentation.

Fusion-Task-Id: FN-4094

.changeset/fn-4094-db-init-perf.md

@@ -0,0 +1,5 @@
+---
+"@runfusion/fusion": patch
+---
+
+Optimize Database.init() schema-compatibility passes: cache per-table PRAGMA results within init and short-circuit unchanged-schema opens via a `schemaCompatFingerprint` in `__meta`. Reduces repeated `db.init()` wall time substantially without weakening the FN-3879/FN-3887/FN-3898 invariant that every declared column exists after init.

docs/storage.md

@@ -248,9 +248,12 @@ The `tasks.githubTracking` JSON column stores per-task GitHub tracking state (`e
 
 ### Schema self-heal on init
 
-`Database.init()` now runs an unconditional schema-compatibility reconciliation pass after versioned migrations. The pass unions table definitions from `SCHEMA_SQL` plus `MIGRATION_ONLY_TABLE_SCHEMAS`, then backfills missing columns with `addColumnIfMissing()` for tables that already exist.
+`Database.init()` runs versioned migrations first, then checks `__meta.schemaCompatFingerprint` against a process-local fingerprint derived from `SCHEMA_VERSION` plus the canonicalized table declarations from `SCHEMA_SQL` and `MIGRATION_ONLY_TABLE_SCHEMAS`.
 
-Invariant: after init, every declared column for covered tables exists regardless of `__meta.schemaVersion`, preventing legacy drift from causing `no such column` regressions on newly added fields.
+- **Fingerprint match:** skip the expensive column-reconciliation walk, but still run the cheap idempotent side effects that must always happen on open (for example `CREATE INDEX IF NOT EXISTS ...` and routines NULL backfills).
+- **Fingerprint absent or mismatched:** run the full schema-compatibility reconciliation pass, unioning table definitions from `SCHEMA_SQL` plus `MIGRATION_ONLY_TABLE_SCHEMAS` and backfilling missing columns on tables that already exist, then persist the new fingerprint.
+
+Invariant: after init, every declared column for covered tables exists regardless of `__meta.schemaVersion` whenever the fingerprint is stale or missing, preventing legacy drift from causing `no such column` regressions on newly added fields while keeping unchanged-schema opens fast.
 
 ---
 

packages/core/src/__tests__/architecture-schema-compat.test.ts

@@ -11,8 +11,8 @@ function readDbSource(): string {
 describe("architecture schema compatibility", () => {
   it("invokes ensureSchemaCompatibility() from init()", () => {
     const source = readDbSource();
-    expect(source).toMatch(/private ensureSchemaCompatibility\(\): void/);
+    expect(source).toMatch(/private ensureSchemaCompatibility\(options: SchemaCompatibilityOptions = \{\}\): void/);
-    expect(source).toMatch(/this\.migrate\(\);\s*[\s\S]*?this\.ensureSchemaCompatibility\(\);/);
+    expect(source).toMatch(/this\.migrate\(\);\s*[\s\S]*?this\.ensureSchemaCompatibility\([^)]*\);\s*[\s\S]*?this\.ensureRoutinesSchemaCompatibility\([^)]*\);\s*[\s\S]*?this\.ensureInsightRunsSchemaCompatibility\([^)]*\);\s*[\s\S]*?this\.ensureEvalTaskResultsSchemaCompatibility\([^)]*\);/);
   });
 
   it("restores missing declared columns for SCHEMA_SQL tables", () => {

packages/core/src/__tests__/db-init-perf.test.ts

@@ -0,0 +1,120 @@
+import { describe, expect, it, vi } from "vitest";
+import { Database, SCHEMA_COMPAT_FINGERPRINT } from "../db.js";
+
+function createInMemoryDatabase(): Database {
+  return new Database("/tmp/fn-db-init-perf", { inMemory: true });
+}
+
+function getMetaValue(db: Database, key: string): string | null {
+  const row = db.prepare("SELECT value FROM __meta WHERE key = ?").get(key) as { value: string } | undefined;
+  return row?.value ?? null;
+}
+
+function getColumnNames(db: Database, table: string): string[] {
+  return (db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map((column) => column.name);
+}
+
+function median(values: number[]): number {
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0
+    ? (sorted[middle - 1] + sorted[middle]) / 2
+    : sorted[middle];
+}
+
+describe("Database.init() schema compatibility performance", () => {
+  it("writes schemaCompatFingerprint to __meta for a fresh database", () => {
+    const db = createInMemoryDatabase();
+
+    try {
+      db.init();
+
+      expect(getMetaValue(db, "schemaCompatFingerprint")).toBe(SCHEMA_COMPAT_FINGERPRINT);
+    } finally {
+      db.close();
+    }
+  });
+
+  it("skips ALTER TABLE work and keeps PRAGMA table_info calls under a strict ceiling on unchanged-schema re-init", () => {
+    const db = createInMemoryDatabase();
+
+    try {
+      db.init();
+
+      const execSpy = vi.spyOn((db as any).db, "exec");
+      const prepareSpy = vi.spyOn((db as any).db, "prepare");
+
+      db.init();
+
+      const alterTableStatements = execSpy.mock.calls.filter(([sql]) => sql.includes("ALTER TABLE"));
+      expect(alterTableStatements).toHaveLength(0);
+
+      const pragmaTableInfoCalls = prepareSpy.mock.calls.filter(([sql]) => sql.includes("PRAGMA table_info("));
+      // Current-schema re-init still probes tasks twice from migrate()'s legacy guard;
+      // the fingerprint hit should prevent the broader schema-compatibility sweep.
+      expect(pragmaTableInfoCalls.length).toBeLessThanOrEqual(2);
+    } finally {
+      db.close();
+    }
+  });
+
+  it("restores a missing declared column when the fingerprint is absent", () => {
+    const db = createInMemoryDatabase();
+
+    try {
+      db.init();
+      db.exec("ALTER TABLE tasks DROP COLUMN modifiedFiles");
+      db.exec("DELETE FROM __meta WHERE key = 'schemaCompatFingerprint'");
+
+      expect(getColumnNames(db, "tasks")).not.toContain("modifiedFiles");
+
+      db.init();
+
+      expect(getColumnNames(db, "tasks")).toContain("modifiedFiles");
+      expect(getMetaValue(db, "schemaCompatFingerprint")).toBe(SCHEMA_COMPAT_FINGERPRINT);
+    } finally {
+      db.close();
+    }
+  });
+
+  it("restores a missing declared column when the fingerprint is stale", () => {
+    const db = createInMemoryDatabase();
+
+    try {
+      db.init();
+      db.exec("ALTER TABLE tasks DROP COLUMN modifiedFiles");
+      db.exec("INSERT OR REPLACE INTO __meta (key, value) VALUES ('schemaCompatFingerprint', 'stale-fingerprint')");
+
+      expect(getColumnNames(db, "tasks")).not.toContain("modifiedFiles");
+
+      db.init();
+
+      expect(getColumnNames(db, "tasks")).toContain("modifiedFiles");
+      expect(getMetaValue(db, "schemaCompatFingerprint")).toBe(SCHEMA_COMPAT_FINGERPRINT);
+    } finally {
+      db.close();
+    }
+  });
+
+  it("keeps repeated unchanged-schema init() calls comfortably below the coarse perf guard", () => {
+    const db = createInMemoryDatabase();
+
+    try {
+      db.init();
+
+      const durationsMs: number[] = [];
+      for (let index = 0; index < 50; index += 1) {
+        const startedAt = process.hrtime.bigint();
+        db.init();
+        const endedAt = process.hrtime.bigint();
+        durationsMs.push(Number(endedAt - startedAt) / 1_000_000);
+      }
+
+      // Coarse local/CI-safe guard: unchanged-schema re-init should stay well below
+      // tens of milliseconds once the fingerprint short-circuits reconciliation.
+      expect(median(durationsMs)).toBeLessThan(50);
+    } finally {
+      db.close();
+    }
+  });
+});

packages/core/src/__tests__/insight-store.test.ts

@@ -985,6 +985,7 @@ describe("Migration: pre-33 DB upgrade", () => {
       `);
       db1.exec("DROP TABLE project_insight_runs");
       db1.exec("ALTER TABLE project_insight_runs_legacy RENAME TO project_insight_runs");
+      db1.exec("DELETE FROM __meta WHERE key = 'schemaCompatFingerprint'");
 
       // Verify lifecycle column is gone
       const colsBefore = db1.prepare("PRAGMA table_info(project_insight_runs)").all() as Array<{ name: string }>;

packages/core/src/db.ts

@@ -12,7 +12,7 @@ import { DatabaseSync } from "./sqlite-adapter.js";
 import { isAbsolute, join } from "node:path";
 import { mkdirSync, existsSync, statSync } from "node:fs";
 import { spawnSync } from "node:child_process";
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { DEFAULT_PROJECT_SETTINGS } from "./types.js";
 import type { PluginOnSchemaInit } from "./plugin-types.js";
 import type { SteeringComment, TaskComment } from "./types.js";
@@ -34,6 +34,12 @@ const DEFAULT_SQLITE_LOCK_RECOVERY_WINDOW_MS = 1_000;
 const DEFAULT_SQLITE_LOCK_RECOVERY_DELAY_MS = 50;
 
 type TransactionMode = "deferred" | "immediate";
+type TableColumnsCache = Map<string, Set<string>>;
+
+type SchemaCompatibilityOptions = {
+  tableColumnsCache?: TableColumnsCache;
+  skipColumnReconciliation?: boolean;
+};
 
 // ── JSON Helpers ─────────────────────────────────────────────────────
 
@@ -903,6 +909,19 @@ export function getSchemaCompatibilityTableSchemas(): Map<string, Map<string, st
   return tables;
 }
 
+function canonicalizeSchemaTables(tables: Map<string, Map<string, string>>): Record<string, Record<string, string>> {
+  return Object.fromEntries(
+    [...tables.entries()]
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(([tableName, columns]) => [
+        tableName,
+        Object.fromEntries(
+          [...columns.entries()].sort(([left], [right]) => left.localeCompare(right)),
+        ),
+      ]),
+  );
+}
+
 export const MIGRATION_ONLY_TABLE_SCHEMAS: Record<string, Record<string, string>> = {
   ai_sessions: {
     id: "TEXT PRIMARY KEY",
@@ -1099,6 +1118,33 @@ export const MIGRATION_ONLY_TABLE_SCHEMAS: Record<string, Record<string, string>
   },
 };
 
+/**
+ * Process-local fingerprint of the additive schema compatibility contract.
+ *
+ * The hash covers the current schema version plus the canonicalized column
+ * declarations from both SCHEMA_SQL and MIGRATION_ONLY_TABLE_SCHEMAS, so any
+ * schema edit that changes the compatibility surface automatically invalidates
+ * the persisted __meta cache on next init().
+ */
+export const SCHEMA_COMPAT_FINGERPRINT = createHash("sha1")
+  .update(
+    JSON.stringify({
+      schemaVersion: SCHEMA_VERSION,
+      schemaSqlTables: canonicalizeSchemaTables(SCHEMA_TABLE_SCHEMAS),
+      migrationOnlyTableSchemas: Object.fromEntries(
+        Object.entries(MIGRATION_ONLY_TABLE_SCHEMAS)
+          .sort(([left], [right]) => left.localeCompare(right))
+          .map(([tableName, columns]) => [
+            tableName,
+            Object.fromEntries(
+              Object.entries(columns).sort(([left], [right]) => left.localeCompare(right)),
+            ),
+          ]),
+      ),
+    }),
+  )
+  .digest("hex");
+
 // ── Database Class ───────────────────────────────────────────────────
 
 type SharedIntegrityCheckState = {
@@ -1397,11 +1443,23 @@ export class Database {
     // Run schema migrations
     this.migrate();
 
+    const schemaCompatFingerprint = this.getMetaValue("schemaCompatFingerprint");
+    const skipColumnReconciliation = schemaCompatFingerprint === SCHEMA_COMPAT_FINGERPRINT;
+    const tableColumnsCache = skipColumnReconciliation ? undefined : new Map<string, Set<string>>();
+    const compatibilityOptions: SchemaCompatibilityOptions = {
+      tableColumnsCache,
+      skipColumnReconciliation,
+    };
+
     // Compatibility backfills that must run even when schemaVersion is current.
-    this.ensureSchemaCompatibility();
+    this.ensureSchemaCompatibility(compatibilityOptions);
-    this.ensureRoutinesSchemaCompatibility();
+    this.ensureRoutinesSchemaCompatibility(compatibilityOptions);
-    this.ensureInsightRunsSchemaCompatibility();
+    this.ensureInsightRunsSchemaCompatibility(compatibilityOptions);
-    this.ensureEvalTaskResultsSchemaCompatibility();
+    this.ensureEvalTaskResultsSchemaCompatibility(compatibilityOptions);
+
+    if (!skipColumnReconciliation) {
+      this.setMetaValue("schemaCompatFingerprint", SCHEMA_COMPAT_FINGERPRINT);
+    }
 
     // Seed config row idempotently with default settings
     const configNow = new Date().toISOString();
@@ -1422,21 +1480,29 @@ export class Database {
    * re-run even if a previous migration partially applied.
    */
   /**
-   * Applies unconditional column reconciliation for all known project DB tables.
+   * Reconciles additive columns for every known project DB table unless the
+   * persisted `schemaCompatFingerprint` already matches SCHEMA_COMPAT_FINGERPRINT.
    *
-   * FN-3879 introduced a tasks checkout-column self-heal, FN-3898 formalized it,
+   * The fingerprint is invalidated automatically by SCHEMA_VERSION changes and by
-   * and FN-3887 generalized the guardrail so migration-version drift no longer
+   * edits to the canonicalized column declarations from SCHEMA_SQL or
-   * determines whether additive columns exist. Invariant: every column declared
+   * MIGRATION_ONLY_TABLE_SCHEMAS. When it is absent or stale, this method runs the
-   * in SCHEMA_SQL or MIGRATION_ONLY_TABLE_SCHEMAS exists on any live table after
+   * full FN-3879/FN-3887/FN-3898 safety pass so every declared column exists on
-   * this method returns, regardless of the persisted schemaVersion.
+   * every live table after init() returns.
    */
-  private ensureSchemaCompatibility(): void {
+  private ensureSchemaCompatibility(options: SchemaCompatibilityOptions = {}): void {
+    if (options.skipColumnReconciliation) {
+      return;
+    }
+
     const knownTableSchemas = getSchemaCompatibilityTableSchemas();
+    const tableColumnsCache = options.tableColumnsCache;
 
     for (const [tableName, columns] of knownTableSchemas) {
       if (!this.hasTable(tableName)) continue;
+      const cachedColumns = this.getTableColumns(tableName, true, tableColumnsCache);
       for (const [columnName, columnDefinition] of columns) {
-        this.addColumnIfMissing(tableName, columnName, columnDefinition);
+        if (cachedColumns.has(columnName)) continue;
+        this.addColumnIfMissingCached(tableName, columnName, columnDefinition, tableColumnsCache);
       }
     }
   }
@@ -1448,11 +1514,16 @@ export class Database {
    * agent IDs from earlier table definitions. `RoutineStore.rowToRoutine()` and
    * backup routine sync expect a safe string value, so normalize to ''.
    */
-  private ensureRoutinesSchemaCompatibility(): void {
+  private ensureRoutinesSchemaCompatibility(options: SchemaCompatibilityOptions = {}): void {
     if (!this.hasTable("routines")) {
       return;
     }
 
+    if (!options.skipColumnReconciliation) {
+      this.addColumnIfMissingCached("routines", "agentId", "TEXT DEFAULT ''", options.tableColumnsCache);
+      this.addColumnIfMissingCached("routines", "scope", "TEXT DEFAULT 'project'", options.tableColumnsCache);
+    }
+
     this.db.exec("UPDATE routines SET agentId = '' WHERE agentId IS NULL");
     this.db.exec("UPDATE routines SET scope = 'project' WHERE scope IS NULL OR TRIM(scope) = ''");
 
@@ -1468,15 +1539,20 @@ export class Database {
    * remains focused on index creation that should run after the generic column
    * backfill pass.
    */
-  private ensureInsightRunsSchemaCompatibility(): void {
+  private ensureInsightRunsSchemaCompatibility(options: SchemaCompatibilityOptions = {}): void {
     if (!this.hasTable("project_insight_runs")) {
       return;
     }
 
+    if (!options.skipColumnReconciliation) {
+      this.addColumnIfMissingCached("project_insight_runs", "lifecycle", "TEXT", options.tableColumnsCache);
+      this.addColumnIfMissingCached("project_insight_runs", "cancelledAt", "TEXT", options.tableColumnsCache);
+    }
+
     this.db.exec(`CREATE INDEX IF NOT EXISTS idxInsightRunsProjectTriggerStatus ON project_insight_runs(projectId, trigger, status)`);
   }
 
-  private ensureEvalTaskResultsSchemaCompatibility(): void {
+  private ensureEvalTaskResultsSchemaCompatibility(_options: SchemaCompatibilityOptions = {}): void {
     if (!this.hasTable("eval_task_results")) {
       return;
     }
@@ -3108,14 +3184,30 @@ export class Database {
     );
   }
 
+  /**
+   * Read the declared columns for a table.
+   */
+  private getTableColumns(table: string, useCache = false, cache?: TableColumnsCache): Set<string> {
+    if (useCache && cache?.has(table)) {
+      return cache.get(table) ?? new Set<string>();
+    }
+
+    const columns = new Set(
+      (this.db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map((column) => column.name),
+    );
+
+    if (useCache && cache) {
+      cache.set(table, columns);
+    }
+
+    return columns;
+  }
+
   /**
    * Check whether a table has a given column.
    */
   private hasColumn(table: string, column: string): boolean {
-    const cols = this.db
+    return this.getTableColumns(table).has(column);
-      .prepare(`PRAGMA table_info(${table})`)
-      .all() as Array<{ name: string }>;
-    return cols.some((c) => c.name === column);
   }
 
   /**
@@ -3127,6 +3219,27 @@ export class Database {
     }
   }
 
+  /**
+   * Add a column using a per-init table-info cache when available.
+   */
+  private addColumnIfMissingCached(
+    table: string,
+    column: string,
+    definition: string,
+    cache?: TableColumnsCache,
+  ): void {
+    const columns = this.getTableColumns(table, Boolean(cache), cache);
+    if (columns.has(column)) {
+      return;
+    }
+
+    this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
+    columns.add(column);
+    if (cache) {
+      cache.set(table, columns);
+    }
+  }
+
   /**
    * Normalize legacy steering comments into the unified comments field exactly once.
    *
@@ -3381,16 +3494,28 @@ export class Database {
     this.db.exec(sql);
   }
 
+  private getMetaValue(key: string): string | undefined {
+    const row = this.db.prepare("SELECT value FROM __meta WHERE key = ?").get(key) as
+      | { value: string }
+      | undefined;
+    return row?.value;
+  }
+
+  /**
+   * Persist a __meta value idempotently.
+   */
+  private setMetaValue(key: string, value: string): void {
+    this.db.prepare("INSERT OR REPLACE INTO __meta (key, value) VALUES (?, ?)").run(key, value);
+  }
+
   /**
    * Get the last modification timestamp (epoch ms).
    * Returns 0 if the value is not set.
    */
   getLastModified(): number {
-    const row = this.db.prepare("SELECT value FROM __meta WHERE key = 'lastModified'").get() as
+    const value = this.getMetaValue("lastModified");
-      | { value: string }
+    if (!value) return 0;
-      | undefined;
+    return parseInt(value, 10) || 0;
-    if (!row) return 0;
-    return parseInt(row.value, 10) || 0;
   }
 
   /**
@@ -3411,11 +3536,9 @@ export class Database {
    * Get the schema version number.
    */
   getSchemaVersion(): number {
-    const row = this.db.prepare("SELECT value FROM __meta WHERE key = 'schemaVersion'").get() as
+    const value = this.getMetaValue("schemaVersion");
-      | { value: string }
+    if (!value) return 0;
-      | undefined;
+    return parseInt(value, 10) || 0;
-    if (!row) return 0;
-    return parseInt(row.value, 10) || 0;
   }
 
   /**