feat(KB-310): migrate storage layer to SQLite with WAL mode and auto-migration

- Add SQLite database foundation with WAL mode, transactions, and JSON column helpers
- Implement auto-migration system from legacy file-based storage with .bak backup strategy
- Migrate TaskStore to SQLite with full test coverage (256 tests)
- Migrate AutomationStore to SQLite with full test coverage (48 tests)
- Add SQLite Storage Architecture documentation to AGENTS.md
- Include changeset for patch release
This commit is contained in:
gsxdsm
2026-03-31 13:04:34 -07:00
parent 9af06c81dc
commit 1c788455ff
12 changed files with 2673 additions and 237 deletions

View File

@@ -0,0 +1,645 @@
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { Database, createDatabase, toJson, toJsonNullable, fromJson } from "./db.js";
import { mkdtempSync, existsSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { rm } from "node:fs/promises";
function makeTmpDir(): string {
return mkdtempSync(join(tmpdir(), "kb-db-test-"));
}
describe("Database", () => {
let tmpDir: string;
let kbDir: string;
let db: Database;
beforeEach(() => {
tmpDir = makeTmpDir();
kbDir = join(tmpDir, ".kb");
db = new Database(kbDir);
db.init(); // Explicit init required — createDatabase() does not auto-init
});
afterEach(async () => {
try {
db.close();
} catch {
// already closed
}
await rm(tmpDir, { recursive: true, force: true });
});
describe("initialization", () => {
it("creates the database file", () => {
expect(existsSync(join(kbDir, "kb.db"))).toBe(true);
});
it("creates the .kb directory if missing", () => {
expect(existsSync(kbDir)).toBe(true);
});
it("sets WAL journal mode", () => {
const row = db.prepare("PRAGMA journal_mode").get() as { journal_mode: string };
expect(row.journal_mode).toBe("wal");
});
it("enables foreign keys", () => {
const row = db.prepare("PRAGMA foreign_keys").get() as { foreign_keys: number };
expect(row.foreign_keys).toBe(1);
});
it("creates all expected tables", () => {
const tables = db.prepare(
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
).all() as { name: string }[];
const tableNames = tables.map((t) => t.name).sort();
expect(tableNames).toContain("tasks");
expect(tableNames).toContain("config");
expect(tableNames).toContain("activityLog");
expect(tableNames).toContain("archivedTasks");
expect(tableNames).toContain("automations");
expect(tableNames).toContain("agents");
expect(tableNames).toContain("agentHeartbeats");
expect(tableNames).toContain("__meta");
});
it("creates all expected indexes", () => {
const indexes = db.prepare(
"SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%' ORDER BY name"
).all() as { name: string }[];
const indexNames = indexes.map((i) => i.name).sort();
expect(indexNames).toContain("idxActivityLogTimestamp");
expect(indexNames).toContain("idxActivityLogType");
expect(indexNames).toContain("idxActivityLogTaskId");
expect(indexNames).toContain("idxArchivedTasksId");
expect(indexNames).toContain("idxAgentHeartbeatsAgentId");
expect(indexNames).toContain("idxAgentHeartbeatsRunId");
});
it("seeds schema version", () => {
expect(db.getSchemaVersion()).toBe(1);
});
it("seeds lastModified", () => {
const ts = db.getLastModified();
expect(ts).toBeGreaterThan(0);
expect(ts).toBeLessThanOrEqual(Date.now());
});
it("seeds config row with all required fields", () => {
const row = db.prepare("SELECT * FROM config WHERE id = 1").get() as any;
expect(row).toBeDefined();
expect(row.nextId).toBe(1);
expect(row.nextWorkflowStepId).toBe(1);
expect(row.settings).toBe("{}");
expect(row.workflowSteps).toBe("[]");
expect(row.updatedAt).toBeTruthy();
// updatedAt should be a valid ISO timestamp
expect(new Date(row.updatedAt).toISOString()).toBe(row.updatedAt);
});
it("is idempotent - calling init() twice does not fail", () => {
expect(() => db.init()).not.toThrow();
expect(db.getSchemaVersion()).toBe(1);
});
it("does not overwrite existing config on re-init", () => {
// Update the config
db.prepare("UPDATE config SET nextId = 42 WHERE id = 1").run();
// Re-init
db.init();
// Should keep updated value
const row = db.prepare("SELECT nextId FROM config WHERE id = 1").get() as any;
expect(row.nextId).toBe(42);
});
});
describe("change detection", () => {
it("getLastModified returns a timestamp", () => {
const ts = db.getLastModified();
expect(typeof ts).toBe("number");
expect(ts).toBeGreaterThan(0);
});
it("bumpLastModified strictly increases the timestamp", () => {
// Set lastModified to a known past value
db.prepare("UPDATE __meta SET value = '1000' WHERE key = 'lastModified'").run();
expect(db.getLastModified()).toBe(1000);
db.bumpLastModified();
const after = db.getLastModified();
expect(after).toBeGreaterThan(1000);
});
it("bumpLastModified is monotonic across rapid consecutive calls", () => {
const values: number[] = [];
for (let i = 0; i < 5; i++) {
db.bumpLastModified();
values.push(db.getLastModified());
}
// Each value must be strictly greater than the previous
for (let i = 1; i < values.length; i++) {
expect(values[i]).toBeGreaterThan(values[i - 1]);
}
});
it("lastModified survives close and reopen", () => {
db.bumpLastModified();
const ts = db.getLastModified();
expect(ts).toBeGreaterThan(0);
// Close and reopen
db.close();
const db2 = new Database(kbDir);
db2.init();
expect(db2.getLastModified()).toBe(ts);
db2.close();
// Re-assign so afterEach doesn't fail
db = new Database(kbDir);
db.init();
});
it("lastModified is stored as a row in __meta", () => {
db.bumpLastModified();
const row = db.prepare("SELECT key, value FROM __meta WHERE key = 'lastModified'").get() as { key: string; value: string };
expect(row).toBeDefined();
expect(row.key).toBe("lastModified");
expect(parseInt(row.value, 10)).toBeGreaterThan(0);
});
it("both schemaVersion and lastModified exist in __meta", () => {
const rows = db.prepare("SELECT key FROM __meta ORDER BY key").all() as { key: string }[];
const keys = rows.map(r => r.key);
expect(keys).toContain("schemaVersion");
expect(keys).toContain("lastModified");
});
});
describe("transactions", () => {
it("commits on success", () => {
db.transaction(() => {
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-001", "Test task", "triage", "2025-01-01", "2025-01-01");
});
const row = db.prepare("SELECT * FROM tasks WHERE id = 'KB-001'").get() as any;
expect(row).toBeDefined();
expect(row.description).toBe("Test task");
});
it("rolls back on error", () => {
expect(() => {
db.transaction(() => {
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-002", "Test task 2", "triage", "2025-01-01", "2025-01-01");
throw new Error("Simulated failure");
});
}).toThrow("Simulated failure");
const row = db.prepare("SELECT * FROM tasks WHERE id = 'KB-002'").get();
expect(row).toBeUndefined();
});
it("returns the function result", () => {
const result = db.transaction(() => {
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-003", "Test", "todo", "2025-01-01", "2025-01-01");
return 42;
});
expect(result).toBe(42);
});
it("supports nested transactions via savepoints", () => {
db.transaction(() => {
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-OUTER", "Outer task", "triage", "2025-01-01", "2025-01-01");
// Nested transaction
db.transaction(() => {
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-INNER", "Inner task", "triage", "2025-01-01", "2025-01-01");
});
});
// Both should exist
const outer = db.prepare("SELECT * FROM tasks WHERE id = 'KB-OUTER'").get();
const inner = db.prepare("SELECT * FROM tasks WHERE id = 'KB-INNER'").get();
expect(outer).toBeDefined();
expect(inner).toBeDefined();
});
it("nested transaction rollback only affects inner scope", () => {
db.transaction(() => {
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-OUTER2", "Outer task 2", "triage", "2025-01-01", "2025-01-01");
try {
db.transaction(() => {
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-INNER2", "Inner task 2", "triage", "2025-01-01", "2025-01-01");
throw new Error("Inner failure");
});
} catch {
// Expected — inner transaction rolled back
}
});
// Outer should exist, inner should not
const outer = db.prepare("SELECT * FROM tasks WHERE id = 'KB-OUTER2'").get();
const inner = db.prepare("SELECT * FROM tasks WHERE id = 'KB-INNER2'").get();
expect(outer).toBeDefined();
expect(inner).toBeUndefined();
});
it("outer transaction can continue after inner rollback", () => {
db.transaction(() => {
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-PRE", "Before inner", "triage", "2025-01-01", "2025-01-01");
// Inner transaction fails
try {
db.transaction(() => {
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-FAIL", "Inner fail", "triage", "2025-01-01", "2025-01-01");
throw new Error("Inner failure");
});
} catch {
// Expected
}
// Additional work in outer transaction after inner rollback
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-POST", "After inner", "triage", "2025-01-01", "2025-01-01");
});
// PRE and POST should exist, FAIL should not
expect(db.prepare("SELECT * FROM tasks WHERE id = 'KB-PRE'").get()).toBeDefined();
expect(db.prepare("SELECT * FROM tasks WHERE id = 'KB-POST'").get()).toBeDefined();
expect(db.prepare("SELECT * FROM tasks WHERE id = 'KB-FAIL'").get()).toBeUndefined();
});
it("transaction is atomic — partial writes roll back", () => {
try {
db.transaction(() => {
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-A", "Task A", "triage", "2025-01-01", "2025-01-01");
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-B", "Task B", "triage", "2025-01-01", "2025-01-01");
// This should fail - duplicate PK
db.prepare(
"INSERT INTO tasks (id, description, \"column\", createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)"
).run("KB-A", "Duplicate", "triage", "2025-01-01", "2025-01-01");
});
} catch {
// expected
}
// Neither task should exist
const rowA = db.prepare("SELECT * FROM tasks WHERE id = 'KB-A'").get();
const rowB = db.prepare("SELECT * FROM tasks WHERE id = 'KB-B'").get();
expect(rowA).toBeUndefined();
expect(rowB).toBeUndefined();
});
});
describe("foreign key cascade", () => {
it("deleting an agent cascades to heartbeats", () => {
const now = new Date().toISOString();
db.prepare(
"INSERT INTO agents (id, name, role, state, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?)"
).run("agent-1", "Agent 1", "executor", "idle", now, now);
db.prepare(
"INSERT INTO agentHeartbeats (agentId, timestamp, status, runId) VALUES (?, ?, ?, ?)"
).run("agent-1", now, "ok", "run-1");
db.prepare(
"INSERT INTO agentHeartbeats (agentId, timestamp, status, runId) VALUES (?, ?, ?, ?)"
).run("agent-1", now, "ok", "run-1");
// Delete agent
db.prepare("DELETE FROM agents WHERE id = 'agent-1'").run();
// Heartbeats should be cascade-deleted
const heartbeats = db.prepare("SELECT * FROM agentHeartbeats WHERE agentId = 'agent-1'").all();
expect(heartbeats).toHaveLength(0);
});
});
describe("foreign key cascade across reopen", () => {
it("cascade delete works after closing and reopening the database", () => {
const now = new Date().toISOString();
// Insert agent and heartbeats
db.prepare(
"INSERT INTO agents (id, name, role, state, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?, ?)"
).run("agent-reopen", "Agent", "executor", "idle", now, now);
db.prepare(
"INSERT INTO agentHeartbeats (agentId, timestamp, status, runId) VALUES (?, ?, ?, ?)"
).run("agent-reopen", now, "ok", "run-1");
// Close and reopen
db.close();
db = new Database(kbDir);
db.init();
// Verify foreign key enforcement is active after reopen
const fk = db.prepare("PRAGMA foreign_keys").get() as { foreign_keys: number };
expect(fk.foreign_keys).toBe(1);
// Delete agent — heartbeats should cascade
db.prepare("DELETE FROM agents WHERE id = 'agent-reopen'").run();
const heartbeats = db.prepare("SELECT * FROM agentHeartbeats WHERE agentId = 'agent-reopen'").all();
expect(heartbeats).toHaveLength(0);
});
});
describe("task round-trip", () => {
it("stores and retrieves a fully populated task record", () => {
const now = new Date().toISOString();
const task = {
id: "KB-100",
title: "Full task test",
description: "Test all fields",
column: "in-progress",
status: "running",
size: "L",
reviewLevel: 3,
currentStep: 2,
worktree: "/tmp/wt",
blockedBy: "KB-099",
paused: 1,
baseBranch: "main",
modelPresetId: "complex",
modelProvider: "anthropic",
modelId: "claude-sonnet-4-5",
validatorModelProvider: "openai",
validatorModelId: "gpt-4o",
mergeRetries: 2,
error: "Something went wrong",
summary: "Fixed the bug",
thinkingLevel: "high",
createdAt: now,
updatedAt: now,
columnMovedAt: now,
dependencies: JSON.stringify(["KB-098", "KB-097"]),
steps: JSON.stringify([{ name: "Step 1", status: "done" }, { name: "Step 2", status: "in-progress" }]),
log: JSON.stringify([{ timestamp: now, action: "Created" }]),
attachments: JSON.stringify([{ filename: "test.png", originalName: "test.png", mimeType: "image/png", size: 1024, createdAt: now }]),
steeringComments: JSON.stringify([{ id: "c1", text: "Do this", createdAt: now, author: "user" }]),
workflowStepResults: JSON.stringify([{ workflowStepId: "WS-001", workflowStepName: "QA", status: "passed" }]),
prInfo: JSON.stringify({ url: "https://github.com/test/pr/1", number: 1, status: "open", title: "PR", headBranch: "feature", baseBranch: "main", commentCount: 0 }),
issueInfo: JSON.stringify({ url: "https://github.com/test/issues/1", number: 1, state: "open", title: "Issue" }),
breakIntoSubtasks: 1,
enabledWorkflowSteps: JSON.stringify(["WS-001", "WS-002"]),
};
db.prepare(`
INSERT INTO tasks (
id, title, description, "column", status, size, reviewLevel, currentStep,
worktree, blockedBy, paused, baseBranch, modelPresetId, modelProvider,
modelId, validatorModelProvider, validatorModelId, mergeRetries, error,
summary, thinkingLevel, createdAt, updatedAt, columnMovedAt,
dependencies, steps, log, attachments, steeringComments,
workflowStepResults, prInfo, issueInfo, breakIntoSubtasks,
enabledWorkflowSteps
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
`).run(
task.id, task.title, task.description, task.column, task.status,
task.size, task.reviewLevel, task.currentStep, task.worktree,
task.blockedBy, task.paused, task.baseBranch, task.modelPresetId,
task.modelProvider, task.modelId, task.validatorModelProvider,
task.validatorModelId, task.mergeRetries, task.error, task.summary,
task.thinkingLevel, task.createdAt, task.updatedAt, task.columnMovedAt,
task.dependencies, task.steps, task.log, task.attachments,
task.steeringComments, task.workflowStepResults, task.prInfo,
task.issueInfo, task.breakIntoSubtasks, task.enabledWorkflowSteps,
);
const row = db.prepare("SELECT * FROM tasks WHERE id = 'KB-100'").get() as any;
expect(row.id).toBe("KB-100");
expect(row.title).toBe("Full task test");
expect(row.column).toBe("in-progress");
expect(row.thinkingLevel).toBe("high");
expect(row.mergeRetries).toBe(2);
expect(row.paused).toBe(1);
expect(row.breakIntoSubtasks).toBe(1);
// Verify JSON round-trip
expect(JSON.parse(row.dependencies)).toEqual(["KB-098", "KB-097"]);
expect(JSON.parse(row.steps)).toHaveLength(2);
expect(JSON.parse(row.log)).toHaveLength(1);
expect(JSON.parse(row.attachments)).toHaveLength(1);
expect(JSON.parse(row.steeringComments)).toHaveLength(1);
expect(JSON.parse(row.workflowStepResults)).toHaveLength(1);
expect(JSON.parse(row.prInfo).number).toBe(1);
expect(JSON.parse(row.issueInfo).state).toBe("open");
expect(JSON.parse(row.enabledWorkflowSteps)).toEqual(["WS-001", "WS-002"]);
});
});
describe("config round-trip", () => {
it("stores and retrieves config with nested settings and workflow steps", () => {
const settings = {
maxConcurrent: 4,
autoMerge: false,
taskPrefix: "PROJ",
};
const workflowSteps = [
{ id: "WS-001", name: "Doc Review", description: "Review docs", prompt: "Check docs", enabled: true, createdAt: "2025-01-01", updatedAt: "2025-01-01" },
];
db.prepare("UPDATE config SET settings = ?, workflowSteps = ?, nextId = ?, nextWorkflowStepId = ? WHERE id = 1")
.run(JSON.stringify(settings), JSON.stringify(workflowSteps), 42, 2);
const row = db.prepare("SELECT * FROM config WHERE id = 1").get() as any;
expect(row.nextId).toBe(42);
expect(row.nextWorkflowStepId).toBe(2);
expect(JSON.parse(row.settings).maxConcurrent).toBe(4);
expect(JSON.parse(row.settings).taskPrefix).toBe("PROJ");
expect(JSON.parse(row.workflowSteps)).toHaveLength(1);
expect(JSON.parse(row.workflowSteps)[0].id).toBe("WS-001");
});
});
});
describe("JSON helpers", () => {
describe("toJson", () => {
it("stringifies arrays", () => {
expect(toJson(["a", "b"])).toBe('["a","b"]');
});
it("stringifies objects", () => {
expect(toJson({ a: 1 })).toBe('{"a":1}');
});
it("returns '[]' for empty arrays", () => {
expect(toJson([])).toBe("[]");
});
it("returns '[]' for undefined", () => {
expect(toJson(undefined)).toBe("[]");
});
it("returns '[]' for null", () => {
expect(toJson(null)).toBe("[]");
});
it("stringifies booleans", () => {
expect(toJson(true)).toBe("true");
});
it("stringifies numbers", () => {
expect(toJson(42)).toBe("42");
});
});
describe("toJsonNullable", () => {
it("stringifies objects", () => {
expect(toJsonNullable({ a: 1 })).toBe('{"a":1}');
});
it("returns null for undefined", () => {
expect(toJsonNullable(undefined)).toBeNull();
});
it("returns null for null", () => {
expect(toJsonNullable(null)).toBeNull();
});
it("stringifies arrays", () => {
expect(toJsonNullable(["a"])).toBe('["a"]');
});
});
describe("fromJson", () => {
it("parses arrays", () => {
expect(fromJson<string[]>('["a","b"]')).toEqual(["a", "b"]);
});
it("parses objects", () => {
expect(fromJson<{ a: number }>('{"a":1}')).toEqual({ a: 1 });
});
it("returns undefined for null", () => {
expect(fromJson(null)).toBeUndefined();
});
it("returns undefined for undefined", () => {
expect(fromJson(undefined)).toBeUndefined();
});
it("returns undefined for empty string", () => {
expect(fromJson("")).toBeUndefined();
});
it("returns undefined for 'null' string", () => {
expect(fromJson("null")).toBeUndefined();
});
it("returns undefined for invalid JSON", () => {
expect(fromJson("{bad json")).toBeUndefined();
});
it("round-trips: fromJson(toJson([])) returns empty array", () => {
expect(fromJson(toJson([]))).toEqual([]);
});
it("round-trips: fromJson(toJson(['a'])) returns the array", () => {
expect(fromJson(toJson(["a"]))).toEqual(["a"]);
});
it("round-trips: fromJson(toJson({a:1})) returns the object", () => {
expect(fromJson(toJson({ a: 1 }))).toEqual({ a: 1 });
});
it("round-trips: fromJson(toJson(undefined)) returns empty array (array-default)", () => {
// toJson(undefined) = '[]', fromJson('[]') = []
const result = fromJson(toJson(undefined));
expect(result).toEqual([]);
});
});
});
describe("createDatabase factory", () => {
let tmpDir: string;
afterEach(async () => {
await rm(tmpDir, { recursive: true, force: true });
});
it("creates a database instance without auto-init", () => {
tmpDir = makeTmpDir();
const kbDir = join(tmpDir, ".kb");
const db = createDatabase(kbDir);
// DB file exists (created on open) but schema not initialized
expect(existsSync(join(kbDir, "kb.db"))).toBe(true);
// Schema is NOT yet created — querying __meta would fail
expect(() => db.getSchemaVersion()).toThrow();
db.close();
});
it("works after explicit init()", () => {
tmpDir = makeTmpDir();
const kbDir = join(tmpDir, ".kb");
const db = createDatabase(kbDir);
db.init();
expect(db.getSchemaVersion()).toBe(1);
expect(db.getLastModified()).toBeGreaterThan(0);
db.close();
});
it("getPath returns the database file path", () => {
tmpDir = makeTmpDir();
const kbDir = join(tmpDir, ".kb");
const db = createDatabase(kbDir);
expect(db.getPath()).toBe(join(kbDir, "kb.db"));
db.close();
});
it("is idempotent when init() called multiple times", () => {
tmpDir = makeTmpDir();
const kbDir = join(tmpDir, ".kb");
// First call
const db1 = createDatabase(kbDir);
db1.init();
db1.prepare("UPDATE config SET nextId = 99 WHERE id = 1").run();
db1.close();
// Second call — init should not overwrite data
const db2 = createDatabase(kbDir);
db2.init();
const row = db2.prepare("SELECT nextId FROM config WHERE id = 1").get() as any;
expect(row.nextId).toBe(99);
db2.close();
});
});