feat(knowledge): U14 — persistent knowledge index
Adds knowledge_pages (db migration 118→119) + a deterministic, model-free keyword index of task/PR history in packages/dashboard/src, incrementally refreshed on task completion (task:moved→done listener) and queryable via an auth-gated, project-scoped API. Complements the LLM-extracted insights/memory surfaces rather than duplicating them. Follow-ups: no React view yet; PR-history page population attaches via U18.
This commit is contained in:
10
.changeset/u14-knowledge-index.md
Normal file
10
.changeset/u14-knowledge-index.md
Normal file
@@ -0,0 +1,10 @@
|
||||
---
|
||||
"@runfusion/fusion": minor
|
||||
---
|
||||
|
||||
Add a persistent, incrementally-refreshed knowledge index (U14) downstream agents can query.
|
||||
|
||||
- **Schema** — new `knowledge_pages` SQLite table (`packages/core/src/db.ts`) with `SCHEMA_VERSION` bumped 118 → 119 (added in the same change as the migration; the fingerprint auto-covers SCHEMA_SQL tables). Keyword search uses a denormalized lowercased `searchText` column with AND-of-terms `LIKE` matching, deliberately avoiding SQLite FTS5 (not available on every build) and any external embedding API.
|
||||
- **Index module** (`packages/dashboard/src/knowledge-index.ts`) — upsert-by-source-key pages, a model-free keyword query API, and `refreshKnowledgeForTask` that re-indexes a single completed task (one upsert, never a full re-index, so unaffected pages keep their timestamps). This is the delta over the existing `insights`/`memoryView` surfaces, which are LLM-extracted learnings, not a deterministic searchable index of concrete task/PR history.
|
||||
- **Refresh hook** — `KnowledgeIndexRefreshService` listens for `task:moved → done` (mirroring `GitHubSourceIssueCloseService`) and is wired alongside the other completion listeners; fail-soft so it can never disrupt task completion.
|
||||
- **Query API** (`register-knowledge-routes.ts`) — `GET /api/knowledge/query` and `POST /api/knowledge/refresh`, registered as an `ApiRouteRegistrar` so they inherit the dashboard's standard session/auth middleware (401 when unauthenticated) and apply `getScopedStore(req)` (no cross-project reads), exactly like U9.
|
||||
@@ -715,7 +715,7 @@ describe("schema migration", () => {
|
||||
|
||||
const row = db.prepare("SELECT deletedAt FROM tasks WHERE id = 'FN-legacy'").get() as { deletedAt: string | null };
|
||||
expect(row.deletedAt).toBeNull();
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
db.close();
|
||||
});
|
||||
@@ -748,7 +748,7 @@ describe("schema migration", () => {
|
||||
{ id: "WS-001", mode: "prompt", gateMode: "advisory" },
|
||||
{ id: "WS-002", mode: "script", gateMode: "advisory" },
|
||||
]);
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
db.close();
|
||||
});
|
||||
@@ -798,7 +798,7 @@ describe("schema migration", () => {
|
||||
reviewerContextRetryCount: 0,
|
||||
reviewerFallbackRetryCount: 0,
|
||||
});
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
db.close();
|
||||
});
|
||||
@@ -827,7 +827,7 @@ describe("schema migration", () => {
|
||||
|
||||
const columns = db.prepare("PRAGMA table_info(milestones)").all() as Array<{ name: string }>;
|
||||
expect(columns.map((column) => column.name)).toContain("acceptanceCriteria");
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
db.close();
|
||||
});
|
||||
@@ -868,7 +868,7 @@ describe("schema migration", () => {
|
||||
const missionColumns = db.prepare("PRAGMA table_info(missions)").all() as Array<{ name: string }>;
|
||||
expect(missionColumns.map((column) => column.name)).toContain("autoMerge");
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
db.close();
|
||||
});
|
||||
|
||||
@@ -902,7 +902,7 @@ describe("schema migration", () => {
|
||||
{ id: "WS-002", mode: "script", enabled: 1, gateMode: "advisory" },
|
||||
{ id: "WS-003", mode: "prompt", enabled: 0, gateMode: "advisory" },
|
||||
]);
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
db.close();
|
||||
});
|
||||
@@ -939,7 +939,7 @@ describe("schema migration", () => {
|
||||
|
||||
const indexes = db.prepare("PRAGMA index_list(mission_goals)").all() as Array<{ name: string }>;
|
||||
expect(indexes.some((index) => index.name === "idxMissionGoalsGoalId")).toBe(true);
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
db.close();
|
||||
});
|
||||
@@ -1000,7 +1000,7 @@ describe("schema migration", () => {
|
||||
expect(customFieldsColumn).toBeDefined();
|
||||
expect(customFieldsColumn?.dflt_value).toBe("'{}'");
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
db.close();
|
||||
});
|
||||
|
||||
@@ -1038,7 +1038,7 @@ describe("schema migration", () => {
|
||||
const indexes = db.prepare("PRAGMA index_list(workflow_settings)").all() as Array<{ name: string }>;
|
||||
expect(indexes.some((index) => index.name === "idx_workflow_settings_project")).toBe(true);
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
db.close();
|
||||
});
|
||||
|
||||
@@ -1120,7 +1120,7 @@ describe("schema migration", () => {
|
||||
expect(indexNames).toContain("idx_cli_sessions_chatSessionId");
|
||||
expect(indexNames).toContain("idx_cli_sessions_project_state");
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
db.close();
|
||||
});
|
||||
|
||||
@@ -1152,7 +1152,7 @@ describe("schema migration", () => {
|
||||
.all() as Array<{ name: string }>;
|
||||
expect(columns.map((column) => column.name)).toContain("cliExecutorAdapterId");
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
db.close();
|
||||
});
|
||||
|
||||
@@ -1162,7 +1162,7 @@ describe("schema migration", () => {
|
||||
|
||||
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as Array<{ name: string }>;
|
||||
expect(tables.map((row) => row.name)).toContain("cli_sessions");
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
db.close();
|
||||
});
|
||||
|
||||
@@ -1219,20 +1219,20 @@ describe("schema migration", () => {
|
||||
.get() as { migrated_fragment_id: string | null };
|
||||
expect(stepRow.migrated_fragment_id).toBeNull();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
db.close();
|
||||
});
|
||||
|
||||
it("migration 109 is idempotent on re-init", () => {
|
||||
const db = new Database(fusionDir);
|
||||
db.init();
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
db.close();
|
||||
|
||||
// Re-open the same on-disk DB: already at 109, the 109 block must be a no-op.
|
||||
const reopened = new Database(fusionDir);
|
||||
reopened.init();
|
||||
expect(reopened.getSchemaVersion()).toBe(118);
|
||||
expect(reopened.getSchemaVersion()).toBe(119);
|
||||
const workflowColumns = reopened.prepare("PRAGMA table_info(workflows)").all() as Array<{ name: string }>;
|
||||
expect(workflowColumns.filter((c) => c.name === "kind")).toHaveLength(1);
|
||||
const stepColumns = reopened.prepare("PRAGMA table_info(workflow_steps)").all() as Array<{ name: string }>;
|
||||
|
||||
@@ -334,7 +334,7 @@ describe("Database", () => {
|
||||
});
|
||||
|
||||
it("seeds schema version", () => {
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
});
|
||||
|
||||
it("includes tokenUsageCacheWriteTokens on freshly initialized tasks table", () => {
|
||||
@@ -393,7 +393,7 @@ describe("Database", () => {
|
||||
|
||||
it("is idempotent - calling init() twice does not fail", () => {
|
||||
expect(() => db.init()).not.toThrow();
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
});
|
||||
it("does not overwrite existing config on re-init", () => {
|
||||
// Update the config
|
||||
@@ -1463,7 +1463,7 @@ describe("schema migrations", () => {
|
||||
db.init();
|
||||
|
||||
// Verify version bumped to 29 (includes v1→v2 through v26→v29)
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
// Verify new columns exist and existing data is intact
|
||||
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
@@ -1488,15 +1488,15 @@ describe("schema migrations", () => {
|
||||
const db = new Database(fusionDir);
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
// Re-init should not fail
|
||||
db.init();
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
// Re-init should not fail
|
||||
db.init();
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
db.close();
|
||||
});
|
||||
@@ -1531,7 +1531,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
expect(cols.map((col) => col.name)).toContain("priority");
|
||||
@@ -1572,7 +1572,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
const colNames = cols.map((col) => col.name);
|
||||
@@ -1644,7 +1644,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
const colNames = cols.map((col) => col.name);
|
||||
@@ -1884,7 +1884,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
const cols = db.prepare("PRAGMA table_info(chat_messages)").all() as Array<{ name: string }>;
|
||||
expect(cols.map((col) => col.name)).toContain("attachments");
|
||||
@@ -1958,7 +1958,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>;
|
||||
expect(tables).toEqual([{ name: "agentRatings" }]);
|
||||
@@ -1982,7 +1982,7 @@ describe("schema migrations", () => {
|
||||
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>;
|
||||
expect(tables).toEqual([{ name: "mission_events" }]);
|
||||
@@ -2086,7 +2086,7 @@ describe("schema migrations", () => {
|
||||
db.init();
|
||||
|
||||
// Verify version bumped to 29
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
// Verify new columns exist and existing data is intact
|
||||
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
@@ -2305,7 +2305,7 @@ describe("schema migrations", () => {
|
||||
|
||||
localDb.init();
|
||||
|
||||
expect(localDb.getSchemaVersion()).toBe(118);
|
||||
expect(localDb.getSchemaVersion()).toBe(119);
|
||||
const columns = localDb.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
expect(columns.map((column) => column.name)).toContain("tokenUsageCacheWriteTokens");
|
||||
|
||||
@@ -2616,7 +2616,7 @@ describe("createDatabase factory", () => {
|
||||
const db = createDatabase(fusionDir);
|
||||
db.init();
|
||||
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
expect(db.getLastModified()).toBeGreaterThan(0);
|
||||
|
||||
db.close();
|
||||
@@ -2770,7 +2770,7 @@ describe("migration v77 task token budget columns", () => {
|
||||
|
||||
migrated = new Database(fusion);
|
||||
migrated.init();
|
||||
expect(migrated.getSchemaVersion()).toBe(118);
|
||||
expect(migrated.getSchemaVersion()).toBe(119);
|
||||
const rows = migrated.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
|
||||
const names = new Set(rows.map((row) => row.name));
|
||||
expect(names.has("tokenBudgetSoftAlertedAt")).toBe(true);
|
||||
@@ -2801,7 +2801,7 @@ describe("migration v106 adds tasks.transitionPending (FN-1417)", () => {
|
||||
const fresh = new Database(fusion);
|
||||
try {
|
||||
fresh.init();
|
||||
expect(fresh.getSchemaVersion()).toBe(118);
|
||||
expect(fresh.getSchemaVersion()).toBe(119);
|
||||
const names = new Set(
|
||||
(fresh.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>).map((r) => r.name),
|
||||
);
|
||||
@@ -2829,7 +2829,7 @@ describe("migration v106 adds tasks.transitionPending (FN-1417)", () => {
|
||||
|
||||
migrated = new Database(fusion);
|
||||
migrated.init();
|
||||
expect(migrated.getSchemaVersion()).toBe(118);
|
||||
expect(migrated.getSchemaVersion()).toBe(119);
|
||||
const names = new Set(
|
||||
(migrated.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>).map((r) => r.name),
|
||||
);
|
||||
@@ -2855,7 +2855,7 @@ describe("migration v107 adds workflow_run_branches + index (FN-1417)", () => {
|
||||
const fresh = new Database(fusion);
|
||||
try {
|
||||
fresh.init();
|
||||
expect(fresh.getSchemaVersion()).toBe(118);
|
||||
expect(fresh.getSchemaVersion()).toBe(119);
|
||||
const table = fresh
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'workflow_run_branches'")
|
||||
.get() as { name: string } | undefined;
|
||||
@@ -2889,7 +2889,7 @@ describe("migration v107 adds workflow_run_branches + index (FN-1417)", () => {
|
||||
|
||||
migrated = new Database(fusion);
|
||||
migrated.init();
|
||||
expect(migrated.getSchemaVersion()).toBe(118);
|
||||
expect(migrated.getSchemaVersion()).toBe(119);
|
||||
const table = migrated
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'workflow_run_branches'")
|
||||
.get() as { name: string } | undefined;
|
||||
@@ -2930,7 +2930,7 @@ describe("migration v67 drops orphan project auth tables", () => {
|
||||
|
||||
migrated = new Database(fusion);
|
||||
migrated.init();
|
||||
expect(migrated.getSchemaVersion()).toBe(118);
|
||||
expect(migrated.getSchemaVersion()).toBe(119);
|
||||
const tables = migrated
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'project_auth_%'")
|
||||
.all() as Array<{ name: string }>;
|
||||
@@ -2957,7 +2957,7 @@ describe("migration v67 drops orphan project auth tables", () => {
|
||||
|
||||
try {
|
||||
fresh.init();
|
||||
expect(fresh.getSchemaVersion()).toBe(118);
|
||||
expect(fresh.getSchemaVersion()).toBe(119);
|
||||
const tables = fresh
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'project_auth_%'")
|
||||
.all() as Array<{ name: string }>;
|
||||
|
||||
@@ -91,6 +91,6 @@ describe("goals schema", () => {
|
||||
});
|
||||
|
||||
it("reports schema version 101", () => {
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1000,7 +1000,7 @@ describe("Migration: pre-33 DB upgrade", () => {
|
||||
// Step 1: Create a fresh database at v33 (runs all migrations up to 33)
|
||||
const db1 = createDatabase(legacyDir);
|
||||
db1.init();
|
||||
expect(db1.getSchemaVersion()).toBe(118);
|
||||
expect(db1.getSchemaVersion()).toBe(119);
|
||||
db1.close();
|
||||
|
||||
// Step 2: Manually downgrade to version 32 and drop insight tables
|
||||
@@ -1035,7 +1035,7 @@ describe("Migration: pre-33 DB upgrade", () => {
|
||||
expect(tableNamesBefore).not.toContain("project_insight_runs");
|
||||
// Now run init — this triggers the v32→v33 migration
|
||||
db3.init();
|
||||
expect(db3.getSchemaVersion()).toBe(118);
|
||||
expect(db3.getSchemaVersion()).toBe(119);
|
||||
|
||||
// Step 4: Verify insight tables exist after migration
|
||||
const tablesAfter = db3.prepare(
|
||||
@@ -1066,12 +1066,12 @@ describe("Migration: pre-33 DB upgrade", () => {
|
||||
try {
|
||||
const db1 = createDatabase(testDir);
|
||||
db1.init();
|
||||
expect(db1.getSchemaVersion()).toBe(118);
|
||||
expect(db1.getSchemaVersion()).toBe(119);
|
||||
db1.close();
|
||||
|
||||
const db2 = createDatabase(testDir);
|
||||
expect(() => db2.init()).not.toThrow();
|
||||
expect(db2.getSchemaVersion()).toBe(118);
|
||||
expect(db2.getSchemaVersion()).toBe(119);
|
||||
db2.close();
|
||||
} finally {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
@@ -1085,7 +1085,7 @@ describe("Migration: pre-33 DB upgrade", () => {
|
||||
// Step 1: Create a fresh DB and run migrations
|
||||
const db1 = createDatabase(compatDir);
|
||||
db1.init();
|
||||
expect(db1.getSchemaVersion()).toBe(118);
|
||||
expect(db1.getSchemaVersion()).toBe(119);
|
||||
|
||||
// Step 2: Strip lifecycle and cancelledAt columns by recreating the
|
||||
// table without them. This simulates a DB that was created before the
|
||||
|
||||
@@ -38,7 +38,7 @@ describe("TaskStore merge request record + completion handoff marker", () => {
|
||||
.all() as Array<{ name: string }>;
|
||||
|
||||
expect(tableRows).toEqual([{ name: "completion_handoff_markers" }, { name: "merge_requests" }]);
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
});
|
||||
|
||||
it("upserts merge request records", async () => {
|
||||
|
||||
@@ -3746,7 +3746,7 @@ describe("MissionStore", () => {
|
||||
|
||||
describe("Loop State & Validator Run Schema (v31)", () => {
|
||||
it("schema version is 101 after migration", () => {
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
});
|
||||
|
||||
it("mission_features table has loop state columns", () => {
|
||||
|
||||
@@ -583,8 +583,8 @@ describe("Run Audit", () => {
|
||||
expect(indexNames).toContain("idxRunAuditEventsTimestamp");
|
||||
});
|
||||
|
||||
it("schema version is bumped to 118", () => {
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
it("schema version is bumped to 119", () => {
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -60,7 +60,7 @@ describe("TaskStore merge queue", () => {
|
||||
expect.arrayContaining(["idx_mergeQueue_lease_ready", "idx_mergeQueue_leaseExpiresAt"]),
|
||||
);
|
||||
|
||||
expect(store.getDatabase().getSchemaVersion()).toBe(118);
|
||||
expect(store.getDatabase().getSchemaVersion()).toBe(119);
|
||||
});
|
||||
|
||||
it("migrates a legacy v88 database and preserves task rows", async () => {
|
||||
|
||||
@@ -51,7 +51,7 @@ describe("TaskStore task documents", () => {
|
||||
|
||||
expect(tableNames.has("task_documents")).toBe(true);
|
||||
expect(tableNames.has("task_document_revisions")).toBe(true);
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
|
||||
const index = db
|
||||
.prepare(
|
||||
|
||||
@@ -175,15 +175,18 @@ describe("usage_events", () => {
|
||||
expect(rows[0].agentId).toBe("A-chat");
|
||||
});
|
||||
|
||||
// Migration: seed a DB at the PREVIOUS schema version, run migrate, assert
|
||||
// the table exists and SCHEMA_VERSION equals the highest migration target.
|
||||
// Fresh-DB tests cannot catch the early-return bug this guards.
|
||||
// Migration: seed a DB at the version JUST BEFORE usage_events was introduced
|
||||
// (117 — usage_events is the v118 migration), run migrate, assert the table
|
||||
// exists and SCHEMA_VERSION reaches the highest migration target. Pinned to
|
||||
// 117 (not SCHEMA_VERSION-1) so it keeps exercising usage_events' own
|
||||
// migration as later migrations are added. Fresh-DB tests cannot catch the
|
||||
// migrate-loop early-return bug this guards.
|
||||
it("creates usage_events when migrating from the previous schema version", () => {
|
||||
db.exec("DROP INDEX IF EXISTS idxUsageEventsTs");
|
||||
db.exec("DROP INDEX IF EXISTS idxUsageEventsTaskId");
|
||||
db.exec("DROP INDEX IF EXISTS idxUsageEventsAgentId");
|
||||
db.exec("DROP TABLE IF EXISTS usage_events");
|
||||
db.prepare("UPDATE __meta SET value = ? WHERE key = 'schemaVersion'").run(String(SCHEMA_VERSION - 1));
|
||||
db.prepare("UPDATE __meta SET value = ? WHERE key = 'schemaVersion'").run("117");
|
||||
|
||||
(db as unknown as { migrate: () => void }).migrate();
|
||||
|
||||
|
||||
@@ -162,7 +162,7 @@ export function isFts5CorruptionError(error: unknown): boolean {
|
||||
|
||||
// ── Schema Definition ────────────────────────────────────────────────
|
||||
|
||||
const SCHEMA_VERSION = 118;
|
||||
const SCHEMA_VERSION = 119;
|
||||
|
||||
const TASKS_FTS_AUTOMERGE = 8;
|
||||
const TASKS_FTS_CRISISMERGE = 16;
|
||||
@@ -1230,6 +1230,31 @@ CREATE TABLE IF NOT EXISTS usage_events (
|
||||
CREATE INDEX IF NOT EXISTS idxUsageEventsTs ON usage_events(ts);
|
||||
CREATE INDEX IF NOT EXISTS idxUsageEventsTaskId ON usage_events(taskId);
|
||||
CREATE INDEX IF NOT EXISTS idxUsageEventsAgentId ON usage_events(agentId);
|
||||
|
||||
-- Persistent, incrementally-refreshed knowledge index (U14). One row per
|
||||
-- knowledge page (currently one page per completed task; PR-history pages
|
||||
-- share the same shape). Downstream agents query it through the dashboard's
|
||||
-- scoped knowledge-index endpoint. searchText is a denormalized lowercased
|
||||
-- concatenation of the page's title/summary/content + tags used for keyword
|
||||
-- LIKE matching, so the index works without requiring SQLite FTS5 (which is
|
||||
-- not available on every build -- see probeFts5 above). Refresh is per-source
|
||||
-- (upsert by sourceKey), never a full re-index, so unaffected pages keep their
|
||||
-- timestamps.
|
||||
CREATE TABLE IF NOT EXISTS knowledge_pages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
sourceKind TEXT NOT NULL,
|
||||
sourceId TEXT NOT NULL,
|
||||
sourceKey TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
summary TEXT,
|
||||
content TEXT NOT NULL,
|
||||
tags TEXT,
|
||||
searchText TEXT NOT NULL,
|
||||
createdAt TEXT NOT NULL,
|
||||
updatedAt TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idxKnowledgePagesSourceKind ON knowledge_pages(sourceKind);
|
||||
CREATE INDEX IF NOT EXISTS idxKnowledgePagesUpdatedAt ON knowledge_pages(updatedAt);
|
||||
`;
|
||||
|
||||
const TABLE_LEVEL_CONSTRAINT_PREFIXES = new Set([
|
||||
@@ -4773,6 +4798,36 @@ export class Database {
|
||||
});
|
||||
}
|
||||
|
||||
// Migration 119: Persistent knowledge index (U14). One queryable page per
|
||||
// completed task / PR-history entry, refreshed incrementally (upsert by
|
||||
// sourceKey) on task completion. Mirrors the SCHEMA_SQL definition above so
|
||||
// a fresh-from-SCHEMA_SQL DB and a migrated DB converge on the same table.
|
||||
if (version < 119) {
|
||||
this.applyMigration(119, () => {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS knowledge_pages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
sourceKind TEXT NOT NULL,
|
||||
sourceId TEXT NOT NULL,
|
||||
sourceKey TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL,
|
||||
summary TEXT,
|
||||
content TEXT NOT NULL,
|
||||
tags TEXT,
|
||||
searchText TEXT NOT NULL,
|
||||
createdAt TEXT NOT NULL,
|
||||
updatedAt TEXT NOT NULL
|
||||
)
|
||||
`);
|
||||
this.db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idxKnowledgePagesSourceKind ON knowledge_pages(sourceKind)
|
||||
`);
|
||||
this.db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idxKnowledgePagesUpdatedAt ON knowledge_pages(updatedAt)
|
||||
`);
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
240
packages/dashboard/src/__tests__/knowledge-index.test.ts
Normal file
240
packages/dashboard/src/__tests__/knowledge-index.test.ts
Normal file
@@ -0,0 +1,240 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { EventEmitter } from "node:events";
|
||||
|
||||
import { Database, SCHEMA_VERSION } from "@fusion/core";
|
||||
import type { TaskStore } from "@fusion/core";
|
||||
import {
|
||||
upsertKnowledgePage,
|
||||
queryKnowledgePages,
|
||||
getKnowledgePage,
|
||||
countKnowledgePages,
|
||||
refreshKnowledgeForTask,
|
||||
renderTaskPage,
|
||||
tokenizeQuery,
|
||||
buildSearchText,
|
||||
} from "../knowledge-index.js";
|
||||
|
||||
function makeDb(): { db: Database; tmpDir: string } {
|
||||
const tmpDir = mkdtempSync(join(tmpdir(), "kb-knowledge-index-"));
|
||||
const db = new Database(join(tmpDir, ".fusion"));
|
||||
db.init();
|
||||
return { db, tmpDir };
|
||||
}
|
||||
|
||||
describe("knowledge-index store", () => {
|
||||
let db: Database;
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
({ db, tmpDir } = makeDb());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("creates knowledge_pages with the expected columns on fresh init", () => {
|
||||
const cols = (db.prepare("PRAGMA table_info(knowledge_pages)").all() as Array<{ name: string }>).map(
|
||||
(c) => c.name,
|
||||
);
|
||||
expect(cols).toEqual([
|
||||
"id",
|
||||
"sourceKind",
|
||||
"sourceId",
|
||||
"sourceKey",
|
||||
"title",
|
||||
"summary",
|
||||
"content",
|
||||
"tags",
|
||||
"searchText",
|
||||
"createdAt",
|
||||
"updatedAt",
|
||||
]);
|
||||
});
|
||||
|
||||
it("upserts a page and returns it via a keyword query", () => {
|
||||
const { created } = upsertKnowledgePage(db, {
|
||||
sourceKind: "task",
|
||||
sourceId: "T-1",
|
||||
title: "Add caching layer",
|
||||
content: "Introduced an LRU cache in fetcher.ts",
|
||||
tags: ["fetcher.ts"],
|
||||
});
|
||||
expect(created).toBe(true);
|
||||
const hits = queryKnowledgePages(db, { query: "cache" });
|
||||
expect(hits).toHaveLength(1);
|
||||
expect(hits[0].sourceId).toBe("T-1");
|
||||
expect(hits[0].tags).toEqual(["fetcher.ts"]);
|
||||
});
|
||||
|
||||
it("AND-matches all query terms", () => {
|
||||
upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-1", title: "alpha gadget", content: "only alpha here" });
|
||||
upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-2", title: "alpha thing", content: "beta widget" });
|
||||
expect(queryKnowledgePages(db, { query: "alpha widget" }).map((p) => p.sourceId)).toEqual(["T-2"]);
|
||||
});
|
||||
|
||||
it("a blank/termless query returns nothing (never the whole index)", () => {
|
||||
upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-1", title: "x", content: "y" });
|
||||
expect(queryKnowledgePages(db, { query: "" })).toHaveLength(0);
|
||||
expect(queryKnowledgePages(db, { query: " " })).toHaveLength(0);
|
||||
expect(countKnowledgePages(db)).toBe(1);
|
||||
});
|
||||
|
||||
it("escapes LIKE wildcards so user input can't widen the match", () => {
|
||||
upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-1", title: "literal", content: "100% done" });
|
||||
// A bare "%" must not match every row; it has no alphanumeric token at all.
|
||||
expect(queryKnowledgePages(db, { query: "%" })).toHaveLength(0);
|
||||
// The literal token does match.
|
||||
expect(queryKnowledgePages(db, { query: "100" })).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("incremental refresh updates only the affected page; others keep their timestamps", () => {
|
||||
const { page: a } = upsertKnowledgePage(db, {
|
||||
sourceKind: "task",
|
||||
sourceId: "T-A",
|
||||
title: "A",
|
||||
content: "a",
|
||||
now: "2026-01-01T00:00:00.000Z",
|
||||
});
|
||||
const { page: b } = upsertKnowledgePage(db, {
|
||||
sourceKind: "task",
|
||||
sourceId: "T-B",
|
||||
title: "B",
|
||||
content: "b",
|
||||
now: "2026-01-01T00:00:00.000Z",
|
||||
});
|
||||
expect(a.createdAt).toBe("2026-01-01T00:00:00.000Z");
|
||||
|
||||
// Re-index only T-A at a later time.
|
||||
const { created, page: aUpdated } = upsertKnowledgePage(db, {
|
||||
sourceKind: "task",
|
||||
sourceId: "T-A",
|
||||
title: "A v2",
|
||||
content: "a v2",
|
||||
now: "2026-02-02T00:00:00.000Z",
|
||||
});
|
||||
expect(created).toBe(false);
|
||||
expect(aUpdated.createdAt).toBe("2026-01-01T00:00:00.000Z"); // createdAt preserved
|
||||
expect(aUpdated.updatedAt).toBe("2026-02-02T00:00:00.000Z"); // updatedAt advanced
|
||||
|
||||
// T-B is untouched: same updatedAt as when it was created.
|
||||
const bAfter = getKnowledgePage(db, "task", "T-B");
|
||||
expect(bAfter?.updatedAt).toBe(b.updatedAt);
|
||||
expect(bAfter?.updatedAt).toBe("2026-01-01T00:00:00.000Z");
|
||||
// Still exactly two pages — no duplicate created on re-index.
|
||||
expect(countKnowledgePages(db)).toBe(2);
|
||||
});
|
||||
|
||||
// Seed a DB at the PREVIOUS schema version (118), run migrate, assert the
|
||||
// table exists and SCHEMA_VERSION lands at the highest migration target (119).
|
||||
// Fresh-DB tests cannot catch the migrate-loop early-return bug this guards.
|
||||
it("creates knowledge_pages when migrating from the previous schema version", () => {
|
||||
db.exec("DROP INDEX IF EXISTS idxKnowledgePagesSourceKind");
|
||||
db.exec("DROP INDEX IF EXISTS idxKnowledgePagesUpdatedAt");
|
||||
db.exec("DROP TABLE IF EXISTS knowledge_pages");
|
||||
db.prepare("UPDATE __meta SET value = ? WHERE key = 'schemaVersion'").run(String(SCHEMA_VERSION - 1));
|
||||
|
||||
(db as unknown as { migrate: () => void }).migrate();
|
||||
|
||||
const table = db
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='knowledge_pages'")
|
||||
.get() as { name: string } | undefined;
|
||||
expect(table?.name).toBe("knowledge_pages");
|
||||
expect(db.getSchemaVersion()).toBe(SCHEMA_VERSION);
|
||||
|
||||
// The migrated table is writable and queryable.
|
||||
upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-mig", title: "migrated", content: "ok" });
|
||||
expect(queryKnowledgePages(db, { query: "migrated" })).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("SCHEMA_VERSION matches the highest applied migration on a fresh DB", () => {
|
||||
expect(db.getSchemaVersion()).toBe(SCHEMA_VERSION);
|
||||
});
|
||||
});
|
||||
|
||||
describe("knowledge-index pure helpers", () => {
|
||||
it("tokenizeQuery splits on non-word chars and lowercases", () => {
|
||||
expect(tokenizeQuery("Add OAuth, login-flow!")).toEqual(["add", "oauth", "login", "flow"]);
|
||||
expect(tokenizeQuery(" ")).toEqual([]);
|
||||
});
|
||||
|
||||
it("buildSearchText concatenates and lowercases all fields", () => {
|
||||
const text = buildSearchText({ title: "Title", summary: "Sum", content: "Body", tags: ["Tag"] });
|
||||
expect(text).toBe("title sum body tag");
|
||||
});
|
||||
|
||||
it("renderTaskPage builds a deterministic page from task facts", () => {
|
||||
const page = renderTaskPage({
|
||||
id: "FN-7",
|
||||
title: "Fix bug",
|
||||
description: "Null deref in parser",
|
||||
modifiedFiles: ["src/parser.ts"],
|
||||
commitSubjects: ["fix: guard null"],
|
||||
prUrl: "https://example.com/pr/7",
|
||||
});
|
||||
expect(page.sourceKind).toBe("task");
|
||||
expect(page.sourceId).toBe("FN-7");
|
||||
expect(page.title).toBe("Fix bug");
|
||||
expect(page.content).toContain("Null deref in parser");
|
||||
expect(page.content).toContain("src/parser.ts");
|
||||
expect(page.content).toContain("fix: guard null");
|
||||
expect(page.content).toContain("https://example.com/pr/7");
|
||||
expect(page.tags).toEqual(["parser.ts"]);
|
||||
});
|
||||
|
||||
it("renderTaskPage falls back to a generated title when none is set", () => {
|
||||
const page = renderTaskPage({ id: "FN-8", description: "", modifiedFiles: [] });
|
||||
expect(page.title).toBe("Task FN-8");
|
||||
});
|
||||
});
|
||||
|
||||
describe("refreshKnowledgeForTask hook", () => {
|
||||
let db: Database;
|
||||
let tmpDir: string;
|
||||
|
||||
function storeFor(database: Database, tasks: Record<string, unknown>): TaskStore {
|
||||
const store = new EventEmitter() as unknown as TaskStore & {
|
||||
getDatabase(): Database;
|
||||
getTask(id: string): Promise<unknown>;
|
||||
};
|
||||
store.getDatabase = () => database;
|
||||
store.getTask = async (id: string) => tasks[id] ?? null;
|
||||
return store;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
({ db, tmpDir } = makeDb());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("indexes a completed task so it becomes queryable", async () => {
|
||||
const store = storeFor(db, {
|
||||
"FN-1": {
|
||||
id: "FN-1",
|
||||
title: "Implement retry",
|
||||
description: "Exponential backoff in client.ts",
|
||||
modifiedFiles: ["client.ts"],
|
||||
column: "done",
|
||||
},
|
||||
});
|
||||
const page = await refreshKnowledgeForTask(store, "FN-1");
|
||||
expect(page?.sourceId).toBe("FN-1");
|
||||
expect(queryKnowledgePages(db, { query: "backoff" })).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("is fail-soft: returns null for a missing task without throwing", async () => {
|
||||
const store = storeFor(db, {});
|
||||
await expect(refreshKnowledgeForTask(store, "nope")).resolves.toBeNull();
|
||||
expect(countKnowledgePages(db)).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
// @vitest-environment node
|
||||
|
||||
/**
|
||||
* Auth integration for the knowledge-index endpoints (U14): every endpoint must
|
||||
* be rejected with 401 when unauthenticated and accepted with a valid bearer
|
||||
* token. Mirrors `register-command-center-routes.auth.test.ts` — the registrar
|
||||
* adds no auth of its own; it inherits the server-level middleware, which is
|
||||
* exactly what this asserts.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { EventEmitter } from "node:events";
|
||||
import type { Task, TaskStore } from "@fusion/core";
|
||||
import { request } from "../test-request.js";
|
||||
import { createServer } from "../server.js";
|
||||
|
||||
vi.mock("@fusion/core", async (importOriginal) => {
|
||||
const { createCoreMock } = await import("../test/mockCoreEngine.js");
|
||||
return createCoreMock(() => importOriginal<typeof import("@fusion/core")>(), {});
|
||||
});
|
||||
|
||||
class MockStore extends EventEmitter {
|
||||
getRootDir(): string {
|
||||
return "/tmp/fn-knowledge-auth-test";
|
||||
}
|
||||
|
||||
getFusionDir(): string {
|
||||
return "/tmp/fn-knowledge-auth-test/.fusion";
|
||||
}
|
||||
|
||||
getDatabase() {
|
||||
return {
|
||||
exec: vi.fn(),
|
||||
prepare: vi.fn().mockReturnValue({
|
||||
run: vi.fn().mockReturnValue({ changes: 0 }),
|
||||
get: vi.fn().mockReturnValue({ count: 0 }),
|
||||
all: vi.fn().mockReturnValue([]),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
getDatabaseHealth() {
|
||||
return {
|
||||
healthy: true,
|
||||
corruptionDetected: false,
|
||||
corruptionErrors: [],
|
||||
isRunning: false,
|
||||
lastCheckedAt: null,
|
||||
};
|
||||
}
|
||||
|
||||
async listTasks(): Promise<Task[]> {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const TOKEN = "fn_knowledge_test1234567890abc";
|
||||
|
||||
describe("Knowledge routes — auth", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("rejects an unauthenticated query with 401", async () => {
|
||||
const app = createServer(new MockStore() as unknown as TaskStore, {
|
||||
daemon: { token: TOKEN },
|
||||
});
|
||||
const res = await request(app, "GET", "/api/knowledge/query?q=anything");
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it("accepts the query with a valid bearer token", async () => {
|
||||
const app = createServer(new MockStore() as unknown as TaskStore, {
|
||||
daemon: { token: TOKEN },
|
||||
});
|
||||
const res = await request(app, "GET", "/api/knowledge/query?q=anything", undefined, {
|
||||
Authorization: `Bearer ${TOKEN}`,
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,185 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import express, { type NextFunction, type Request, type Response } from "express";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { EventEmitter } from "node:events";
|
||||
|
||||
import { Database } from "@fusion/core";
|
||||
import type { TaskStore } from "@fusion/core";
|
||||
import { request } from "../test-request.js";
|
||||
import { ApiError } from "../api-error.js";
|
||||
import { registerKnowledgeRoutes } from "../routes/register-knowledge-routes.js";
|
||||
import { upsertKnowledgePage } from "../knowledge-index.js";
|
||||
import type { ApiRoutesContext } from "../routes/types.js";
|
||||
|
||||
interface QueryResponse {
|
||||
query: string;
|
||||
pages: Array<{ sourceId: string }>;
|
||||
total: number;
|
||||
}
|
||||
interface RefreshResponse {
|
||||
page: { sourceId: string };
|
||||
}
|
||||
|
||||
/** POST JSON helper over the bare `request` (which only accepts string bodies). */
|
||||
function postJson(
|
||||
app: ReturnType<typeof buildApp>,
|
||||
path: string,
|
||||
body: unknown,
|
||||
): ReturnType<typeof request> {
|
||||
return request(app, "POST", path, JSON.stringify(body), {
|
||||
"content-type": "application/json",
|
||||
});
|
||||
}
|
||||
|
||||
/** A minimal TaskStore exposing getDatabase()/getTask(), which is all routes use. */
|
||||
function storeFor(db: Database, tasks: Record<string, unknown> = {}): TaskStore {
|
||||
const store = new EventEmitter() as unknown as TaskStore & {
|
||||
getDatabase(): Database;
|
||||
getTask(id: string): Promise<unknown>;
|
||||
};
|
||||
store.getDatabase = () => db;
|
||||
store.getTask = async (id: string) => tasks[id] ?? null;
|
||||
return store;
|
||||
}
|
||||
|
||||
function buildApp(stores: Record<string, TaskStore>, fallback: TaskStore) {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
const router = express.Router();
|
||||
const ctx = {
|
||||
router,
|
||||
getScopedStore: async (req: Request): Promise<TaskStore> => {
|
||||
const projectId = typeof req.query.projectId === "string" ? req.query.projectId : undefined;
|
||||
return projectId && stores[projectId] ? stores[projectId] : fallback;
|
||||
},
|
||||
rethrowAsApiError: (error: unknown, fallbackMessage?: string): never => {
|
||||
if (error instanceof ApiError) throw error;
|
||||
throw new ApiError(500, fallbackMessage ?? "Internal error");
|
||||
},
|
||||
} as unknown as ApiRoutesContext;
|
||||
registerKnowledgeRoutes(ctx);
|
||||
app.use("/api", router);
|
||||
app.use((err: unknown, _req: Request, res: Response, _next: NextFunction) => {
|
||||
if (err instanceof ApiError) {
|
||||
res.status(err.statusCode).json({ error: err.message });
|
||||
return;
|
||||
}
|
||||
res.status(500).json({ error: "Internal error" });
|
||||
});
|
||||
return app;
|
||||
}
|
||||
|
||||
describe("register-knowledge-routes", () => {
|
||||
let tmpDir: string;
|
||||
let dbA: Database;
|
||||
let dbB: Database;
|
||||
let app: ReturnType<typeof buildApp>;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "kb-knowledge-routes-"));
|
||||
dbA = new Database(join(tmpDir, "a", ".fusion"));
|
||||
dbA.init();
|
||||
dbB = new Database(join(tmpDir, "b", ".fusion"));
|
||||
dbB.init();
|
||||
|
||||
upsertKnowledgePage(dbA, {
|
||||
sourceKind: "task",
|
||||
sourceId: "FN-A1",
|
||||
title: "Add OAuth login flow",
|
||||
content: "Implemented oauth login with token refresh in auth.ts",
|
||||
tags: ["auth.ts"],
|
||||
});
|
||||
upsertKnowledgePage(dbB, {
|
||||
sourceKind: "task",
|
||||
sourceId: "FN-B1",
|
||||
title: "Secret project-B widget",
|
||||
content: "Project B only — confidential widget rendering",
|
||||
tags: ["widget.ts"],
|
||||
});
|
||||
|
||||
const storeA = storeFor(dbA, {
|
||||
"FN-A2": {
|
||||
id: "FN-A2",
|
||||
title: "Refactor payment module",
|
||||
description: "Cleaned up the stripe payment handler",
|
||||
modifiedFiles: ["payment.ts"],
|
||||
column: "done",
|
||||
},
|
||||
});
|
||||
const storeB = storeFor(dbB);
|
||||
app = buildApp({ "proj-a": storeA, "proj-b": storeB }, storeA);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
dbA.close();
|
||||
dbB.close();
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("returns relevant pages for a keyword query (fixture)", async () => {
|
||||
const res = await request(app, "GET", "/api/knowledge/query?q=oauth&projectId=proj-a");
|
||||
expect(res.status).toBe(200);
|
||||
const body = res.body as QueryResponse;
|
||||
expect(body.pages).toHaveLength(1);
|
||||
expect(body.pages[0].sourceId).toBe("FN-A1");
|
||||
expect(body.total).toBe(1);
|
||||
});
|
||||
|
||||
it("returns empty for a non-matching keyword", async () => {
|
||||
const res = await request(app, "GET", "/api/knowledge/query?q=kubernetes&projectId=proj-a");
|
||||
expect(res.status).toBe(200);
|
||||
expect((res.body as QueryResponse).pages).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("returns empty for a blank query rather than the whole index", async () => {
|
||||
const res = await request(app, "GET", "/api/knowledge/query?q=&projectId=proj-a");
|
||||
expect(res.status).toBe(200);
|
||||
const body = res.body as QueryResponse;
|
||||
expect(body.pages).toHaveLength(0);
|
||||
expect(body.total).toBe(1);
|
||||
});
|
||||
|
||||
it("project scoping — project-A query cannot read project-B pages", async () => {
|
||||
// The project-B-only term must never surface for project A.
|
||||
const leak = await request(app, "GET", "/api/knowledge/query?q=widget&projectId=proj-a");
|
||||
expect(leak.status).toBe(200);
|
||||
expect((leak.body as QueryResponse).pages).toHaveLength(0);
|
||||
|
||||
// ...but is visible to project B.
|
||||
const ok = await request(app, "GET", "/api/knowledge/query?q=widget&projectId=proj-b");
|
||||
expect(ok.status).toBe(200);
|
||||
const okBody = ok.body as QueryResponse;
|
||||
expect(okBody.pages).toHaveLength(1);
|
||||
expect(okBody.pages[0].sourceId).toBe("FN-B1");
|
||||
});
|
||||
|
||||
it("POST /refresh incrementally indexes a completed task, then it is queryable", async () => {
|
||||
const refresh = await postJson(app, "/api/knowledge/refresh?projectId=proj-a", {
|
||||
taskId: "FN-A2",
|
||||
});
|
||||
expect(refresh.status).toBe(200);
|
||||
expect((refresh.body as RefreshResponse).page.sourceId).toBe("FN-A2");
|
||||
|
||||
const q = await request(app, "GET", "/api/knowledge/query?q=stripe&projectId=proj-a");
|
||||
expect(q.status).toBe(200);
|
||||
const qBody = q.body as QueryResponse;
|
||||
expect(qBody.pages).toHaveLength(1);
|
||||
expect(qBody.pages[0].sourceId).toBe("FN-A2");
|
||||
});
|
||||
|
||||
it("POST /refresh returns 404 for an unknown task", async () => {
|
||||
const res = await postJson(app, "/api/knowledge/refresh?projectId=proj-a", {
|
||||
taskId: "does-not-exist",
|
||||
});
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
|
||||
it("POST /refresh requires a taskId", async () => {
|
||||
const res = await postJson(app, "/api/knowledge/refresh?projectId=proj-a", {});
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
@@ -37,6 +37,23 @@ export { rateLimit, RATE_LIMITS, type RateLimitOptions } from "./rate-limit.js";
|
||||
export { GitHubPollingService, type GitHubPollingServiceOptions, type TaskWatchInput, type WatchedBadgeType } from "./github-poll.js";
|
||||
export { GitHubIssueCommentService, DEFAULT_COMMENT_TEMPLATE } from "./github-issue-comment.js";
|
||||
export { GitHubSourceIssueCloseService } from "./github-source-issue-close.js";
|
||||
export {
|
||||
upsertKnowledgePage,
|
||||
queryKnowledgePages,
|
||||
getKnowledgePage,
|
||||
countKnowledgePages,
|
||||
refreshKnowledgeForTask,
|
||||
renderTaskPage,
|
||||
buildSearchText,
|
||||
tokenizeQuery,
|
||||
KNOWLEDGE_QUERY_DEFAULT_LIMIT,
|
||||
KNOWLEDGE_QUERY_MAX_LIMIT,
|
||||
type KnowledgePage,
|
||||
type KnowledgePageInput,
|
||||
type KnowledgeSourceKind,
|
||||
type KnowledgeQueryOptions,
|
||||
} from "./knowledge-index.js";
|
||||
export { KnowledgeIndexRefreshService } from "./knowledge-index-refresh.js";
|
||||
export { GitHubTrackingCommentService, formatTrackingComment } from "./github-tracking-comments.js";
|
||||
export { GitHubTrackingStateService, decideIssueAction } from "./github-tracking-state.js";
|
||||
export { GitHubTrackingReconciler, RECONCILE_CONCURRENCY_LIMIT, RECONCILE_SCAN_LIMIT } from "./github-tracking-reconciler.js";
|
||||
|
||||
67
packages/dashboard/src/knowledge-index-refresh.ts
Normal file
67
packages/dashboard/src/knowledge-index-refresh.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
import type { TaskStore } from "@fusion/core";
|
||||
import { refreshKnowledgeForTask } from "./knowledge-index.js";
|
||||
|
||||
/**
|
||||
* Task-completion refresh hook for the persistent knowledge index (U14).
|
||||
*
|
||||
* Listens for `task:moved` and, when a task reaches `done`, incrementally
|
||||
* re-indexes just that task as a knowledge page (one upsert, never a full
|
||||
* re-index). Mirrors the attach/detach/start/stop lifecycle of
|
||||
* `GitHubSourceIssueCloseService` so it can be wired the same way alongside the
|
||||
* other `task:moved` listeners. All refresh work is fail-soft (see
|
||||
* {@link refreshKnowledgeForTask}) so it can never disrupt task completion.
|
||||
*/
|
||||
interface TaskMovedEvent {
|
||||
task: { id: string };
|
||||
// store's `task:moved` carries `ColumnId`; this handler only literal-compares
|
||||
// legacy ids, so the widened string field is safe.
|
||||
from: string;
|
||||
to: string;
|
||||
}
|
||||
|
||||
export class KnowledgeIndexRefreshService {
|
||||
private readonly defaultStore: TaskStore;
|
||||
private readonly listeners = new Map<TaskStore, { onTaskMoved: (event: TaskMovedEvent) => void }>();
|
||||
private started = false;
|
||||
|
||||
constructor(store: TaskStore) {
|
||||
this.defaultStore = store;
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.started) return;
|
||||
this.started = true;
|
||||
this.attach(this.defaultStore);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (!this.started) return;
|
||||
this.started = false;
|
||||
for (const store of this.listeners.keys()) {
|
||||
this.detach(store);
|
||||
}
|
||||
}
|
||||
|
||||
attach(store: TaskStore): void {
|
||||
if (this.listeners.has(store)) return;
|
||||
const onTaskMoved = (event: TaskMovedEvent): void => {
|
||||
void this.handleTaskMoved(store, event);
|
||||
};
|
||||
this.listeners.set(store, { onTaskMoved });
|
||||
if (this.started) {
|
||||
store.on("task:moved", onTaskMoved);
|
||||
}
|
||||
}
|
||||
|
||||
detach(store: TaskStore): void {
|
||||
const handlers = this.listeners.get(store);
|
||||
if (!handlers) return;
|
||||
store.off("task:moved", handlers.onTaskMoved);
|
||||
this.listeners.delete(store);
|
||||
}
|
||||
|
||||
private async handleTaskMoved(store: TaskStore, event: TaskMovedEvent): Promise<void> {
|
||||
if (event.to !== "done") return;
|
||||
await refreshKnowledgeForTask(store, event.task.id);
|
||||
}
|
||||
}
|
||||
386
packages/dashboard/src/knowledge-index.ts
Normal file
386
packages/dashboard/src/knowledge-index.ts
Normal file
@@ -0,0 +1,386 @@
|
||||
/**
|
||||
* Persistent knowledge index (U14).
|
||||
*
|
||||
* A persistent, incrementally-refreshed knowledge layer that downstream agents
|
||||
* can query. Each "page" captures the durable, queryable summary of one source
|
||||
* (currently one page per completed task; PR-history pages share the same row
|
||||
* shape). Pages are stored in the `knowledge_pages` SQLite table (schema +
|
||||
* migration 119 in `packages/core/src/db.ts`).
|
||||
*
|
||||
* ## Delta over `insights` / `memoryView`
|
||||
*
|
||||
* This is intentionally NOT a second copy of the existing surfaces:
|
||||
*
|
||||
* - `InsightStore` / `insights-routes.ts` / `InsightsView` store **LLM-extracted
|
||||
* durable project learnings** ("patterns/principles/pitfalls" mined from
|
||||
* working memory by an agent run). `memoryView` renders the freeform working/
|
||||
* insights **markdown memory files**. Both are *interpretation* layers and both
|
||||
* require a model run to populate.
|
||||
* - The knowledge index is a **deterministic, model-free, keyword-searchable
|
||||
* index of concrete task/PR history** (title, description, modified files,
|
||||
* commits, PR links). It is refreshed **incrementally on task completion** —
|
||||
* one upsert per affected page, never a full re-index — and exposes a plain
|
||||
* keyword **query API** an agent can call to recall "what work touched X".
|
||||
*
|
||||
* So the genuinely new capability is: (1) a persistent per-task/PR page store,
|
||||
* (2) an incremental refresh hook on task completion, and (3) a keyword query
|
||||
* API — none of which the insights/memory surfaces provide.
|
||||
*
|
||||
* ## Search
|
||||
*
|
||||
* Matching is plain keyword `LIKE` over a denormalized lowercased `searchText`
|
||||
* column (AND-of-terms), deliberately avoiding SQLite FTS5 — FTS5 is not
|
||||
* available on every SQLite build the engine runs on (see `probeFts5` in
|
||||
* `db.ts`), and the plan scopes this unit to "SQLite full-text/keyword search,
|
||||
* NOT an external embedding API".
|
||||
*
|
||||
* ## Security
|
||||
*
|
||||
* The query API is registered as an {@link ApiRouteRegistrar} (see
|
||||
* `routes/register-knowledge-routes.ts`) so it inherits the dashboard's standard
|
||||
* session/auth middleware AND resolves the database through `getScopedStore(req)`
|
||||
* before reading — exactly like U9. The index holds sensitive repo/commit/PR
|
||||
* content, so it is an information-disclosure surface, never an open endpoint.
|
||||
*/
|
||||
|
||||
import type { Database, TaskStore } from "@fusion/core";
|
||||
|
||||
/** The kind of source a knowledge page was indexed from. */
|
||||
export type KnowledgeSourceKind = "task" | "pr";
|
||||
|
||||
/** A knowledge page row as stored/read from `knowledge_pages`. */
|
||||
export interface KnowledgePage {
|
||||
id: number;
|
||||
sourceKind: KnowledgeSourceKind;
|
||||
sourceId: string;
|
||||
/** Stable dedupe key (`<sourceKind>:<sourceId>`); upserts target this. */
|
||||
sourceKey: string;
|
||||
title: string;
|
||||
summary: string | null;
|
||||
content: string;
|
||||
tags: string[];
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
/** Input for {@link upsertKnowledgePage}. */
|
||||
export interface KnowledgePageInput {
|
||||
sourceKind: KnowledgeSourceKind;
|
||||
sourceId: string;
|
||||
title: string;
|
||||
summary?: string | null;
|
||||
content: string;
|
||||
tags?: string[];
|
||||
/** Injectable clock for deterministic tests. Defaults to now. */
|
||||
now?: string;
|
||||
}
|
||||
|
||||
interface KnowledgePageRow {
|
||||
id: number;
|
||||
sourceKind: string;
|
||||
sourceId: string;
|
||||
sourceKey: string;
|
||||
title: string;
|
||||
summary: string | null;
|
||||
content: string;
|
||||
tags: string | null;
|
||||
searchText: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
/** Maximum number of pages a single keyword query returns. */
|
||||
export const KNOWLEDGE_QUERY_DEFAULT_LIMIT = 20;
|
||||
export const KNOWLEDGE_QUERY_MAX_LIMIT = 100;
|
||||
|
||||
function sourceKeyFor(kind: KnowledgeSourceKind, id: string): string {
|
||||
return `${kind}:${id}`;
|
||||
}
|
||||
|
||||
function rowToPage(row: KnowledgePageRow): KnowledgePage {
|
||||
let tags: string[] = [];
|
||||
if (row.tags) {
|
||||
try {
|
||||
const parsed = JSON.parse(row.tags) as unknown;
|
||||
if (Array.isArray(parsed)) tags = parsed.filter((t): t is string => typeof t === "string");
|
||||
} catch {
|
||||
tags = [];
|
||||
}
|
||||
}
|
||||
return {
|
||||
id: row.id,
|
||||
sourceKind: row.sourceKind as KnowledgeSourceKind,
|
||||
sourceId: row.sourceId,
|
||||
sourceKey: row.sourceKey,
|
||||
title: row.title,
|
||||
summary: row.summary,
|
||||
content: row.content,
|
||||
tags,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the denormalized, lowercased search blob a page is matched against.
|
||||
* Pure so it can be unit-tested independently of the DB.
|
||||
*/
|
||||
export function buildSearchText(input: {
|
||||
title: string;
|
||||
summary?: string | null;
|
||||
content: string;
|
||||
tags?: string[];
|
||||
}): string {
|
||||
return [
|
||||
input.title,
|
||||
input.summary ?? "",
|
||||
input.content,
|
||||
(input.tags ?? []).join(" "),
|
||||
]
|
||||
.join(" ")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize a free-text query into lowercased keyword terms. Empty / whitespace
|
||||
* input yields no terms (callers treat that as "match nothing", not "match all",
|
||||
* to avoid returning the whole sensitive index for a blank query).
|
||||
*/
|
||||
export function tokenizeQuery(query: string): string[] {
|
||||
return query
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9_]+/i)
|
||||
.map((t) => t.trim())
|
||||
.filter((t) => t.length > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert or update a knowledge page, keyed by `(sourceKind, sourceId)`.
|
||||
*
|
||||
* **Incremental by construction:** only the row for this source is touched, so a
|
||||
* refresh of one task never rewrites (or re-timestamps) any other page. On an
|
||||
* update, `createdAt` is preserved and only `updatedAt` advances.
|
||||
*
|
||||
* @returns the upserted page and whether it was newly created.
|
||||
*/
|
||||
export function upsertKnowledgePage(
|
||||
db: Database,
|
||||
input: KnowledgePageInput,
|
||||
): { page: KnowledgePage; created: boolean } {
|
||||
const now = input.now ?? new Date().toISOString();
|
||||
const sourceKey = sourceKeyFor(input.sourceKind, input.sourceId);
|
||||
const tags = input.tags ?? [];
|
||||
const searchText = buildSearchText({
|
||||
title: input.title,
|
||||
summary: input.summary,
|
||||
content: input.content,
|
||||
tags,
|
||||
});
|
||||
const tagsJson = JSON.stringify(tags);
|
||||
|
||||
const existing = db
|
||||
.prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?")
|
||||
.get(sourceKey) as KnowledgePageRow | undefined;
|
||||
|
||||
if (existing) {
|
||||
db.prepare(
|
||||
`UPDATE knowledge_pages
|
||||
SET title = ?, summary = ?, content = ?, tags = ?, searchText = ?, updatedAt = ?
|
||||
WHERE sourceKey = ?`,
|
||||
).run(input.title, input.summary ?? null, input.content, tagsJson, searchText, now, sourceKey);
|
||||
const updated = db
|
||||
.prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?")
|
||||
.get(sourceKey) as KnowledgePageRow;
|
||||
return { page: rowToPage(updated), created: false };
|
||||
}
|
||||
|
||||
db.prepare(
|
||||
`INSERT INTO knowledge_pages
|
||||
(sourceKind, sourceId, sourceKey, title, summary, content, tags, searchText, createdAt, updatedAt)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
).run(
|
||||
input.sourceKind,
|
||||
input.sourceId,
|
||||
sourceKey,
|
||||
input.title,
|
||||
input.summary ?? null,
|
||||
input.content,
|
||||
tagsJson,
|
||||
searchText,
|
||||
now,
|
||||
now,
|
||||
);
|
||||
const inserted = db
|
||||
.prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?")
|
||||
.get(sourceKey) as KnowledgePageRow;
|
||||
return { page: rowToPage(inserted), created: true };
|
||||
}
|
||||
|
||||
/** Fetch a single page by its source identity, or `undefined`. */
|
||||
export function getKnowledgePage(
|
||||
db: Database,
|
||||
sourceKind: KnowledgeSourceKind,
|
||||
sourceId: string,
|
||||
): KnowledgePage | undefined {
|
||||
const row = db
|
||||
.prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?")
|
||||
.get(sourceKeyFor(sourceKind, sourceId)) as KnowledgePageRow | undefined;
|
||||
return row ? rowToPage(row) : undefined;
|
||||
}
|
||||
|
||||
/** Options for {@link queryKnowledgePages}. */
|
||||
export interface KnowledgeQueryOptions {
|
||||
query: string;
|
||||
sourceKind?: KnowledgeSourceKind;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keyword search the index. Returns pages whose `searchText` contains **all**
|
||||
* query terms (AND), most-recently-updated first. A blank/termless query returns
|
||||
* an empty list rather than the whole index.
|
||||
*/
|
||||
export function queryKnowledgePages(db: Database, options: KnowledgeQueryOptions): KnowledgePage[] {
|
||||
const terms = tokenizeQuery(options.query);
|
||||
if (terms.length === 0) return [];
|
||||
|
||||
const limit = Math.min(
|
||||
Math.max(1, options.limit ?? KNOWLEDGE_QUERY_DEFAULT_LIMIT),
|
||||
KNOWLEDGE_QUERY_MAX_LIMIT,
|
||||
);
|
||||
|
||||
const clauses: string[] = [];
|
||||
const params: string[] = [];
|
||||
for (const term of terms) {
|
||||
clauses.push("searchText LIKE ? ESCAPE '\\'");
|
||||
params.push(`%${escapeLike(term)}%`);
|
||||
}
|
||||
if (options.sourceKind) {
|
||||
clauses.push("sourceKind = ?");
|
||||
params.push(options.sourceKind);
|
||||
}
|
||||
|
||||
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
|
||||
const rows = db
|
||||
.prepare(`SELECT * FROM knowledge_pages ${where} ORDER BY updatedAt DESC, id DESC LIMIT ?`)
|
||||
.all(...params, limit) as KnowledgePageRow[];
|
||||
return rows.map(rowToPage);
|
||||
}
|
||||
|
||||
/** Escape SQLite `LIKE` wildcards in a term so user input can't inject them. */
|
||||
function escapeLike(term: string): string {
|
||||
return term.replace(/[\\%_]/g, (ch) => `\\${ch}`);
|
||||
}
|
||||
|
||||
/** Total number of pages in the index. */
|
||||
export function countKnowledgePages(db: Database): number {
|
||||
const row = db.prepare("SELECT COUNT(*) AS count FROM knowledge_pages").get() as { count: number };
|
||||
return row.count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a completed task into a deterministic knowledge page body. Pure so the
|
||||
* refresh hook is testable without a real store. Concatenates the durable,
|
||||
* non-sensitive facts: title, description, modified files, associated commit
|
||||
* subjects, and PR link if present.
|
||||
*/
|
||||
export function renderTaskPage(task: {
|
||||
id: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
modifiedFiles?: string[];
|
||||
commitSubjects?: string[];
|
||||
prUrl?: string | null;
|
||||
column?: string;
|
||||
}): KnowledgePageInput {
|
||||
const title = (task.title ?? "").trim() || `Task ${task.id}`;
|
||||
const lines: string[] = [];
|
||||
if (task.description?.trim()) {
|
||||
lines.push(task.description.trim());
|
||||
}
|
||||
if (task.modifiedFiles && task.modifiedFiles.length > 0) {
|
||||
lines.push(`Files: ${task.modifiedFiles.join(", ")}`);
|
||||
}
|
||||
if (task.commitSubjects && task.commitSubjects.length > 0) {
|
||||
lines.push(`Commits:\n${task.commitSubjects.map((s) => `- ${s}`).join("\n")}`);
|
||||
}
|
||||
if (task.prUrl) {
|
||||
lines.push(`PR: ${task.prUrl}`);
|
||||
}
|
||||
const tags = (task.modifiedFiles ?? [])
|
||||
.map((f) => f.split("/").pop() ?? f)
|
||||
.filter((t) => t.length > 0);
|
||||
return {
|
||||
sourceKind: "task",
|
||||
sourceId: task.id,
|
||||
title,
|
||||
summary: task.description?.trim().slice(0, 280) || null,
|
||||
content: lines.join("\n\n") || title,
|
||||
tags,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Incremental refresh hook: index (or re-index) a single task as a knowledge
|
||||
* page. Intended to be invoked from the task-completion path (or by code that
|
||||
* observes a task reaching `done`). It reads only the one task and upserts only
|
||||
* its page, so unaffected pages are never touched.
|
||||
*
|
||||
* **Fail-soft:** any read/write error is logged and swallowed so a knowledge
|
||||
* refresh can never break the task-completion flow that called it.
|
||||
*
|
||||
* @returns the upserted page, or `null` if the task could not be loaded/indexed.
|
||||
*/
|
||||
export async function refreshKnowledgeForTask(
|
||||
store: TaskStore,
|
||||
taskId: string,
|
||||
options?: { now?: string },
|
||||
): Promise<KnowledgePage | null> {
|
||||
try {
|
||||
const detail = await store.getTask(taskId);
|
||||
if (!detail) return null;
|
||||
|
||||
let commitSubjects: string[] = [];
|
||||
try {
|
||||
const lineageId = (detail as { lineageId?: string }).lineageId ?? detail.id;
|
||||
const rows = store
|
||||
.getDatabase()
|
||||
.prepare(
|
||||
"SELECT commitSubject FROM task_commit_associations WHERE taskLineageId = ? ORDER BY authoredAt ASC",
|
||||
)
|
||||
.all(lineageId) as Array<{ commitSubject: string }>;
|
||||
commitSubjects = rows.map((r) => r.commitSubject);
|
||||
} catch {
|
||||
commitSubjects = [];
|
||||
}
|
||||
|
||||
const prUrl = extractPrUrl(detail);
|
||||
const input = renderTaskPage({
|
||||
id: detail.id,
|
||||
title: detail.title,
|
||||
description: detail.description,
|
||||
modifiedFiles: (detail as { modifiedFiles?: string[] }).modifiedFiles,
|
||||
commitSubjects,
|
||||
prUrl,
|
||||
column: detail.column,
|
||||
});
|
||||
if (options?.now) input.now = options.now;
|
||||
|
||||
const { page } = upsertKnowledgePage(store.getDatabase(), input);
|
||||
return page;
|
||||
} catch (err) {
|
||||
console.warn(`[knowledge-index] refresh skipped for task ${taskId}:`, err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Best-effort extraction of a PR URL from a task detail, tolerant of shape. */
|
||||
function extractPrUrl(detail: unknown): string | null {
|
||||
if (!detail || typeof detail !== "object") return null;
|
||||
const d = detail as Record<string, unknown>;
|
||||
if (typeof d.prUrl === "string" && d.prUrl) return d.prUrl;
|
||||
const pr = d.pullRequest as Record<string, unknown> | undefined;
|
||||
if (pr && typeof pr.url === "string" && pr.url) return pr.url;
|
||||
if (pr && typeof pr.htmlUrl === "string" && pr.htmlUrl) return pr.htmlUrl;
|
||||
return null;
|
||||
}
|
||||
@@ -169,6 +169,7 @@ import { registerModelRoutes } from "./routes/register-model-routes.js";
|
||||
import { registerCustomProviderRoutes } from "./routes/register-custom-provider-routes.js";
|
||||
import { registerUsageRoutes } from "./routes/register-usage-routes.js";
|
||||
import { registerCommandCenterRoutes } from "./routes/register-command-center-routes.js";
|
||||
import { registerKnowledgeRoutes } from "./routes/register-knowledge-routes.js";
|
||||
import { registerSignalRoutes } from "./routes/register-signal-routes.js";
|
||||
import { registerAuthRoutes } from "./routes/register-auth-routes.js";
|
||||
import { registerRuntimeProviderRoutes } from "./routes/register-runtime-provider-routes.js";
|
||||
@@ -1994,6 +1995,11 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout
|
||||
// U9 — Command Center analytics + live snapshot endpoints. Thin adapters over
|
||||
// the core aggregators; inherit standard auth + getScopedStore project scoping.
|
||||
registerCommandCenterRoutes(routeContext);
|
||||
// U14 — persistent knowledge index query + incremental-refresh endpoints.
|
||||
// Inherit standard auth + getScopedStore project scoping (same as U9); the
|
||||
// index holds sensitive repo/PR content so no endpoint is unauthenticated or
|
||||
// cross-project readable.
|
||||
registerKnowledgeRoutes(routeContext);
|
||||
// U11 — inbound external signal webhooks (Sentry/Datadog/PagerDuty/generic).
|
||||
// Each route HMAC-verifies against a per-provider secret; never an
|
||||
// unauthenticated task-creation endpoint.
|
||||
|
||||
@@ -43,6 +43,7 @@ import { GitHubTrackingCommentService } from "../github-tracking-comments.js";
|
||||
import { GitHubTrackingStateService } from "../github-tracking-state.js";
|
||||
import { GitHubTrackingReconciler, RECONCILE_SCAN_LIMIT } from "../github-tracking-reconciler.js";
|
||||
import { GitHubSourceIssueCloseService } from "../github-source-issue-close.js";
|
||||
import { KnowledgeIndexRefreshService } from "../knowledge-index-refresh.js";
|
||||
import { githubRateLimiter } from "../github-poll.js";
|
||||
import * as projectStoreResolver from "../project-store-resolver.js";
|
||||
import { generatePrMetadata } from "../pr-metadata-generator.js";
|
||||
@@ -2485,6 +2486,12 @@ export function registerGitGitHubRoutes(ctx: ApiRoutesContext): void {
|
||||
githubSourceIssueCloseService.start();
|
||||
ctx.registerDispose(() => githubSourceIssueCloseService.stop());
|
||||
|
||||
// U14 — incremental knowledge-index refresh on task completion. Listens for
|
||||
// task:moved → done and re-indexes just that task as a knowledge page.
|
||||
const knowledgeIndexRefreshService = new KnowledgeIndexRefreshService(store);
|
||||
knowledgeIndexRefreshService.start();
|
||||
ctx.registerDispose(() => knowledgeIndexRefreshService.stop());
|
||||
|
||||
const githubTrackingStateService = new GitHubTrackingStateService(store);
|
||||
const githubTrackingReconciler = new GitHubTrackingReconciler();
|
||||
const reconcileScheduledStores = new WeakSet<TaskStore>();
|
||||
|
||||
95
packages/dashboard/src/routes/register-knowledge-routes.ts
Normal file
95
packages/dashboard/src/routes/register-knowledge-routes.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
import { ApiError } from "../api-error.js";
|
||||
import {
|
||||
queryKnowledgePages,
|
||||
countKnowledgePages,
|
||||
refreshKnowledgeForTask,
|
||||
KNOWLEDGE_QUERY_DEFAULT_LIMIT,
|
||||
KNOWLEDGE_QUERY_MAX_LIMIT,
|
||||
type KnowledgeSourceKind,
|
||||
} from "../knowledge-index.js";
|
||||
import type { ApiRouteRegistrar } from "./types.js";
|
||||
|
||||
/**
|
||||
* Persistent knowledge-index API (U14).
|
||||
*
|
||||
* Thin HTTP adapter over the keyword index in `knowledge-index.ts`. Downstream
|
||||
* agents call `GET /api/knowledge/query` to recall task/PR history.
|
||||
*
|
||||
* Security (same contract as U9 — `register-command-center-routes.ts`):
|
||||
* - Every route inherits the dashboard's standard session/auth middleware via
|
||||
* the {@link ApiRouteRegistrar} contract, so an unauthenticated request is
|
||||
* rejected with 401 by the server-level auth middleware before reaching these
|
||||
* handlers. No knowledge endpoint is unauthenticated.
|
||||
* - Every endpoint resolves the database through `getScopedStore(req)` before
|
||||
* reading/writing, so a project-A caller can never read project-B pages. The
|
||||
* index holds sensitive repo/commit/PR content, so it is an information-
|
||||
* disclosure surface, not an open endpoint.
|
||||
*/
|
||||
|
||||
const VALID_SOURCE_KINDS: ReadonlySet<string> = new Set<KnowledgeSourceKind>(["task", "pr"]);
|
||||
|
||||
function resolveSourceKind(query: { sourceKind?: unknown }): KnowledgeSourceKind | undefined {
|
||||
const raw = typeof query.sourceKind === "string" ? query.sourceKind : undefined;
|
||||
return raw !== undefined && VALID_SOURCE_KINDS.has(raw)
|
||||
? (raw as KnowledgeSourceKind)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function resolveLimit(query: { limit?: unknown }): number {
|
||||
const raw = typeof query.limit === "string" ? Number.parseInt(query.limit, 10) : NaN;
|
||||
if (!Number.isFinite(raw)) return KNOWLEDGE_QUERY_DEFAULT_LIMIT;
|
||||
return Math.min(Math.max(1, raw), KNOWLEDGE_QUERY_MAX_LIMIT);
|
||||
}
|
||||
|
||||
export const registerKnowledgeRoutes: ApiRouteRegistrar = (ctx) => {
|
||||
const { router, getScopedStore, rethrowAsApiError } = ctx;
|
||||
|
||||
/**
|
||||
* GET /api/knowledge/query?q=<keywords>&sourceKind=task|pr&limit=N
|
||||
* Keyword search over the project-scoped knowledge index. Returns the matching
|
||||
* pages (most-recently-updated first) and the total index size.
|
||||
*/
|
||||
router.get("/knowledge/query", async (req, res) => {
|
||||
try {
|
||||
const store = await getScopedStore(req);
|
||||
const q = typeof req.query.q === "string" ? req.query.q : "";
|
||||
const pages = queryKnowledgePages(store.getDatabase(), {
|
||||
query: q,
|
||||
sourceKind: resolveSourceKind(req.query),
|
||||
limit: resolveLimit(req.query),
|
||||
});
|
||||
res.json({
|
||||
query: q,
|
||||
pages,
|
||||
total: countKnowledgePages(store.getDatabase()),
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof ApiError) throw err;
|
||||
rethrowAsApiError(err, "Failed to query knowledge index");
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/knowledge/refresh { taskId }
|
||||
* Incrementally re-index a single task as a knowledge page. Exposes the
|
||||
* task-completion refresh hook over HTTP so the completion path (or an
|
||||
* operator) can trigger an incremental refresh without a full re-index.
|
||||
*/
|
||||
router.post("/knowledge/refresh", async (req, res) => {
|
||||
try {
|
||||
const store = await getScopedStore(req);
|
||||
const taskId = typeof req.body?.taskId === "string" ? req.body.taskId.trim() : "";
|
||||
if (!taskId) {
|
||||
throw new ApiError(400, "taskId is required");
|
||||
}
|
||||
const page = await refreshKnowledgeForTask(store, taskId);
|
||||
if (!page) {
|
||||
throw new ApiError(404, `Task not found or could not be indexed: ${taskId}`);
|
||||
}
|
||||
res.json({ page });
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof ApiError) throw err;
|
||||
rethrowAsApiError(err, "Failed to refresh knowledge index");
|
||||
}
|
||||
});
|
||||
};
|
||||
@@ -743,10 +743,10 @@ describe("RoadmapStore", () => {
|
||||
});
|
||||
|
||||
describe("schema version", () => {
|
||||
it("schema version is 118 after init", () => {
|
||||
it("schema version is 119 after init", () => {
|
||||
// Tracks @fusion/core's SCHEMA_VERSION (the roadmap store layers on core's
|
||||
// Database). Bump this in lockstep when core adds a migration.
|
||||
expect(db.getSchemaVersion()).toBe(118);
|
||||
expect(db.getSchemaVersion()).toBe(119);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user