From 0a87890bd861294ff5d1efa56667a08c86d89468 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Mon, 15 Jun 2026 20:34:52 -0700 Subject: [PATCH] =?UTF-8?q?feat(knowledge):=20U14=20=E2=80=94=20persistent?= =?UTF-8?q?=20knowledge=20index?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds knowledge_pages (db migration 118→119) + a deterministic, model-free keyword index of task/PR history in packages/dashboard/src, incrementally refreshed on task completion (task:moved→done listener) and queryable via an auth-gated, project-scoped API. Complements the LLM-extracted insights/memory surfaces rather than duplicating them. Follow-ups: no React view yet; PR-history page population attaches via U18. --- .changeset/u14-knowledge-index.md | 10 + .../core/src/__tests__/db-migrate.test.ts | 30 +- packages/core/src/__tests__/db.test.ts | 44 +- .../core/src/__tests__/goals-schema.test.ts | 2 +- .../core/src/__tests__/insight-store.test.ts | 10 +- .../__tests__/merge-request-record.test.ts | 2 +- .../core/src/__tests__/mission-store.test.ts | 2 +- packages/core/src/__tests__/run-audit.test.ts | 4 +- .../src/__tests__/store-merge-queue.test.ts | 2 +- .../core/src/__tests__/task-documents.test.ts | 2 +- .../core/src/__tests__/usage-events.test.ts | 11 +- packages/core/src/db.ts | 57 ++- .../src/__tests__/knowledge-index.test.ts | 240 +++++++++++ .../register-knowledge-routes.auth.test.ts | 81 ++++ .../register-knowledge-routes.test.ts | 185 +++++++++ packages/dashboard/src/index.ts | 17 + .../dashboard/src/knowledge-index-refresh.ts | 67 +++ packages/dashboard/src/knowledge-index.ts | 386 ++++++++++++++++++ packages/dashboard/src/routes.ts | 6 + .../src/routes/register-git-github.ts | 7 + .../src/routes/register-knowledge-routes.ts | 95 +++++ .../src/store/__tests__/roadmap-store.test.ts | 4 +- 22 files changed, 1208 insertions(+), 56 deletions(-) create mode 100644 .changeset/u14-knowledge-index.md create mode 100644 packages/dashboard/src/__tests__/knowledge-index.test.ts create mode 100644 packages/dashboard/src/__tests__/register-knowledge-routes.auth.test.ts create mode 100644 packages/dashboard/src/__tests__/register-knowledge-routes.test.ts create mode 100644 packages/dashboard/src/knowledge-index-refresh.ts create mode 100644 packages/dashboard/src/knowledge-index.ts create mode 100644 packages/dashboard/src/routes/register-knowledge-routes.ts diff --git a/.changeset/u14-knowledge-index.md b/.changeset/u14-knowledge-index.md new file mode 100644 index 0000000000..0027d4e171 --- /dev/null +++ b/.changeset/u14-knowledge-index.md @@ -0,0 +1,10 @@ +--- +"@runfusion/fusion": minor +--- + +Add a persistent, incrementally-refreshed knowledge index (U14) downstream agents can query. + +- **Schema** — new `knowledge_pages` SQLite table (`packages/core/src/db.ts`) with `SCHEMA_VERSION` bumped 118 → 119 (added in the same change as the migration; the fingerprint auto-covers SCHEMA_SQL tables). Keyword search uses a denormalized lowercased `searchText` column with AND-of-terms `LIKE` matching, deliberately avoiding SQLite FTS5 (not available on every build) and any external embedding API. +- **Index module** (`packages/dashboard/src/knowledge-index.ts`) — upsert-by-source-key pages, a model-free keyword query API, and `refreshKnowledgeForTask` that re-indexes a single completed task (one upsert, never a full re-index, so unaffected pages keep their timestamps). This is the delta over the existing `insights`/`memoryView` surfaces, which are LLM-extracted learnings, not a deterministic searchable index of concrete task/PR history. +- **Refresh hook** — `KnowledgeIndexRefreshService` listens for `task:moved → done` (mirroring `GitHubSourceIssueCloseService`) and is wired alongside the other completion listeners; fail-soft so it can never disrupt task completion. +- **Query API** (`register-knowledge-routes.ts`) — `GET /api/knowledge/query` and `POST /api/knowledge/refresh`, registered as an `ApiRouteRegistrar` so they inherit the dashboard's standard session/auth middleware (401 when unauthenticated) and apply `getScopedStore(req)` (no cross-project reads), exactly like U9. diff --git a/packages/core/src/__tests__/db-migrate.test.ts b/packages/core/src/__tests__/db-migrate.test.ts index 5f91c30949..b1ecb81dfb 100644 --- a/packages/core/src/__tests__/db-migrate.test.ts +++ b/packages/core/src/__tests__/db-migrate.test.ts @@ -715,7 +715,7 @@ describe("schema migration", () => { const row = db.prepare("SELECT deletedAt FROM tasks WHERE id = 'FN-legacy'").get() as { deletedAt: string | null }; expect(row.deletedAt).toBeNull(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -748,7 +748,7 @@ describe("schema migration", () => { { id: "WS-001", mode: "prompt", gateMode: "advisory" }, { id: "WS-002", mode: "script", gateMode: "advisory" }, ]); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -798,7 +798,7 @@ describe("schema migration", () => { reviewerContextRetryCount: 0, reviewerFallbackRetryCount: 0, }); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -827,7 +827,7 @@ describe("schema migration", () => { const columns = db.prepare("PRAGMA table_info(milestones)").all() as Array<{ name: string }>; expect(columns.map((column) => column.name)).toContain("acceptanceCriteria"); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -868,7 +868,7 @@ describe("schema migration", () => { const missionColumns = db.prepare("PRAGMA table_info(missions)").all() as Array<{ name: string }>; expect(missionColumns.map((column) => column.name)).toContain("autoMerge"); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -902,7 +902,7 @@ describe("schema migration", () => { { id: "WS-002", mode: "script", enabled: 1, gateMode: "advisory" }, { id: "WS-003", mode: "prompt", enabled: 0, gateMode: "advisory" }, ]); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -939,7 +939,7 @@ describe("schema migration", () => { const indexes = db.prepare("PRAGMA index_list(mission_goals)").all() as Array<{ name: string }>; expect(indexes.some((index) => index.name === "idxMissionGoalsGoalId")).toBe(true); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -1000,7 +1000,7 @@ describe("schema migration", () => { expect(customFieldsColumn).toBeDefined(); expect(customFieldsColumn?.dflt_value).toBe("'{}'"); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -1038,7 +1038,7 @@ describe("schema migration", () => { const indexes = db.prepare("PRAGMA index_list(workflow_settings)").all() as Array<{ name: string }>; expect(indexes.some((index) => index.name === "idx_workflow_settings_project")).toBe(true); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -1120,7 +1120,7 @@ describe("schema migration", () => { expect(indexNames).toContain("idx_cli_sessions_chatSessionId"); expect(indexNames).toContain("idx_cli_sessions_project_state"); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -1152,7 +1152,7 @@ describe("schema migration", () => { .all() as Array<{ name: string }>; expect(columns.map((column) => column.name)).toContain("cliExecutorAdapterId"); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -1162,7 +1162,7 @@ describe("schema migration", () => { const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as Array<{ name: string }>; expect(tables.map((row) => row.name)).toContain("cli_sessions"); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -1219,20 +1219,20 @@ describe("schema migration", () => { .get() as { migrated_fragment_id: string | null }; expect(stepRow.migrated_fragment_id).toBeNull(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); it("migration 109 is idempotent on re-init", () => { const db = new Database(fusionDir); db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); // Re-open the same on-disk DB: already at 109, the 109 block must be a no-op. const reopened = new Database(fusionDir); reopened.init(); - expect(reopened.getSchemaVersion()).toBe(118); + expect(reopened.getSchemaVersion()).toBe(119); const workflowColumns = reopened.prepare("PRAGMA table_info(workflows)").all() as Array<{ name: string }>; expect(workflowColumns.filter((c) => c.name === "kind")).toHaveLength(1); const stepColumns = reopened.prepare("PRAGMA table_info(workflow_steps)").all() as Array<{ name: string }>; diff --git a/packages/core/src/__tests__/db.test.ts b/packages/core/src/__tests__/db.test.ts index aa49d59aa6..755b4ecbcb 100644 --- a/packages/core/src/__tests__/db.test.ts +++ b/packages/core/src/__tests__/db.test.ts @@ -334,7 +334,7 @@ describe("Database", () => { }); it("seeds schema version", () => { - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); }); it("includes tokenUsageCacheWriteTokens on freshly initialized tasks table", () => { @@ -393,7 +393,7 @@ describe("Database", () => { it("is idempotent - calling init() twice does not fail", () => { expect(() => db.init()).not.toThrow(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); }); it("does not overwrite existing config on re-init", () => { // Update the config @@ -1463,7 +1463,7 @@ describe("schema migrations", () => { db.init(); // Verify version bumped to 29 (includes v1→v2 through v26→v29) - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); // Verify new columns exist and existing data is intact const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; @@ -1488,15 +1488,15 @@ describe("schema migrations", () => { const db = new Database(fusionDir); db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); // Re-init should not fail db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); // Re-init should not fail db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); db.close(); }); @@ -1531,7 +1531,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; expect(cols.map((col) => col.name)).toContain("priority"); @@ -1572,7 +1572,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; const colNames = cols.map((col) => col.name); @@ -1644,7 +1644,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; const colNames = cols.map((col) => col.name); @@ -1884,7 +1884,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); const cols = db.prepare("PRAGMA table_info(chat_messages)").all() as Array<{ name: string }>; expect(cols.map((col) => col.name)).toContain("attachments"); @@ -1958,7 +1958,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>; expect(tables).toEqual([{ name: "agentRatings" }]); @@ -1982,7 +1982,7 @@ describe("schema migrations", () => { db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>; expect(tables).toEqual([{ name: "mission_events" }]); @@ -2086,7 +2086,7 @@ describe("schema migrations", () => { db.init(); // Verify version bumped to 29 - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); // Verify new columns exist and existing data is intact const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; @@ -2305,7 +2305,7 @@ describe("schema migrations", () => { localDb.init(); - expect(localDb.getSchemaVersion()).toBe(118); + expect(localDb.getSchemaVersion()).toBe(119); const columns = localDb.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; expect(columns.map((column) => column.name)).toContain("tokenUsageCacheWriteTokens"); @@ -2616,7 +2616,7 @@ describe("createDatabase factory", () => { const db = createDatabase(fusionDir); db.init(); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); expect(db.getLastModified()).toBeGreaterThan(0); db.close(); @@ -2770,7 +2770,7 @@ describe("migration v77 task token budget columns", () => { migrated = new Database(fusion); migrated.init(); - expect(migrated.getSchemaVersion()).toBe(118); + expect(migrated.getSchemaVersion()).toBe(119); const rows = migrated.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>; const names = new Set(rows.map((row) => row.name)); expect(names.has("tokenBudgetSoftAlertedAt")).toBe(true); @@ -2801,7 +2801,7 @@ describe("migration v106 adds tasks.transitionPending (FN-1417)", () => { const fresh = new Database(fusion); try { fresh.init(); - expect(fresh.getSchemaVersion()).toBe(118); + expect(fresh.getSchemaVersion()).toBe(119); const names = new Set( (fresh.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>).map((r) => r.name), ); @@ -2829,7 +2829,7 @@ describe("migration v106 adds tasks.transitionPending (FN-1417)", () => { migrated = new Database(fusion); migrated.init(); - expect(migrated.getSchemaVersion()).toBe(118); + expect(migrated.getSchemaVersion()).toBe(119); const names = new Set( (migrated.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>).map((r) => r.name), ); @@ -2855,7 +2855,7 @@ describe("migration v107 adds workflow_run_branches + index (FN-1417)", () => { const fresh = new Database(fusion); try { fresh.init(); - expect(fresh.getSchemaVersion()).toBe(118); + expect(fresh.getSchemaVersion()).toBe(119); const table = fresh .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'workflow_run_branches'") .get() as { name: string } | undefined; @@ -2889,7 +2889,7 @@ describe("migration v107 adds workflow_run_branches + index (FN-1417)", () => { migrated = new Database(fusion); migrated.init(); - expect(migrated.getSchemaVersion()).toBe(118); + expect(migrated.getSchemaVersion()).toBe(119); const table = migrated .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'workflow_run_branches'") .get() as { name: string } | undefined; @@ -2930,7 +2930,7 @@ describe("migration v67 drops orphan project auth tables", () => { migrated = new Database(fusion); migrated.init(); - expect(migrated.getSchemaVersion()).toBe(118); + expect(migrated.getSchemaVersion()).toBe(119); const tables = migrated .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'project_auth_%'") .all() as Array<{ name: string }>; @@ -2957,7 +2957,7 @@ describe("migration v67 drops orphan project auth tables", () => { try { fresh.init(); - expect(fresh.getSchemaVersion()).toBe(118); + expect(fresh.getSchemaVersion()).toBe(119); const tables = fresh .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'project_auth_%'") .all() as Array<{ name: string }>; diff --git a/packages/core/src/__tests__/goals-schema.test.ts b/packages/core/src/__tests__/goals-schema.test.ts index 75cf8c431e..faea0da0bb 100644 --- a/packages/core/src/__tests__/goals-schema.test.ts +++ b/packages/core/src/__tests__/goals-schema.test.ts @@ -91,6 +91,6 @@ describe("goals schema", () => { }); it("reports schema version 101", () => { - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); }); }); diff --git a/packages/core/src/__tests__/insight-store.test.ts b/packages/core/src/__tests__/insight-store.test.ts index 84f57c78a3..e7bac40c81 100644 --- a/packages/core/src/__tests__/insight-store.test.ts +++ b/packages/core/src/__tests__/insight-store.test.ts @@ -1000,7 +1000,7 @@ describe("Migration: pre-33 DB upgrade", () => { // Step 1: Create a fresh database at v33 (runs all migrations up to 33) const db1 = createDatabase(legacyDir); db1.init(); - expect(db1.getSchemaVersion()).toBe(118); + expect(db1.getSchemaVersion()).toBe(119); db1.close(); // Step 2: Manually downgrade to version 32 and drop insight tables @@ -1035,7 +1035,7 @@ describe("Migration: pre-33 DB upgrade", () => { expect(tableNamesBefore).not.toContain("project_insight_runs"); // Now run init — this triggers the v32→v33 migration db3.init(); - expect(db3.getSchemaVersion()).toBe(118); + expect(db3.getSchemaVersion()).toBe(119); // Step 4: Verify insight tables exist after migration const tablesAfter = db3.prepare( @@ -1066,12 +1066,12 @@ describe("Migration: pre-33 DB upgrade", () => { try { const db1 = createDatabase(testDir); db1.init(); - expect(db1.getSchemaVersion()).toBe(118); + expect(db1.getSchemaVersion()).toBe(119); db1.close(); const db2 = createDatabase(testDir); expect(() => db2.init()).not.toThrow(); - expect(db2.getSchemaVersion()).toBe(118); + expect(db2.getSchemaVersion()).toBe(119); db2.close(); } finally { rmSync(testDir, { recursive: true, force: true }); @@ -1085,7 +1085,7 @@ describe("Migration: pre-33 DB upgrade", () => { // Step 1: Create a fresh DB and run migrations const db1 = createDatabase(compatDir); db1.init(); - expect(db1.getSchemaVersion()).toBe(118); + expect(db1.getSchemaVersion()).toBe(119); // Step 2: Strip lifecycle and cancelledAt columns by recreating the // table without them. This simulates a DB that was created before the diff --git a/packages/core/src/__tests__/merge-request-record.test.ts b/packages/core/src/__tests__/merge-request-record.test.ts index f68511ee11..088c8a2676 100644 --- a/packages/core/src/__tests__/merge-request-record.test.ts +++ b/packages/core/src/__tests__/merge-request-record.test.ts @@ -38,7 +38,7 @@ describe("TaskStore merge request record + completion handoff marker", () => { .all() as Array<{ name: string }>; expect(tableRows).toEqual([{ name: "completion_handoff_markers" }, { name: "merge_requests" }]); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); }); it("upserts merge request records", async () => { diff --git a/packages/core/src/__tests__/mission-store.test.ts b/packages/core/src/__tests__/mission-store.test.ts index 71b9dd6758..a214d7288d 100644 --- a/packages/core/src/__tests__/mission-store.test.ts +++ b/packages/core/src/__tests__/mission-store.test.ts @@ -3746,7 +3746,7 @@ describe("MissionStore", () => { describe("Loop State & Validator Run Schema (v31)", () => { it("schema version is 101 after migration", () => { - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); }); it("mission_features table has loop state columns", () => { diff --git a/packages/core/src/__tests__/run-audit.test.ts b/packages/core/src/__tests__/run-audit.test.ts index 68faeb4cdb..26fb06041e 100644 --- a/packages/core/src/__tests__/run-audit.test.ts +++ b/packages/core/src/__tests__/run-audit.test.ts @@ -583,8 +583,8 @@ describe("Run Audit", () => { expect(indexNames).toContain("idxRunAuditEventsTimestamp"); }); - it("schema version is bumped to 118", () => { - expect(db.getSchemaVersion()).toBe(118); + it("schema version is bumped to 119", () => { + expect(db.getSchemaVersion()).toBe(119); }); }); }); diff --git a/packages/core/src/__tests__/store-merge-queue.test.ts b/packages/core/src/__tests__/store-merge-queue.test.ts index f4311184d8..39cb2c6915 100644 --- a/packages/core/src/__tests__/store-merge-queue.test.ts +++ b/packages/core/src/__tests__/store-merge-queue.test.ts @@ -60,7 +60,7 @@ describe("TaskStore merge queue", () => { expect.arrayContaining(["idx_mergeQueue_lease_ready", "idx_mergeQueue_leaseExpiresAt"]), ); - expect(store.getDatabase().getSchemaVersion()).toBe(118); + expect(store.getDatabase().getSchemaVersion()).toBe(119); }); it("migrates a legacy v88 database and preserves task rows", async () => { diff --git a/packages/core/src/__tests__/task-documents.test.ts b/packages/core/src/__tests__/task-documents.test.ts index ec92575074..f22558db48 100644 --- a/packages/core/src/__tests__/task-documents.test.ts +++ b/packages/core/src/__tests__/task-documents.test.ts @@ -51,7 +51,7 @@ describe("TaskStore task documents", () => { expect(tableNames.has("task_documents")).toBe(true); expect(tableNames.has("task_document_revisions")).toBe(true); - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); const index = db .prepare( diff --git a/packages/core/src/__tests__/usage-events.test.ts b/packages/core/src/__tests__/usage-events.test.ts index d17587b55e..6ae70ac961 100644 --- a/packages/core/src/__tests__/usage-events.test.ts +++ b/packages/core/src/__tests__/usage-events.test.ts @@ -175,15 +175,18 @@ describe("usage_events", () => { expect(rows[0].agentId).toBe("A-chat"); }); - // Migration: seed a DB at the PREVIOUS schema version, run migrate, assert - // the table exists and SCHEMA_VERSION equals the highest migration target. - // Fresh-DB tests cannot catch the early-return bug this guards. + // Migration: seed a DB at the version JUST BEFORE usage_events was introduced + // (117 — usage_events is the v118 migration), run migrate, assert the table + // exists and SCHEMA_VERSION reaches the highest migration target. Pinned to + // 117 (not SCHEMA_VERSION-1) so it keeps exercising usage_events' own + // migration as later migrations are added. Fresh-DB tests cannot catch the + // migrate-loop early-return bug this guards. it("creates usage_events when migrating from the previous schema version", () => { db.exec("DROP INDEX IF EXISTS idxUsageEventsTs"); db.exec("DROP INDEX IF EXISTS idxUsageEventsTaskId"); db.exec("DROP INDEX IF EXISTS idxUsageEventsAgentId"); db.exec("DROP TABLE IF EXISTS usage_events"); - db.prepare("UPDATE __meta SET value = ? WHERE key = 'schemaVersion'").run(String(SCHEMA_VERSION - 1)); + db.prepare("UPDATE __meta SET value = ? WHERE key = 'schemaVersion'").run("117"); (db as unknown as { migrate: () => void }).migrate(); diff --git a/packages/core/src/db.ts b/packages/core/src/db.ts index 78332279f6..9c6eb18572 100644 --- a/packages/core/src/db.ts +++ b/packages/core/src/db.ts @@ -162,7 +162,7 @@ export function isFts5CorruptionError(error: unknown): boolean { // ── Schema Definition ──────────────────────────────────────────────── -const SCHEMA_VERSION = 118; +const SCHEMA_VERSION = 119; const TASKS_FTS_AUTOMERGE = 8; const TASKS_FTS_CRISISMERGE = 16; @@ -1230,6 +1230,31 @@ CREATE TABLE IF NOT EXISTS usage_events ( CREATE INDEX IF NOT EXISTS idxUsageEventsTs ON usage_events(ts); CREATE INDEX IF NOT EXISTS idxUsageEventsTaskId ON usage_events(taskId); CREATE INDEX IF NOT EXISTS idxUsageEventsAgentId ON usage_events(agentId); + +-- Persistent, incrementally-refreshed knowledge index (U14). One row per +-- knowledge page (currently one page per completed task; PR-history pages +-- share the same shape). Downstream agents query it through the dashboard's +-- scoped knowledge-index endpoint. searchText is a denormalized lowercased +-- concatenation of the page's title/summary/content + tags used for keyword +-- LIKE matching, so the index works without requiring SQLite FTS5 (which is +-- not available on every build -- see probeFts5 above). Refresh is per-source +-- (upsert by sourceKey), never a full re-index, so unaffected pages keep their +-- timestamps. +CREATE TABLE IF NOT EXISTS knowledge_pages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sourceKind TEXT NOT NULL, + sourceId TEXT NOT NULL, + sourceKey TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + summary TEXT, + content TEXT NOT NULL, + tags TEXT, + searchText TEXT NOT NULL, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idxKnowledgePagesSourceKind ON knowledge_pages(sourceKind); +CREATE INDEX IF NOT EXISTS idxKnowledgePagesUpdatedAt ON knowledge_pages(updatedAt); `; const TABLE_LEVEL_CONSTRAINT_PREFIXES = new Set([ @@ -4773,6 +4798,36 @@ export class Database { }); } + // Migration 119: Persistent knowledge index (U14). One queryable page per + // completed task / PR-history entry, refreshed incrementally (upsert by + // sourceKey) on task completion. Mirrors the SCHEMA_SQL definition above so + // a fresh-from-SCHEMA_SQL DB and a migrated DB converge on the same table. + if (version < 119) { + this.applyMigration(119, () => { + this.db.exec(` + CREATE TABLE IF NOT EXISTS knowledge_pages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sourceKind TEXT NOT NULL, + sourceId TEXT NOT NULL, + sourceKey TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + summary TEXT, + content TEXT NOT NULL, + tags TEXT, + searchText TEXT NOT NULL, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL + ) + `); + this.db.exec(` + CREATE INDEX IF NOT EXISTS idxKnowledgePagesSourceKind ON knowledge_pages(sourceKind) + `); + this.db.exec(` + CREATE INDEX IF NOT EXISTS idxKnowledgePagesUpdatedAt ON knowledge_pages(updatedAt) + `); + }); + } + } /** diff --git a/packages/dashboard/src/__tests__/knowledge-index.test.ts b/packages/dashboard/src/__tests__/knowledge-index.test.ts new file mode 100644 index 0000000000..28fb228fbb --- /dev/null +++ b/packages/dashboard/src/__tests__/knowledge-index.test.ts @@ -0,0 +1,240 @@ +// @vitest-environment node + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { EventEmitter } from "node:events"; + +import { Database, SCHEMA_VERSION } from "@fusion/core"; +import type { TaskStore } from "@fusion/core"; +import { + upsertKnowledgePage, + queryKnowledgePages, + getKnowledgePage, + countKnowledgePages, + refreshKnowledgeForTask, + renderTaskPage, + tokenizeQuery, + buildSearchText, +} from "../knowledge-index.js"; + +function makeDb(): { db: Database; tmpDir: string } { + const tmpDir = mkdtempSync(join(tmpdir(), "kb-knowledge-index-")); + const db = new Database(join(tmpDir, ".fusion")); + db.init(); + return { db, tmpDir }; +} + +describe("knowledge-index store", () => { + let db: Database; + let tmpDir: string; + + beforeEach(() => { + ({ db, tmpDir } = makeDb()); + }); + + afterEach(() => { + db.close(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("creates knowledge_pages with the expected columns on fresh init", () => { + const cols = (db.prepare("PRAGMA table_info(knowledge_pages)").all() as Array<{ name: string }>).map( + (c) => c.name, + ); + expect(cols).toEqual([ + "id", + "sourceKind", + "sourceId", + "sourceKey", + "title", + "summary", + "content", + "tags", + "searchText", + "createdAt", + "updatedAt", + ]); + }); + + it("upserts a page and returns it via a keyword query", () => { + const { created } = upsertKnowledgePage(db, { + sourceKind: "task", + sourceId: "T-1", + title: "Add caching layer", + content: "Introduced an LRU cache in fetcher.ts", + tags: ["fetcher.ts"], + }); + expect(created).toBe(true); + const hits = queryKnowledgePages(db, { query: "cache" }); + expect(hits).toHaveLength(1); + expect(hits[0].sourceId).toBe("T-1"); + expect(hits[0].tags).toEqual(["fetcher.ts"]); + }); + + it("AND-matches all query terms", () => { + upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-1", title: "alpha gadget", content: "only alpha here" }); + upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-2", title: "alpha thing", content: "beta widget" }); + expect(queryKnowledgePages(db, { query: "alpha widget" }).map((p) => p.sourceId)).toEqual(["T-2"]); + }); + + it("a blank/termless query returns nothing (never the whole index)", () => { + upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-1", title: "x", content: "y" }); + expect(queryKnowledgePages(db, { query: "" })).toHaveLength(0); + expect(queryKnowledgePages(db, { query: " " })).toHaveLength(0); + expect(countKnowledgePages(db)).toBe(1); + }); + + it("escapes LIKE wildcards so user input can't widen the match", () => { + upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-1", title: "literal", content: "100% done" }); + // A bare "%" must not match every row; it has no alphanumeric token at all. + expect(queryKnowledgePages(db, { query: "%" })).toHaveLength(0); + // The literal token does match. + expect(queryKnowledgePages(db, { query: "100" })).toHaveLength(1); + }); + + it("incremental refresh updates only the affected page; others keep their timestamps", () => { + const { page: a } = upsertKnowledgePage(db, { + sourceKind: "task", + sourceId: "T-A", + title: "A", + content: "a", + now: "2026-01-01T00:00:00.000Z", + }); + const { page: b } = upsertKnowledgePage(db, { + sourceKind: "task", + sourceId: "T-B", + title: "B", + content: "b", + now: "2026-01-01T00:00:00.000Z", + }); + expect(a.createdAt).toBe("2026-01-01T00:00:00.000Z"); + + // Re-index only T-A at a later time. + const { created, page: aUpdated } = upsertKnowledgePage(db, { + sourceKind: "task", + sourceId: "T-A", + title: "A v2", + content: "a v2", + now: "2026-02-02T00:00:00.000Z", + }); + expect(created).toBe(false); + expect(aUpdated.createdAt).toBe("2026-01-01T00:00:00.000Z"); // createdAt preserved + expect(aUpdated.updatedAt).toBe("2026-02-02T00:00:00.000Z"); // updatedAt advanced + + // T-B is untouched: same updatedAt as when it was created. + const bAfter = getKnowledgePage(db, "task", "T-B"); + expect(bAfter?.updatedAt).toBe(b.updatedAt); + expect(bAfter?.updatedAt).toBe("2026-01-01T00:00:00.000Z"); + // Still exactly two pages — no duplicate created on re-index. + expect(countKnowledgePages(db)).toBe(2); + }); + + // Seed a DB at the PREVIOUS schema version (118), run migrate, assert the + // table exists and SCHEMA_VERSION lands at the highest migration target (119). + // Fresh-DB tests cannot catch the migrate-loop early-return bug this guards. + it("creates knowledge_pages when migrating from the previous schema version", () => { + db.exec("DROP INDEX IF EXISTS idxKnowledgePagesSourceKind"); + db.exec("DROP INDEX IF EXISTS idxKnowledgePagesUpdatedAt"); + db.exec("DROP TABLE IF EXISTS knowledge_pages"); + db.prepare("UPDATE __meta SET value = ? WHERE key = 'schemaVersion'").run(String(SCHEMA_VERSION - 1)); + + (db as unknown as { migrate: () => void }).migrate(); + + const table = db + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='knowledge_pages'") + .get() as { name: string } | undefined; + expect(table?.name).toBe("knowledge_pages"); + expect(db.getSchemaVersion()).toBe(SCHEMA_VERSION); + + // The migrated table is writable and queryable. + upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-mig", title: "migrated", content: "ok" }); + expect(queryKnowledgePages(db, { query: "migrated" })).toHaveLength(1); + }); + + it("SCHEMA_VERSION matches the highest applied migration on a fresh DB", () => { + expect(db.getSchemaVersion()).toBe(SCHEMA_VERSION); + }); +}); + +describe("knowledge-index pure helpers", () => { + it("tokenizeQuery splits on non-word chars and lowercases", () => { + expect(tokenizeQuery("Add OAuth, login-flow!")).toEqual(["add", "oauth", "login", "flow"]); + expect(tokenizeQuery(" ")).toEqual([]); + }); + + it("buildSearchText concatenates and lowercases all fields", () => { + const text = buildSearchText({ title: "Title", summary: "Sum", content: "Body", tags: ["Tag"] }); + expect(text).toBe("title sum body tag"); + }); + + it("renderTaskPage builds a deterministic page from task facts", () => { + const page = renderTaskPage({ + id: "FN-7", + title: "Fix bug", + description: "Null deref in parser", + modifiedFiles: ["src/parser.ts"], + commitSubjects: ["fix: guard null"], + prUrl: "https://example.com/pr/7", + }); + expect(page.sourceKind).toBe("task"); + expect(page.sourceId).toBe("FN-7"); + expect(page.title).toBe("Fix bug"); + expect(page.content).toContain("Null deref in parser"); + expect(page.content).toContain("src/parser.ts"); + expect(page.content).toContain("fix: guard null"); + expect(page.content).toContain("https://example.com/pr/7"); + expect(page.tags).toEqual(["parser.ts"]); + }); + + it("renderTaskPage falls back to a generated title when none is set", () => { + const page = renderTaskPage({ id: "FN-8", description: "", modifiedFiles: [] }); + expect(page.title).toBe("Task FN-8"); + }); +}); + +describe("refreshKnowledgeForTask hook", () => { + let db: Database; + let tmpDir: string; + + function storeFor(database: Database, tasks: Record): TaskStore { + const store = new EventEmitter() as unknown as TaskStore & { + getDatabase(): Database; + getTask(id: string): Promise; + }; + store.getDatabase = () => database; + store.getTask = async (id: string) => tasks[id] ?? null; + return store; + } + + beforeEach(() => { + ({ db, tmpDir } = makeDb()); + }); + + afterEach(() => { + db.close(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("indexes a completed task so it becomes queryable", async () => { + const store = storeFor(db, { + "FN-1": { + id: "FN-1", + title: "Implement retry", + description: "Exponential backoff in client.ts", + modifiedFiles: ["client.ts"], + column: "done", + }, + }); + const page = await refreshKnowledgeForTask(store, "FN-1"); + expect(page?.sourceId).toBe("FN-1"); + expect(queryKnowledgePages(db, { query: "backoff" })).toHaveLength(1); + }); + + it("is fail-soft: returns null for a missing task without throwing", async () => { + const store = storeFor(db, {}); + await expect(refreshKnowledgeForTask(store, "nope")).resolves.toBeNull(); + expect(countKnowledgePages(db)).toBe(0); + }); +}); diff --git a/packages/dashboard/src/__tests__/register-knowledge-routes.auth.test.ts b/packages/dashboard/src/__tests__/register-knowledge-routes.auth.test.ts new file mode 100644 index 0000000000..1d5e8be481 --- /dev/null +++ b/packages/dashboard/src/__tests__/register-knowledge-routes.auth.test.ts @@ -0,0 +1,81 @@ +// @vitest-environment node + +/** + * Auth integration for the knowledge-index endpoints (U14): every endpoint must + * be rejected with 401 when unauthenticated and accepted with a valid bearer + * token. Mirrors `register-command-center-routes.auth.test.ts` — the registrar + * adds no auth of its own; it inherits the server-level middleware, which is + * exactly what this asserts. + */ + +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { EventEmitter } from "node:events"; +import type { Task, TaskStore } from "@fusion/core"; +import { request } from "../test-request.js"; +import { createServer } from "../server.js"; + +vi.mock("@fusion/core", async (importOriginal) => { + const { createCoreMock } = await import("../test/mockCoreEngine.js"); + return createCoreMock(() => importOriginal(), {}); +}); + +class MockStore extends EventEmitter { + getRootDir(): string { + return "/tmp/fn-knowledge-auth-test"; + } + + getFusionDir(): string { + return "/tmp/fn-knowledge-auth-test/.fusion"; + } + + getDatabase() { + return { + exec: vi.fn(), + prepare: vi.fn().mockReturnValue({ + run: vi.fn().mockReturnValue({ changes: 0 }), + get: vi.fn().mockReturnValue({ count: 0 }), + all: vi.fn().mockReturnValue([]), + }), + }; + } + + getDatabaseHealth() { + return { + healthy: true, + corruptionDetected: false, + corruptionErrors: [], + isRunning: false, + lastCheckedAt: null, + }; + } + + async listTasks(): Promise { + return []; + } +} + +const TOKEN = "fn_knowledge_test1234567890abc"; + +describe("Knowledge routes — auth", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("rejects an unauthenticated query with 401", async () => { + const app = createServer(new MockStore() as unknown as TaskStore, { + daemon: { token: TOKEN }, + }); + const res = await request(app, "GET", "/api/knowledge/query?q=anything"); + expect(res.status).toBe(401); + }); + + it("accepts the query with a valid bearer token", async () => { + const app = createServer(new MockStore() as unknown as TaskStore, { + daemon: { token: TOKEN }, + }); + const res = await request(app, "GET", "/api/knowledge/query?q=anything", undefined, { + Authorization: `Bearer ${TOKEN}`, + }); + expect(res.status).toBe(200); + }); +}); diff --git a/packages/dashboard/src/__tests__/register-knowledge-routes.test.ts b/packages/dashboard/src/__tests__/register-knowledge-routes.test.ts new file mode 100644 index 0000000000..ccba544a60 --- /dev/null +++ b/packages/dashboard/src/__tests__/register-knowledge-routes.test.ts @@ -0,0 +1,185 @@ +// @vitest-environment node + +import express, { type NextFunction, type Request, type Response } from "express"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { EventEmitter } from "node:events"; + +import { Database } from "@fusion/core"; +import type { TaskStore } from "@fusion/core"; +import { request } from "../test-request.js"; +import { ApiError } from "../api-error.js"; +import { registerKnowledgeRoutes } from "../routes/register-knowledge-routes.js"; +import { upsertKnowledgePage } from "../knowledge-index.js"; +import type { ApiRoutesContext } from "../routes/types.js"; + +interface QueryResponse { + query: string; + pages: Array<{ sourceId: string }>; + total: number; +} +interface RefreshResponse { + page: { sourceId: string }; +} + +/** POST JSON helper over the bare `request` (which only accepts string bodies). */ +function postJson( + app: ReturnType, + path: string, + body: unknown, +): ReturnType { + return request(app, "POST", path, JSON.stringify(body), { + "content-type": "application/json", + }); +} + +/** A minimal TaskStore exposing getDatabase()/getTask(), which is all routes use. */ +function storeFor(db: Database, tasks: Record = {}): TaskStore { + const store = new EventEmitter() as unknown as TaskStore & { + getDatabase(): Database; + getTask(id: string): Promise; + }; + store.getDatabase = () => db; + store.getTask = async (id: string) => tasks[id] ?? null; + return store; +} + +function buildApp(stores: Record, fallback: TaskStore) { + const app = express(); + app.use(express.json()); + const router = express.Router(); + const ctx = { + router, + getScopedStore: async (req: Request): Promise => { + const projectId = typeof req.query.projectId === "string" ? req.query.projectId : undefined; + return projectId && stores[projectId] ? stores[projectId] : fallback; + }, + rethrowAsApiError: (error: unknown, fallbackMessage?: string): never => { + if (error instanceof ApiError) throw error; + throw new ApiError(500, fallbackMessage ?? "Internal error"); + }, + } as unknown as ApiRoutesContext; + registerKnowledgeRoutes(ctx); + app.use("/api", router); + app.use((err: unknown, _req: Request, res: Response, _next: NextFunction) => { + if (err instanceof ApiError) { + res.status(err.statusCode).json({ error: err.message }); + return; + } + res.status(500).json({ error: "Internal error" }); + }); + return app; +} + +describe("register-knowledge-routes", () => { + let tmpDir: string; + let dbA: Database; + let dbB: Database; + let app: ReturnType; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "kb-knowledge-routes-")); + dbA = new Database(join(tmpDir, "a", ".fusion")); + dbA.init(); + dbB = new Database(join(tmpDir, "b", ".fusion")); + dbB.init(); + + upsertKnowledgePage(dbA, { + sourceKind: "task", + sourceId: "FN-A1", + title: "Add OAuth login flow", + content: "Implemented oauth login with token refresh in auth.ts", + tags: ["auth.ts"], + }); + upsertKnowledgePage(dbB, { + sourceKind: "task", + sourceId: "FN-B1", + title: "Secret project-B widget", + content: "Project B only — confidential widget rendering", + tags: ["widget.ts"], + }); + + const storeA = storeFor(dbA, { + "FN-A2": { + id: "FN-A2", + title: "Refactor payment module", + description: "Cleaned up the stripe payment handler", + modifiedFiles: ["payment.ts"], + column: "done", + }, + }); + const storeB = storeFor(dbB); + app = buildApp({ "proj-a": storeA, "proj-b": storeB }, storeA); + }); + + afterEach(() => { + dbA.close(); + dbB.close(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("returns relevant pages for a keyword query (fixture)", async () => { + const res = await request(app, "GET", "/api/knowledge/query?q=oauth&projectId=proj-a"); + expect(res.status).toBe(200); + const body = res.body as QueryResponse; + expect(body.pages).toHaveLength(1); + expect(body.pages[0].sourceId).toBe("FN-A1"); + expect(body.total).toBe(1); + }); + + it("returns empty for a non-matching keyword", async () => { + const res = await request(app, "GET", "/api/knowledge/query?q=kubernetes&projectId=proj-a"); + expect(res.status).toBe(200); + expect((res.body as QueryResponse).pages).toHaveLength(0); + }); + + it("returns empty for a blank query rather than the whole index", async () => { + const res = await request(app, "GET", "/api/knowledge/query?q=&projectId=proj-a"); + expect(res.status).toBe(200); + const body = res.body as QueryResponse; + expect(body.pages).toHaveLength(0); + expect(body.total).toBe(1); + }); + + it("project scoping — project-A query cannot read project-B pages", async () => { + // The project-B-only term must never surface for project A. + const leak = await request(app, "GET", "/api/knowledge/query?q=widget&projectId=proj-a"); + expect(leak.status).toBe(200); + expect((leak.body as QueryResponse).pages).toHaveLength(0); + + // ...but is visible to project B. + const ok = await request(app, "GET", "/api/knowledge/query?q=widget&projectId=proj-b"); + expect(ok.status).toBe(200); + const okBody = ok.body as QueryResponse; + expect(okBody.pages).toHaveLength(1); + expect(okBody.pages[0].sourceId).toBe("FN-B1"); + }); + + it("POST /refresh incrementally indexes a completed task, then it is queryable", async () => { + const refresh = await postJson(app, "/api/knowledge/refresh?projectId=proj-a", { + taskId: "FN-A2", + }); + expect(refresh.status).toBe(200); + expect((refresh.body as RefreshResponse).page.sourceId).toBe("FN-A2"); + + const q = await request(app, "GET", "/api/knowledge/query?q=stripe&projectId=proj-a"); + expect(q.status).toBe(200); + const qBody = q.body as QueryResponse; + expect(qBody.pages).toHaveLength(1); + expect(qBody.pages[0].sourceId).toBe("FN-A2"); + }); + + it("POST /refresh returns 404 for an unknown task", async () => { + const res = await postJson(app, "/api/knowledge/refresh?projectId=proj-a", { + taskId: "does-not-exist", + }); + expect(res.status).toBe(404); + }); + + it("POST /refresh requires a taskId", async () => { + const res = await postJson(app, "/api/knowledge/refresh?projectId=proj-a", {}); + expect(res.status).toBe(400); + }); +}); diff --git a/packages/dashboard/src/index.ts b/packages/dashboard/src/index.ts index c7a64658d0..3edf33a469 100644 --- a/packages/dashboard/src/index.ts +++ b/packages/dashboard/src/index.ts @@ -37,6 +37,23 @@ export { rateLimit, RATE_LIMITS, type RateLimitOptions } from "./rate-limit.js"; export { GitHubPollingService, type GitHubPollingServiceOptions, type TaskWatchInput, type WatchedBadgeType } from "./github-poll.js"; export { GitHubIssueCommentService, DEFAULT_COMMENT_TEMPLATE } from "./github-issue-comment.js"; export { GitHubSourceIssueCloseService } from "./github-source-issue-close.js"; +export { + upsertKnowledgePage, + queryKnowledgePages, + getKnowledgePage, + countKnowledgePages, + refreshKnowledgeForTask, + renderTaskPage, + buildSearchText, + tokenizeQuery, + KNOWLEDGE_QUERY_DEFAULT_LIMIT, + KNOWLEDGE_QUERY_MAX_LIMIT, + type KnowledgePage, + type KnowledgePageInput, + type KnowledgeSourceKind, + type KnowledgeQueryOptions, +} from "./knowledge-index.js"; +export { KnowledgeIndexRefreshService } from "./knowledge-index-refresh.js"; export { GitHubTrackingCommentService, formatTrackingComment } from "./github-tracking-comments.js"; export { GitHubTrackingStateService, decideIssueAction } from "./github-tracking-state.js"; export { GitHubTrackingReconciler, RECONCILE_CONCURRENCY_LIMIT, RECONCILE_SCAN_LIMIT } from "./github-tracking-reconciler.js"; diff --git a/packages/dashboard/src/knowledge-index-refresh.ts b/packages/dashboard/src/knowledge-index-refresh.ts new file mode 100644 index 0000000000..5bc000e1d6 --- /dev/null +++ b/packages/dashboard/src/knowledge-index-refresh.ts @@ -0,0 +1,67 @@ +import type { TaskStore } from "@fusion/core"; +import { refreshKnowledgeForTask } from "./knowledge-index.js"; + +/** + * Task-completion refresh hook for the persistent knowledge index (U14). + * + * Listens for `task:moved` and, when a task reaches `done`, incrementally + * re-indexes just that task as a knowledge page (one upsert, never a full + * re-index). Mirrors the attach/detach/start/stop lifecycle of + * `GitHubSourceIssueCloseService` so it can be wired the same way alongside the + * other `task:moved` listeners. All refresh work is fail-soft (see + * {@link refreshKnowledgeForTask}) so it can never disrupt task completion. + */ +interface TaskMovedEvent { + task: { id: string }; + // store's `task:moved` carries `ColumnId`; this handler only literal-compares + // legacy ids, so the widened string field is safe. + from: string; + to: string; +} + +export class KnowledgeIndexRefreshService { + private readonly defaultStore: TaskStore; + private readonly listeners = new Map void }>(); + private started = false; + + constructor(store: TaskStore) { + this.defaultStore = store; + } + + start(): void { + if (this.started) return; + this.started = true; + this.attach(this.defaultStore); + } + + stop(): void { + if (!this.started) return; + this.started = false; + for (const store of this.listeners.keys()) { + this.detach(store); + } + } + + attach(store: TaskStore): void { + if (this.listeners.has(store)) return; + const onTaskMoved = (event: TaskMovedEvent): void => { + void this.handleTaskMoved(store, event); + }; + this.listeners.set(store, { onTaskMoved }); + if (this.started) { + store.on("task:moved", onTaskMoved); + } + } + + detach(store: TaskStore): void { + const handlers = this.listeners.get(store); + if (!handlers) return; + store.off("task:moved", handlers.onTaskMoved); + this.listeners.delete(store); + } + + private async handleTaskMoved(store: TaskStore, event: TaskMovedEvent): Promise { + if (event.to !== "done") return; + await refreshKnowledgeForTask(store, event.task.id); + } +} diff --git a/packages/dashboard/src/knowledge-index.ts b/packages/dashboard/src/knowledge-index.ts new file mode 100644 index 0000000000..f9db93932b --- /dev/null +++ b/packages/dashboard/src/knowledge-index.ts @@ -0,0 +1,386 @@ +/** + * Persistent knowledge index (U14). + * + * A persistent, incrementally-refreshed knowledge layer that downstream agents + * can query. Each "page" captures the durable, queryable summary of one source + * (currently one page per completed task; PR-history pages share the same row + * shape). Pages are stored in the `knowledge_pages` SQLite table (schema + + * migration 119 in `packages/core/src/db.ts`). + * + * ## Delta over `insights` / `memoryView` + * + * This is intentionally NOT a second copy of the existing surfaces: + * + * - `InsightStore` / `insights-routes.ts` / `InsightsView` store **LLM-extracted + * durable project learnings** ("patterns/principles/pitfalls" mined from + * working memory by an agent run). `memoryView` renders the freeform working/ + * insights **markdown memory files**. Both are *interpretation* layers and both + * require a model run to populate. + * - The knowledge index is a **deterministic, model-free, keyword-searchable + * index of concrete task/PR history** (title, description, modified files, + * commits, PR links). It is refreshed **incrementally on task completion** — + * one upsert per affected page, never a full re-index — and exposes a plain + * keyword **query API** an agent can call to recall "what work touched X". + * + * So the genuinely new capability is: (1) a persistent per-task/PR page store, + * (2) an incremental refresh hook on task completion, and (3) a keyword query + * API — none of which the insights/memory surfaces provide. + * + * ## Search + * + * Matching is plain keyword `LIKE` over a denormalized lowercased `searchText` + * column (AND-of-terms), deliberately avoiding SQLite FTS5 — FTS5 is not + * available on every SQLite build the engine runs on (see `probeFts5` in + * `db.ts`), and the plan scopes this unit to "SQLite full-text/keyword search, + * NOT an external embedding API". + * + * ## Security + * + * The query API is registered as an {@link ApiRouteRegistrar} (see + * `routes/register-knowledge-routes.ts`) so it inherits the dashboard's standard + * session/auth middleware AND resolves the database through `getScopedStore(req)` + * before reading — exactly like U9. The index holds sensitive repo/commit/PR + * content, so it is an information-disclosure surface, never an open endpoint. + */ + +import type { Database, TaskStore } from "@fusion/core"; + +/** The kind of source a knowledge page was indexed from. */ +export type KnowledgeSourceKind = "task" | "pr"; + +/** A knowledge page row as stored/read from `knowledge_pages`. */ +export interface KnowledgePage { + id: number; + sourceKind: KnowledgeSourceKind; + sourceId: string; + /** Stable dedupe key (`:`); upserts target this. */ + sourceKey: string; + title: string; + summary: string | null; + content: string; + tags: string[]; + createdAt: string; + updatedAt: string; +} + +/** Input for {@link upsertKnowledgePage}. */ +export interface KnowledgePageInput { + sourceKind: KnowledgeSourceKind; + sourceId: string; + title: string; + summary?: string | null; + content: string; + tags?: string[]; + /** Injectable clock for deterministic tests. Defaults to now. */ + now?: string; +} + +interface KnowledgePageRow { + id: number; + sourceKind: string; + sourceId: string; + sourceKey: string; + title: string; + summary: string | null; + content: string; + tags: string | null; + searchText: string; + createdAt: string; + updatedAt: string; +} + +/** Maximum number of pages a single keyword query returns. */ +export const KNOWLEDGE_QUERY_DEFAULT_LIMIT = 20; +export const KNOWLEDGE_QUERY_MAX_LIMIT = 100; + +function sourceKeyFor(kind: KnowledgeSourceKind, id: string): string { + return `${kind}:${id}`; +} + +function rowToPage(row: KnowledgePageRow): KnowledgePage { + let tags: string[] = []; + if (row.tags) { + try { + const parsed = JSON.parse(row.tags) as unknown; + if (Array.isArray(parsed)) tags = parsed.filter((t): t is string => typeof t === "string"); + } catch { + tags = []; + } + } + return { + id: row.id, + sourceKind: row.sourceKind as KnowledgeSourceKind, + sourceId: row.sourceId, + sourceKey: row.sourceKey, + title: row.title, + summary: row.summary, + content: row.content, + tags, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + +/** + * Build the denormalized, lowercased search blob a page is matched against. + * Pure so it can be unit-tested independently of the DB. + */ +export function buildSearchText(input: { + title: string; + summary?: string | null; + content: string; + tags?: string[]; +}): string { + return [ + input.title, + input.summary ?? "", + input.content, + (input.tags ?? []).join(" "), + ] + .join(" ") + .toLowerCase(); +} + +/** + * Tokenize a free-text query into lowercased keyword terms. Empty / whitespace + * input yields no terms (callers treat that as "match nothing", not "match all", + * to avoid returning the whole sensitive index for a blank query). + */ +export function tokenizeQuery(query: string): string[] { + return query + .toLowerCase() + .split(/[^a-z0-9_]+/i) + .map((t) => t.trim()) + .filter((t) => t.length > 0); +} + +/** + * Insert or update a knowledge page, keyed by `(sourceKind, sourceId)`. + * + * **Incremental by construction:** only the row for this source is touched, so a + * refresh of one task never rewrites (or re-timestamps) any other page. On an + * update, `createdAt` is preserved and only `updatedAt` advances. + * + * @returns the upserted page and whether it was newly created. + */ +export function upsertKnowledgePage( + db: Database, + input: KnowledgePageInput, +): { page: KnowledgePage; created: boolean } { + const now = input.now ?? new Date().toISOString(); + const sourceKey = sourceKeyFor(input.sourceKind, input.sourceId); + const tags = input.tags ?? []; + const searchText = buildSearchText({ + title: input.title, + summary: input.summary, + content: input.content, + tags, + }); + const tagsJson = JSON.stringify(tags); + + const existing = db + .prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?") + .get(sourceKey) as KnowledgePageRow | undefined; + + if (existing) { + db.prepare( + `UPDATE knowledge_pages + SET title = ?, summary = ?, content = ?, tags = ?, searchText = ?, updatedAt = ? + WHERE sourceKey = ?`, + ).run(input.title, input.summary ?? null, input.content, tagsJson, searchText, now, sourceKey); + const updated = db + .prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?") + .get(sourceKey) as KnowledgePageRow; + return { page: rowToPage(updated), created: false }; + } + + db.prepare( + `INSERT INTO knowledge_pages + (sourceKind, sourceId, sourceKey, title, summary, content, tags, searchText, createdAt, updatedAt) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + input.sourceKind, + input.sourceId, + sourceKey, + input.title, + input.summary ?? null, + input.content, + tagsJson, + searchText, + now, + now, + ); + const inserted = db + .prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?") + .get(sourceKey) as KnowledgePageRow; + return { page: rowToPage(inserted), created: true }; +} + +/** Fetch a single page by its source identity, or `undefined`. */ +export function getKnowledgePage( + db: Database, + sourceKind: KnowledgeSourceKind, + sourceId: string, +): KnowledgePage | undefined { + const row = db + .prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?") + .get(sourceKeyFor(sourceKind, sourceId)) as KnowledgePageRow | undefined; + return row ? rowToPage(row) : undefined; +} + +/** Options for {@link queryKnowledgePages}. */ +export interface KnowledgeQueryOptions { + query: string; + sourceKind?: KnowledgeSourceKind; + limit?: number; +} + +/** + * Keyword search the index. Returns pages whose `searchText` contains **all** + * query terms (AND), most-recently-updated first. A blank/termless query returns + * an empty list rather than the whole index. + */ +export function queryKnowledgePages(db: Database, options: KnowledgeQueryOptions): KnowledgePage[] { + const terms = tokenizeQuery(options.query); + if (terms.length === 0) return []; + + const limit = Math.min( + Math.max(1, options.limit ?? KNOWLEDGE_QUERY_DEFAULT_LIMIT), + KNOWLEDGE_QUERY_MAX_LIMIT, + ); + + const clauses: string[] = []; + const params: string[] = []; + for (const term of terms) { + clauses.push("searchText LIKE ? ESCAPE '\\'"); + params.push(`%${escapeLike(term)}%`); + } + if (options.sourceKind) { + clauses.push("sourceKind = ?"); + params.push(options.sourceKind); + } + + const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""; + const rows = db + .prepare(`SELECT * FROM knowledge_pages ${where} ORDER BY updatedAt DESC, id DESC LIMIT ?`) + .all(...params, limit) as KnowledgePageRow[]; + return rows.map(rowToPage); +} + +/** Escape SQLite `LIKE` wildcards in a term so user input can't inject them. */ +function escapeLike(term: string): string { + return term.replace(/[\\%_]/g, (ch) => `\\${ch}`); +} + +/** Total number of pages in the index. */ +export function countKnowledgePages(db: Database): number { + const row = db.prepare("SELECT COUNT(*) AS count FROM knowledge_pages").get() as { count: number }; + return row.count; +} + +/** + * Render a completed task into a deterministic knowledge page body. Pure so the + * refresh hook is testable without a real store. Concatenates the durable, + * non-sensitive facts: title, description, modified files, associated commit + * subjects, and PR link if present. + */ +export function renderTaskPage(task: { + id: string; + title?: string; + description?: string; + modifiedFiles?: string[]; + commitSubjects?: string[]; + prUrl?: string | null; + column?: string; +}): KnowledgePageInput { + const title = (task.title ?? "").trim() || `Task ${task.id}`; + const lines: string[] = []; + if (task.description?.trim()) { + lines.push(task.description.trim()); + } + if (task.modifiedFiles && task.modifiedFiles.length > 0) { + lines.push(`Files: ${task.modifiedFiles.join(", ")}`); + } + if (task.commitSubjects && task.commitSubjects.length > 0) { + lines.push(`Commits:\n${task.commitSubjects.map((s) => `- ${s}`).join("\n")}`); + } + if (task.prUrl) { + lines.push(`PR: ${task.prUrl}`); + } + const tags = (task.modifiedFiles ?? []) + .map((f) => f.split("/").pop() ?? f) + .filter((t) => t.length > 0); + return { + sourceKind: "task", + sourceId: task.id, + title, + summary: task.description?.trim().slice(0, 280) || null, + content: lines.join("\n\n") || title, + tags, + }; +} + +/** + * Incremental refresh hook: index (or re-index) a single task as a knowledge + * page. Intended to be invoked from the task-completion path (or by code that + * observes a task reaching `done`). It reads only the one task and upserts only + * its page, so unaffected pages are never touched. + * + * **Fail-soft:** any read/write error is logged and swallowed so a knowledge + * refresh can never break the task-completion flow that called it. + * + * @returns the upserted page, or `null` if the task could not be loaded/indexed. + */ +export async function refreshKnowledgeForTask( + store: TaskStore, + taskId: string, + options?: { now?: string }, +): Promise { + try { + const detail = await store.getTask(taskId); + if (!detail) return null; + + let commitSubjects: string[] = []; + try { + const lineageId = (detail as { lineageId?: string }).lineageId ?? detail.id; + const rows = store + .getDatabase() + .prepare( + "SELECT commitSubject FROM task_commit_associations WHERE taskLineageId = ? ORDER BY authoredAt ASC", + ) + .all(lineageId) as Array<{ commitSubject: string }>; + commitSubjects = rows.map((r) => r.commitSubject); + } catch { + commitSubjects = []; + } + + const prUrl = extractPrUrl(detail); + const input = renderTaskPage({ + id: detail.id, + title: detail.title, + description: detail.description, + modifiedFiles: (detail as { modifiedFiles?: string[] }).modifiedFiles, + commitSubjects, + prUrl, + column: detail.column, + }); + if (options?.now) input.now = options.now; + + const { page } = upsertKnowledgePage(store.getDatabase(), input); + return page; + } catch (err) { + console.warn(`[knowledge-index] refresh skipped for task ${taskId}:`, err); + return null; + } +} + +/** Best-effort extraction of a PR URL from a task detail, tolerant of shape. */ +function extractPrUrl(detail: unknown): string | null { + if (!detail || typeof detail !== "object") return null; + const d = detail as Record; + if (typeof d.prUrl === "string" && d.prUrl) return d.prUrl; + const pr = d.pullRequest as Record | undefined; + if (pr && typeof pr.url === "string" && pr.url) return pr.url; + if (pr && typeof pr.htmlUrl === "string" && pr.htmlUrl) return pr.htmlUrl; + return null; +} diff --git a/packages/dashboard/src/routes.ts b/packages/dashboard/src/routes.ts index 3b8a903a0e..4a1d627aec 100644 --- a/packages/dashboard/src/routes.ts +++ b/packages/dashboard/src/routes.ts @@ -169,6 +169,7 @@ import { registerModelRoutes } from "./routes/register-model-routes.js"; import { registerCustomProviderRoutes } from "./routes/register-custom-provider-routes.js"; import { registerUsageRoutes } from "./routes/register-usage-routes.js"; import { registerCommandCenterRoutes } from "./routes/register-command-center-routes.js"; +import { registerKnowledgeRoutes } from "./routes/register-knowledge-routes.js"; import { registerSignalRoutes } from "./routes/register-signal-routes.js"; import { registerAuthRoutes } from "./routes/register-auth-routes.js"; import { registerRuntimeProviderRoutes } from "./routes/register-runtime-provider-routes.js"; @@ -1994,6 +1995,11 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout // U9 — Command Center analytics + live snapshot endpoints. Thin adapters over // the core aggregators; inherit standard auth + getScopedStore project scoping. registerCommandCenterRoutes(routeContext); + // U14 — persistent knowledge index query + incremental-refresh endpoints. + // Inherit standard auth + getScopedStore project scoping (same as U9); the + // index holds sensitive repo/PR content so no endpoint is unauthenticated or + // cross-project readable. + registerKnowledgeRoutes(routeContext); // U11 — inbound external signal webhooks (Sentry/Datadog/PagerDuty/generic). // Each route HMAC-verifies against a per-provider secret; never an // unauthenticated task-creation endpoint. diff --git a/packages/dashboard/src/routes/register-git-github.ts b/packages/dashboard/src/routes/register-git-github.ts index 123f1b1d0e..df4b4c0e09 100644 --- a/packages/dashboard/src/routes/register-git-github.ts +++ b/packages/dashboard/src/routes/register-git-github.ts @@ -43,6 +43,7 @@ import { GitHubTrackingCommentService } from "../github-tracking-comments.js"; import { GitHubTrackingStateService } from "../github-tracking-state.js"; import { GitHubTrackingReconciler, RECONCILE_SCAN_LIMIT } from "../github-tracking-reconciler.js"; import { GitHubSourceIssueCloseService } from "../github-source-issue-close.js"; +import { KnowledgeIndexRefreshService } from "../knowledge-index-refresh.js"; import { githubRateLimiter } from "../github-poll.js"; import * as projectStoreResolver from "../project-store-resolver.js"; import { generatePrMetadata } from "../pr-metadata-generator.js"; @@ -2485,6 +2486,12 @@ export function registerGitGitHubRoutes(ctx: ApiRoutesContext): void { githubSourceIssueCloseService.start(); ctx.registerDispose(() => githubSourceIssueCloseService.stop()); + // U14 — incremental knowledge-index refresh on task completion. Listens for + // task:moved → done and re-indexes just that task as a knowledge page. + const knowledgeIndexRefreshService = new KnowledgeIndexRefreshService(store); + knowledgeIndexRefreshService.start(); + ctx.registerDispose(() => knowledgeIndexRefreshService.stop()); + const githubTrackingStateService = new GitHubTrackingStateService(store); const githubTrackingReconciler = new GitHubTrackingReconciler(); const reconcileScheduledStores = new WeakSet(); diff --git a/packages/dashboard/src/routes/register-knowledge-routes.ts b/packages/dashboard/src/routes/register-knowledge-routes.ts new file mode 100644 index 0000000000..07b870e77a --- /dev/null +++ b/packages/dashboard/src/routes/register-knowledge-routes.ts @@ -0,0 +1,95 @@ +import { ApiError } from "../api-error.js"; +import { + queryKnowledgePages, + countKnowledgePages, + refreshKnowledgeForTask, + KNOWLEDGE_QUERY_DEFAULT_LIMIT, + KNOWLEDGE_QUERY_MAX_LIMIT, + type KnowledgeSourceKind, +} from "../knowledge-index.js"; +import type { ApiRouteRegistrar } from "./types.js"; + +/** + * Persistent knowledge-index API (U14). + * + * Thin HTTP adapter over the keyword index in `knowledge-index.ts`. Downstream + * agents call `GET /api/knowledge/query` to recall task/PR history. + * + * Security (same contract as U9 — `register-command-center-routes.ts`): + * - Every route inherits the dashboard's standard session/auth middleware via + * the {@link ApiRouteRegistrar} contract, so an unauthenticated request is + * rejected with 401 by the server-level auth middleware before reaching these + * handlers. No knowledge endpoint is unauthenticated. + * - Every endpoint resolves the database through `getScopedStore(req)` before + * reading/writing, so a project-A caller can never read project-B pages. The + * index holds sensitive repo/commit/PR content, so it is an information- + * disclosure surface, not an open endpoint. + */ + +const VALID_SOURCE_KINDS: ReadonlySet = new Set(["task", "pr"]); + +function resolveSourceKind(query: { sourceKind?: unknown }): KnowledgeSourceKind | undefined { + const raw = typeof query.sourceKind === "string" ? query.sourceKind : undefined; + return raw !== undefined && VALID_SOURCE_KINDS.has(raw) + ? (raw as KnowledgeSourceKind) + : undefined; +} + +function resolveLimit(query: { limit?: unknown }): number { + const raw = typeof query.limit === "string" ? Number.parseInt(query.limit, 10) : NaN; + if (!Number.isFinite(raw)) return KNOWLEDGE_QUERY_DEFAULT_LIMIT; + return Math.min(Math.max(1, raw), KNOWLEDGE_QUERY_MAX_LIMIT); +} + +export const registerKnowledgeRoutes: ApiRouteRegistrar = (ctx) => { + const { router, getScopedStore, rethrowAsApiError } = ctx; + + /** + * GET /api/knowledge/query?q=&sourceKind=task|pr&limit=N + * Keyword search over the project-scoped knowledge index. Returns the matching + * pages (most-recently-updated first) and the total index size. + */ + router.get("/knowledge/query", async (req, res) => { + try { + const store = await getScopedStore(req); + const q = typeof req.query.q === "string" ? req.query.q : ""; + const pages = queryKnowledgePages(store.getDatabase(), { + query: q, + sourceKind: resolveSourceKind(req.query), + limit: resolveLimit(req.query), + }); + res.json({ + query: q, + pages, + total: countKnowledgePages(store.getDatabase()), + }); + } catch (err: unknown) { + if (err instanceof ApiError) throw err; + rethrowAsApiError(err, "Failed to query knowledge index"); + } + }); + + /** + * POST /api/knowledge/refresh { taskId } + * Incrementally re-index a single task as a knowledge page. Exposes the + * task-completion refresh hook over HTTP so the completion path (or an + * operator) can trigger an incremental refresh without a full re-index. + */ + router.post("/knowledge/refresh", async (req, res) => { + try { + const store = await getScopedStore(req); + const taskId = typeof req.body?.taskId === "string" ? req.body.taskId.trim() : ""; + if (!taskId) { + throw new ApiError(400, "taskId is required"); + } + const page = await refreshKnowledgeForTask(store, taskId); + if (!page) { + throw new ApiError(404, `Task not found or could not be indexed: ${taskId}`); + } + res.json({ page }); + } catch (err: unknown) { + if (err instanceof ApiError) throw err; + rethrowAsApiError(err, "Failed to refresh knowledge index"); + } + }); +}; diff --git a/plugins/fusion-plugin-roadmap/src/store/__tests__/roadmap-store.test.ts b/plugins/fusion-plugin-roadmap/src/store/__tests__/roadmap-store.test.ts index 48bd180997..7365b0d724 100644 --- a/plugins/fusion-plugin-roadmap/src/store/__tests__/roadmap-store.test.ts +++ b/plugins/fusion-plugin-roadmap/src/store/__tests__/roadmap-store.test.ts @@ -743,10 +743,10 @@ describe("RoadmapStore", () => { }); describe("schema version", () => { - it("schema version is 118 after init", () => { + it("schema version is 119 after init", () => { // Tracks @fusion/core's SCHEMA_VERSION (the roadmap store layers on core's // Database). Bump this in lockstep when core adds a migration. - expect(db.getSchemaVersion()).toBe(118); + expect(db.getSchemaVersion()).toBe(119); }); });