feat(knowledge): U14 — persistent knowledge index

Adds knowledge_pages (db migration 118→119) + a deterministic, model-free
keyword index of task/PR history in packages/dashboard/src, incrementally
refreshed on task completion (task:moved→done listener) and queryable via an
auth-gated, project-scoped API. Complements the LLM-extracted insights/memory
surfaces rather than duplicating them.

Follow-ups: no React view yet; PR-history page population attaches via U18.
This commit is contained in:
gsxdsm
2026-06-15 20:34:52 -07:00
parent 168dc2f796
commit 0a87890bd8
22 changed files with 1208 additions and 56 deletions

View File

@@ -0,0 +1,10 @@
---
"@runfusion/fusion": minor
---
Add a persistent, incrementally-refreshed knowledge index (U14) downstream agents can query.
- **Schema** — new `knowledge_pages` SQLite table (`packages/core/src/db.ts`) with `SCHEMA_VERSION` bumped 118 → 119 (added in the same change as the migration; the fingerprint auto-covers SCHEMA_SQL tables). Keyword search uses a denormalized lowercased `searchText` column with AND-of-terms `LIKE` matching, deliberately avoiding SQLite FTS5 (not available on every build) and any external embedding API.
- **Index module** (`packages/dashboard/src/knowledge-index.ts`) — upsert-by-source-key pages, a model-free keyword query API, and `refreshKnowledgeForTask` that re-indexes a single completed task (one upsert, never a full re-index, so unaffected pages keep their timestamps). This is the delta over the existing `insights`/`memoryView` surfaces, which are LLM-extracted learnings, not a deterministic searchable index of concrete task/PR history.
- **Refresh hook** — `KnowledgeIndexRefreshService` listens for `task:moved → done` (mirroring `GitHubSourceIssueCloseService`) and is wired alongside the other completion listeners; fail-soft so it can never disrupt task completion.
- **Query API** (`register-knowledge-routes.ts`) — `GET /api/knowledge/query` and `POST /api/knowledge/refresh`, registered as an `ApiRouteRegistrar` so they inherit the dashboard's standard session/auth middleware (401 when unauthenticated) and apply `getScopedStore(req)` (no cross-project reads), exactly like U9.

View File

@@ -715,7 +715,7 @@ describe("schema migration", () => {
const row = db.prepare("SELECT deletedAt FROM tasks WHERE id = 'FN-legacy'").get() as { deletedAt: string | null };
expect(row.deletedAt).toBeNull();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -748,7 +748,7 @@ describe("schema migration", () => {
{ id: "WS-001", mode: "prompt", gateMode: "advisory" },
{ id: "WS-002", mode: "script", gateMode: "advisory" },
]);
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -798,7 +798,7 @@ describe("schema migration", () => {
reviewerContextRetryCount: 0,
reviewerFallbackRetryCount: 0,
});
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -827,7 +827,7 @@ describe("schema migration", () => {
const columns = db.prepare("PRAGMA table_info(milestones)").all() as Array<{ name: string }>;
expect(columns.map((column) => column.name)).toContain("acceptanceCriteria");
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -868,7 +868,7 @@ describe("schema migration", () => {
const missionColumns = db.prepare("PRAGMA table_info(missions)").all() as Array<{ name: string }>;
expect(missionColumns.map((column) => column.name)).toContain("autoMerge");
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -902,7 +902,7 @@ describe("schema migration", () => {
{ id: "WS-002", mode: "script", enabled: 1, gateMode: "advisory" },
{ id: "WS-003", mode: "prompt", enabled: 0, gateMode: "advisory" },
]);
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -939,7 +939,7 @@ describe("schema migration", () => {
const indexes = db.prepare("PRAGMA index_list(mission_goals)").all() as Array<{ name: string }>;
expect(indexes.some((index) => index.name === "idxMissionGoalsGoalId")).toBe(true);
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -1000,7 +1000,7 @@ describe("schema migration", () => {
expect(customFieldsColumn).toBeDefined();
expect(customFieldsColumn?.dflt_value).toBe("'{}'");
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -1038,7 +1038,7 @@ describe("schema migration", () => {
const indexes = db.prepare("PRAGMA index_list(workflow_settings)").all() as Array<{ name: string }>;
expect(indexes.some((index) => index.name === "idx_workflow_settings_project")).toBe(true);
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -1120,7 +1120,7 @@ describe("schema migration", () => {
expect(indexNames).toContain("idx_cli_sessions_chatSessionId");
expect(indexNames).toContain("idx_cli_sessions_project_state");
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -1152,7 +1152,7 @@ describe("schema migration", () => {
.all() as Array<{ name: string }>;
expect(columns.map((column) => column.name)).toContain("cliExecutorAdapterId");
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -1162,7 +1162,7 @@ describe("schema migration", () => {
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as Array<{ name: string }>;
expect(tables.map((row) => row.name)).toContain("cli_sessions");
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -1219,20 +1219,20 @@ describe("schema migration", () => {
.get() as { migrated_fragment_id: string | null };
expect(stepRow.migrated_fragment_id).toBeNull();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
it("migration 109 is idempotent on re-init", () => {
const db = new Database(fusionDir);
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
// Re-open the same on-disk DB: already at 109, the 109 block must be a no-op.
const reopened = new Database(fusionDir);
reopened.init();
expect(reopened.getSchemaVersion()).toBe(118);
expect(reopened.getSchemaVersion()).toBe(119);
const workflowColumns = reopened.prepare("PRAGMA table_info(workflows)").all() as Array<{ name: string }>;
expect(workflowColumns.filter((c) => c.name === "kind")).toHaveLength(1);
const stepColumns = reopened.prepare("PRAGMA table_info(workflow_steps)").all() as Array<{ name: string }>;

View File

@@ -334,7 +334,7 @@ describe("Database", () => {
});
it("seeds schema version", () => {
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
});
it("includes tokenUsageCacheWriteTokens on freshly initialized tasks table", () => {
@@ -393,7 +393,7 @@ describe("Database", () => {
it("is idempotent - calling init() twice does not fail", () => {
expect(() => db.init()).not.toThrow();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
});
it("does not overwrite existing config on re-init", () => {
// Update the config
@@ -1463,7 +1463,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 29 (includes v1→v2 through v26→v29)
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -1488,15 +1488,15 @@ describe("schema migrations", () => {
const db = new Database(fusionDir);
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
// Re-init should not fail
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
// Re-init should not fail
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
db.close();
});
@@ -1531,7 +1531,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
expect(cols.map((col) => col.name)).toContain("priority");
@@ -1572,7 +1572,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
const colNames = cols.map((col) => col.name);
@@ -1644,7 +1644,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
const colNames = cols.map((col) => col.name);
@@ -1884,7 +1884,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
const cols = db.prepare("PRAGMA table_info(chat_messages)").all() as Array<{ name: string }>;
expect(cols.map((col) => col.name)).toContain("attachments");
@@ -1958,7 +1958,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'agentRatings'").all() as Array<{ name: string }>;
expect(tables).toEqual([{ name: "agentRatings" }]);
@@ -1982,7 +1982,7 @@ describe("schema migrations", () => {
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'mission_events'").all() as Array<{ name: string }>;
expect(tables).toEqual([{ name: "mission_events" }]);
@@ -2086,7 +2086,7 @@ describe("schema migrations", () => {
db.init();
// Verify version bumped to 29
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
// Verify new columns exist and existing data is intact
const cols = db.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
@@ -2305,7 +2305,7 @@ describe("schema migrations", () => {
localDb.init();
expect(localDb.getSchemaVersion()).toBe(118);
expect(localDb.getSchemaVersion()).toBe(119);
const columns = localDb.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
expect(columns.map((column) => column.name)).toContain("tokenUsageCacheWriteTokens");
@@ -2616,7 +2616,7 @@ describe("createDatabase factory", () => {
const db = createDatabase(fusionDir);
db.init();
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
expect(db.getLastModified()).toBeGreaterThan(0);
db.close();
@@ -2770,7 +2770,7 @@ describe("migration v77 task token budget columns", () => {
migrated = new Database(fusion);
migrated.init();
expect(migrated.getSchemaVersion()).toBe(118);
expect(migrated.getSchemaVersion()).toBe(119);
const rows = migrated.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>;
const names = new Set(rows.map((row) => row.name));
expect(names.has("tokenBudgetSoftAlertedAt")).toBe(true);
@@ -2801,7 +2801,7 @@ describe("migration v106 adds tasks.transitionPending (FN-1417)", () => {
const fresh = new Database(fusion);
try {
fresh.init();
expect(fresh.getSchemaVersion()).toBe(118);
expect(fresh.getSchemaVersion()).toBe(119);
const names = new Set(
(fresh.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>).map((r) => r.name),
);
@@ -2829,7 +2829,7 @@ describe("migration v106 adds tasks.transitionPending (FN-1417)", () => {
migrated = new Database(fusion);
migrated.init();
expect(migrated.getSchemaVersion()).toBe(118);
expect(migrated.getSchemaVersion()).toBe(119);
const names = new Set(
(migrated.prepare("PRAGMA table_info(tasks)").all() as Array<{ name: string }>).map((r) => r.name),
);
@@ -2855,7 +2855,7 @@ describe("migration v107 adds workflow_run_branches + index (FN-1417)", () => {
const fresh = new Database(fusion);
try {
fresh.init();
expect(fresh.getSchemaVersion()).toBe(118);
expect(fresh.getSchemaVersion()).toBe(119);
const table = fresh
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'workflow_run_branches'")
.get() as { name: string } | undefined;
@@ -2889,7 +2889,7 @@ describe("migration v107 adds workflow_run_branches + index (FN-1417)", () => {
migrated = new Database(fusion);
migrated.init();
expect(migrated.getSchemaVersion()).toBe(118);
expect(migrated.getSchemaVersion()).toBe(119);
const table = migrated
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = 'workflow_run_branches'")
.get() as { name: string } | undefined;
@@ -2930,7 +2930,7 @@ describe("migration v67 drops orphan project auth tables", () => {
migrated = new Database(fusion);
migrated.init();
expect(migrated.getSchemaVersion()).toBe(118);
expect(migrated.getSchemaVersion()).toBe(119);
const tables = migrated
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'project_auth_%'")
.all() as Array<{ name: string }>;
@@ -2957,7 +2957,7 @@ describe("migration v67 drops orphan project auth tables", () => {
try {
fresh.init();
expect(fresh.getSchemaVersion()).toBe(118);
expect(fresh.getSchemaVersion()).toBe(119);
const tables = fresh
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'project_auth_%'")
.all() as Array<{ name: string }>;

View File

@@ -91,6 +91,6 @@ describe("goals schema", () => {
});
it("reports schema version 101", () => {
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
});
});

View File

@@ -1000,7 +1000,7 @@ describe("Migration: pre-33 DB upgrade", () => {
// Step 1: Create a fresh database at v33 (runs all migrations up to 33)
const db1 = createDatabase(legacyDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(118);
expect(db1.getSchemaVersion()).toBe(119);
db1.close();
// Step 2: Manually downgrade to version 32 and drop insight tables
@@ -1035,7 +1035,7 @@ describe("Migration: pre-33 DB upgrade", () => {
expect(tableNamesBefore).not.toContain("project_insight_runs");
// Now run init — this triggers the v32→v33 migration
db3.init();
expect(db3.getSchemaVersion()).toBe(118);
expect(db3.getSchemaVersion()).toBe(119);
// Step 4: Verify insight tables exist after migration
const tablesAfter = db3.prepare(
@@ -1066,12 +1066,12 @@ describe("Migration: pre-33 DB upgrade", () => {
try {
const db1 = createDatabase(testDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(118);
expect(db1.getSchemaVersion()).toBe(119);
db1.close();
const db2 = createDatabase(testDir);
expect(() => db2.init()).not.toThrow();
expect(db2.getSchemaVersion()).toBe(118);
expect(db2.getSchemaVersion()).toBe(119);
db2.close();
} finally {
rmSync(testDir, { recursive: true, force: true });
@@ -1085,7 +1085,7 @@ describe("Migration: pre-33 DB upgrade", () => {
// Step 1: Create a fresh DB and run migrations
const db1 = createDatabase(compatDir);
db1.init();
expect(db1.getSchemaVersion()).toBe(118);
expect(db1.getSchemaVersion()).toBe(119);
// Step 2: Strip lifecycle and cancelledAt columns by recreating the
// table without them. This simulates a DB that was created before the

View File

@@ -38,7 +38,7 @@ describe("TaskStore merge request record + completion handoff marker", () => {
.all() as Array<{ name: string }>;
expect(tableRows).toEqual([{ name: "completion_handoff_markers" }, { name: "merge_requests" }]);
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
});
it("upserts merge request records", async () => {

View File

@@ -3746,7 +3746,7 @@ describe("MissionStore", () => {
describe("Loop State & Validator Run Schema (v31)", () => {
it("schema version is 101 after migration", () => {
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
});
it("mission_features table has loop state columns", () => {

View File

@@ -583,8 +583,8 @@ describe("Run Audit", () => {
expect(indexNames).toContain("idxRunAuditEventsTimestamp");
});
it("schema version is bumped to 118", () => {
expect(db.getSchemaVersion()).toBe(118);
it("schema version is bumped to 119", () => {
expect(db.getSchemaVersion()).toBe(119);
});
});
});

View File

@@ -60,7 +60,7 @@ describe("TaskStore merge queue", () => {
expect.arrayContaining(["idx_mergeQueue_lease_ready", "idx_mergeQueue_leaseExpiresAt"]),
);
expect(store.getDatabase().getSchemaVersion()).toBe(118);
expect(store.getDatabase().getSchemaVersion()).toBe(119);
});
it("migrates a legacy v88 database and preserves task rows", async () => {

View File

@@ -51,7 +51,7 @@ describe("TaskStore task documents", () => {
expect(tableNames.has("task_documents")).toBe(true);
expect(tableNames.has("task_document_revisions")).toBe(true);
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
const index = db
.prepare(

View File

@@ -175,15 +175,18 @@ describe("usage_events", () => {
expect(rows[0].agentId).toBe("A-chat");
});
// Migration: seed a DB at the PREVIOUS schema version, run migrate, assert
// the table exists and SCHEMA_VERSION equals the highest migration target.
// Fresh-DB tests cannot catch the early-return bug this guards.
// Migration: seed a DB at the version JUST BEFORE usage_events was introduced
// (117 — usage_events is the v118 migration), run migrate, assert the table
// exists and SCHEMA_VERSION reaches the highest migration target. Pinned to
// 117 (not SCHEMA_VERSION-1) so it keeps exercising usage_events' own
// migration as later migrations are added. Fresh-DB tests cannot catch the
// migrate-loop early-return bug this guards.
it("creates usage_events when migrating from the previous schema version", () => {
db.exec("DROP INDEX IF EXISTS idxUsageEventsTs");
db.exec("DROP INDEX IF EXISTS idxUsageEventsTaskId");
db.exec("DROP INDEX IF EXISTS idxUsageEventsAgentId");
db.exec("DROP TABLE IF EXISTS usage_events");
db.prepare("UPDATE __meta SET value = ? WHERE key = 'schemaVersion'").run(String(SCHEMA_VERSION - 1));
db.prepare("UPDATE __meta SET value = ? WHERE key = 'schemaVersion'").run("117");
(db as unknown as { migrate: () => void }).migrate();

View File

@@ -162,7 +162,7 @@ export function isFts5CorruptionError(error: unknown): boolean {
// ── Schema Definition ────────────────────────────────────────────────
const SCHEMA_VERSION = 118;
const SCHEMA_VERSION = 119;
const TASKS_FTS_AUTOMERGE = 8;
const TASKS_FTS_CRISISMERGE = 16;
@@ -1230,6 +1230,31 @@ CREATE TABLE IF NOT EXISTS usage_events (
CREATE INDEX IF NOT EXISTS idxUsageEventsTs ON usage_events(ts);
CREATE INDEX IF NOT EXISTS idxUsageEventsTaskId ON usage_events(taskId);
CREATE INDEX IF NOT EXISTS idxUsageEventsAgentId ON usage_events(agentId);
-- Persistent, incrementally-refreshed knowledge index (U14). One row per
-- knowledge page (currently one page per completed task; PR-history pages
-- share the same shape). Downstream agents query it through the dashboard's
-- scoped knowledge-index endpoint. searchText is a denormalized lowercased
-- concatenation of the page's title/summary/content + tags used for keyword
-- LIKE matching, so the index works without requiring SQLite FTS5 (which is
-- not available on every build -- see probeFts5 above). Refresh is per-source
-- (upsert by sourceKey), never a full re-index, so unaffected pages keep their
-- timestamps.
CREATE TABLE IF NOT EXISTS knowledge_pages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
sourceKind TEXT NOT NULL,
sourceId TEXT NOT NULL,
sourceKey TEXT NOT NULL UNIQUE,
title TEXT NOT NULL,
summary TEXT,
content TEXT NOT NULL,
tags TEXT,
searchText TEXT NOT NULL,
createdAt TEXT NOT NULL,
updatedAt TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idxKnowledgePagesSourceKind ON knowledge_pages(sourceKind);
CREATE INDEX IF NOT EXISTS idxKnowledgePagesUpdatedAt ON knowledge_pages(updatedAt);
`;
const TABLE_LEVEL_CONSTRAINT_PREFIXES = new Set([
@@ -4773,6 +4798,36 @@ export class Database {
});
}
// Migration 119: Persistent knowledge index (U14). One queryable page per
// completed task / PR-history entry, refreshed incrementally (upsert by
// sourceKey) on task completion. Mirrors the SCHEMA_SQL definition above so
// a fresh-from-SCHEMA_SQL DB and a migrated DB converge on the same table.
if (version < 119) {
this.applyMigration(119, () => {
this.db.exec(`
CREATE TABLE IF NOT EXISTS knowledge_pages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
sourceKind TEXT NOT NULL,
sourceId TEXT NOT NULL,
sourceKey TEXT NOT NULL UNIQUE,
title TEXT NOT NULL,
summary TEXT,
content TEXT NOT NULL,
tags TEXT,
searchText TEXT NOT NULL,
createdAt TEXT NOT NULL,
updatedAt TEXT NOT NULL
)
`);
this.db.exec(`
CREATE INDEX IF NOT EXISTS idxKnowledgePagesSourceKind ON knowledge_pages(sourceKind)
`);
this.db.exec(`
CREATE INDEX IF NOT EXISTS idxKnowledgePagesUpdatedAt ON knowledge_pages(updatedAt)
`);
});
}
}
/**

View File

@@ -0,0 +1,240 @@
// @vitest-environment node
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { mkdtempSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { EventEmitter } from "node:events";
import { Database, SCHEMA_VERSION } from "@fusion/core";
import type { TaskStore } from "@fusion/core";
import {
upsertKnowledgePage,
queryKnowledgePages,
getKnowledgePage,
countKnowledgePages,
refreshKnowledgeForTask,
renderTaskPage,
tokenizeQuery,
buildSearchText,
} from "../knowledge-index.js";
function makeDb(): { db: Database; tmpDir: string } {
const tmpDir = mkdtempSync(join(tmpdir(), "kb-knowledge-index-"));
const db = new Database(join(tmpDir, ".fusion"));
db.init();
return { db, tmpDir };
}
describe("knowledge-index store", () => {
let db: Database;
let tmpDir: string;
beforeEach(() => {
({ db, tmpDir } = makeDb());
});
afterEach(() => {
db.close();
rmSync(tmpDir, { recursive: true, force: true });
});
it("creates knowledge_pages with the expected columns on fresh init", () => {
const cols = (db.prepare("PRAGMA table_info(knowledge_pages)").all() as Array<{ name: string }>).map(
(c) => c.name,
);
expect(cols).toEqual([
"id",
"sourceKind",
"sourceId",
"sourceKey",
"title",
"summary",
"content",
"tags",
"searchText",
"createdAt",
"updatedAt",
]);
});
it("upserts a page and returns it via a keyword query", () => {
const { created } = upsertKnowledgePage(db, {
sourceKind: "task",
sourceId: "T-1",
title: "Add caching layer",
content: "Introduced an LRU cache in fetcher.ts",
tags: ["fetcher.ts"],
});
expect(created).toBe(true);
const hits = queryKnowledgePages(db, { query: "cache" });
expect(hits).toHaveLength(1);
expect(hits[0].sourceId).toBe("T-1");
expect(hits[0].tags).toEqual(["fetcher.ts"]);
});
it("AND-matches all query terms", () => {
upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-1", title: "alpha gadget", content: "only alpha here" });
upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-2", title: "alpha thing", content: "beta widget" });
expect(queryKnowledgePages(db, { query: "alpha widget" }).map((p) => p.sourceId)).toEqual(["T-2"]);
});
it("a blank/termless query returns nothing (never the whole index)", () => {
upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-1", title: "x", content: "y" });
expect(queryKnowledgePages(db, { query: "" })).toHaveLength(0);
expect(queryKnowledgePages(db, { query: " " })).toHaveLength(0);
expect(countKnowledgePages(db)).toBe(1);
});
it("escapes LIKE wildcards so user input can't widen the match", () => {
upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-1", title: "literal", content: "100% done" });
// A bare "%" must not match every row; it has no alphanumeric token at all.
expect(queryKnowledgePages(db, { query: "%" })).toHaveLength(0);
// The literal token does match.
expect(queryKnowledgePages(db, { query: "100" })).toHaveLength(1);
});
it("incremental refresh updates only the affected page; others keep their timestamps", () => {
const { page: a } = upsertKnowledgePage(db, {
sourceKind: "task",
sourceId: "T-A",
title: "A",
content: "a",
now: "2026-01-01T00:00:00.000Z",
});
const { page: b } = upsertKnowledgePage(db, {
sourceKind: "task",
sourceId: "T-B",
title: "B",
content: "b",
now: "2026-01-01T00:00:00.000Z",
});
expect(a.createdAt).toBe("2026-01-01T00:00:00.000Z");
// Re-index only T-A at a later time.
const { created, page: aUpdated } = upsertKnowledgePage(db, {
sourceKind: "task",
sourceId: "T-A",
title: "A v2",
content: "a v2",
now: "2026-02-02T00:00:00.000Z",
});
expect(created).toBe(false);
expect(aUpdated.createdAt).toBe("2026-01-01T00:00:00.000Z"); // createdAt preserved
expect(aUpdated.updatedAt).toBe("2026-02-02T00:00:00.000Z"); // updatedAt advanced
// T-B is untouched: same updatedAt as when it was created.
const bAfter = getKnowledgePage(db, "task", "T-B");
expect(bAfter?.updatedAt).toBe(b.updatedAt);
expect(bAfter?.updatedAt).toBe("2026-01-01T00:00:00.000Z");
// Still exactly two pages — no duplicate created on re-index.
expect(countKnowledgePages(db)).toBe(2);
});
// Seed a DB at the PREVIOUS schema version (118), run migrate, assert the
// table exists and SCHEMA_VERSION lands at the highest migration target (119).
// Fresh-DB tests cannot catch the migrate-loop early-return bug this guards.
it("creates knowledge_pages when migrating from the previous schema version", () => {
db.exec("DROP INDEX IF EXISTS idxKnowledgePagesSourceKind");
db.exec("DROP INDEX IF EXISTS idxKnowledgePagesUpdatedAt");
db.exec("DROP TABLE IF EXISTS knowledge_pages");
db.prepare("UPDATE __meta SET value = ? WHERE key = 'schemaVersion'").run(String(SCHEMA_VERSION - 1));
(db as unknown as { migrate: () => void }).migrate();
const table = db
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='knowledge_pages'")
.get() as { name: string } | undefined;
expect(table?.name).toBe("knowledge_pages");
expect(db.getSchemaVersion()).toBe(SCHEMA_VERSION);
// The migrated table is writable and queryable.
upsertKnowledgePage(db, { sourceKind: "task", sourceId: "T-mig", title: "migrated", content: "ok" });
expect(queryKnowledgePages(db, { query: "migrated" })).toHaveLength(1);
});
it("SCHEMA_VERSION matches the highest applied migration on a fresh DB", () => {
expect(db.getSchemaVersion()).toBe(SCHEMA_VERSION);
});
});
describe("knowledge-index pure helpers", () => {
it("tokenizeQuery splits on non-word chars and lowercases", () => {
expect(tokenizeQuery("Add OAuth, login-flow!")).toEqual(["add", "oauth", "login", "flow"]);
expect(tokenizeQuery(" ")).toEqual([]);
});
it("buildSearchText concatenates and lowercases all fields", () => {
const text = buildSearchText({ title: "Title", summary: "Sum", content: "Body", tags: ["Tag"] });
expect(text).toBe("title sum body tag");
});
it("renderTaskPage builds a deterministic page from task facts", () => {
const page = renderTaskPage({
id: "FN-7",
title: "Fix bug",
description: "Null deref in parser",
modifiedFiles: ["src/parser.ts"],
commitSubjects: ["fix: guard null"],
prUrl: "https://example.com/pr/7",
});
expect(page.sourceKind).toBe("task");
expect(page.sourceId).toBe("FN-7");
expect(page.title).toBe("Fix bug");
expect(page.content).toContain("Null deref in parser");
expect(page.content).toContain("src/parser.ts");
expect(page.content).toContain("fix: guard null");
expect(page.content).toContain("https://example.com/pr/7");
expect(page.tags).toEqual(["parser.ts"]);
});
it("renderTaskPage falls back to a generated title when none is set", () => {
const page = renderTaskPage({ id: "FN-8", description: "", modifiedFiles: [] });
expect(page.title).toBe("Task FN-8");
});
});
describe("refreshKnowledgeForTask hook", () => {
let db: Database;
let tmpDir: string;
function storeFor(database: Database, tasks: Record<string, unknown>): TaskStore {
const store = new EventEmitter() as unknown as TaskStore & {
getDatabase(): Database;
getTask(id: string): Promise<unknown>;
};
store.getDatabase = () => database;
store.getTask = async (id: string) => tasks[id] ?? null;
return store;
}
beforeEach(() => {
({ db, tmpDir } = makeDb());
});
afterEach(() => {
db.close();
rmSync(tmpDir, { recursive: true, force: true });
});
it("indexes a completed task so it becomes queryable", async () => {
const store = storeFor(db, {
"FN-1": {
id: "FN-1",
title: "Implement retry",
description: "Exponential backoff in client.ts",
modifiedFiles: ["client.ts"],
column: "done",
},
});
const page = await refreshKnowledgeForTask(store, "FN-1");
expect(page?.sourceId).toBe("FN-1");
expect(queryKnowledgePages(db, { query: "backoff" })).toHaveLength(1);
});
it("is fail-soft: returns null for a missing task without throwing", async () => {
const store = storeFor(db, {});
await expect(refreshKnowledgeForTask(store, "nope")).resolves.toBeNull();
expect(countKnowledgePages(db)).toBe(0);
});
});

View File

@@ -0,0 +1,81 @@
// @vitest-environment node
/**
* Auth integration for the knowledge-index endpoints (U14): every endpoint must
* be rejected with 401 when unauthenticated and accepted with a valid bearer
* token. Mirrors `register-command-center-routes.auth.test.ts` — the registrar
* adds no auth of its own; it inherits the server-level middleware, which is
* exactly what this asserts.
*/
import { describe, it, expect, vi, beforeEach } from "vitest";
import { EventEmitter } from "node:events";
import type { Task, TaskStore } from "@fusion/core";
import { request } from "../test-request.js";
import { createServer } from "../server.js";
vi.mock("@fusion/core", async (importOriginal) => {
const { createCoreMock } = await import("../test/mockCoreEngine.js");
return createCoreMock(() => importOriginal<typeof import("@fusion/core")>(), {});
});
class MockStore extends EventEmitter {
getRootDir(): string {
return "/tmp/fn-knowledge-auth-test";
}
getFusionDir(): string {
return "/tmp/fn-knowledge-auth-test/.fusion";
}
getDatabase() {
return {
exec: vi.fn(),
prepare: vi.fn().mockReturnValue({
run: vi.fn().mockReturnValue({ changes: 0 }),
get: vi.fn().mockReturnValue({ count: 0 }),
all: vi.fn().mockReturnValue([]),
}),
};
}
getDatabaseHealth() {
return {
healthy: true,
corruptionDetected: false,
corruptionErrors: [],
isRunning: false,
lastCheckedAt: null,
};
}
async listTasks(): Promise<Task[]> {
return [];
}
}
const TOKEN = "fn_knowledge_test1234567890abc";
describe("Knowledge routes — auth", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("rejects an unauthenticated query with 401", async () => {
const app = createServer(new MockStore() as unknown as TaskStore, {
daemon: { token: TOKEN },
});
const res = await request(app, "GET", "/api/knowledge/query?q=anything");
expect(res.status).toBe(401);
});
it("accepts the query with a valid bearer token", async () => {
const app = createServer(new MockStore() as unknown as TaskStore, {
daemon: { token: TOKEN },
});
const res = await request(app, "GET", "/api/knowledge/query?q=anything", undefined, {
Authorization: `Bearer ${TOKEN}`,
});
expect(res.status).toBe(200);
});
});

View File

@@ -0,0 +1,185 @@
// @vitest-environment node
import express, { type NextFunction, type Request, type Response } from "express";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { mkdtempSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { EventEmitter } from "node:events";
import { Database } from "@fusion/core";
import type { TaskStore } from "@fusion/core";
import { request } from "../test-request.js";
import { ApiError } from "../api-error.js";
import { registerKnowledgeRoutes } from "../routes/register-knowledge-routes.js";
import { upsertKnowledgePage } from "../knowledge-index.js";
import type { ApiRoutesContext } from "../routes/types.js";
interface QueryResponse {
query: string;
pages: Array<{ sourceId: string }>;
total: number;
}
interface RefreshResponse {
page: { sourceId: string };
}
/** POST JSON helper over the bare `request` (which only accepts string bodies). */
function postJson(
app: ReturnType<typeof buildApp>,
path: string,
body: unknown,
): ReturnType<typeof request> {
return request(app, "POST", path, JSON.stringify(body), {
"content-type": "application/json",
});
}
/** A minimal TaskStore exposing getDatabase()/getTask(), which is all routes use. */
function storeFor(db: Database, tasks: Record<string, unknown> = {}): TaskStore {
const store = new EventEmitter() as unknown as TaskStore & {
getDatabase(): Database;
getTask(id: string): Promise<unknown>;
};
store.getDatabase = () => db;
store.getTask = async (id: string) => tasks[id] ?? null;
return store;
}
function buildApp(stores: Record<string, TaskStore>, fallback: TaskStore) {
const app = express();
app.use(express.json());
const router = express.Router();
const ctx = {
router,
getScopedStore: async (req: Request): Promise<TaskStore> => {
const projectId = typeof req.query.projectId === "string" ? req.query.projectId : undefined;
return projectId && stores[projectId] ? stores[projectId] : fallback;
},
rethrowAsApiError: (error: unknown, fallbackMessage?: string): never => {
if (error instanceof ApiError) throw error;
throw new ApiError(500, fallbackMessage ?? "Internal error");
},
} as unknown as ApiRoutesContext;
registerKnowledgeRoutes(ctx);
app.use("/api", router);
app.use((err: unknown, _req: Request, res: Response, _next: NextFunction) => {
if (err instanceof ApiError) {
res.status(err.statusCode).json({ error: err.message });
return;
}
res.status(500).json({ error: "Internal error" });
});
return app;
}
describe("register-knowledge-routes", () => {
let tmpDir: string;
let dbA: Database;
let dbB: Database;
let app: ReturnType<typeof buildApp>;
beforeEach(() => {
tmpDir = mkdtempSync(join(tmpdir(), "kb-knowledge-routes-"));
dbA = new Database(join(tmpDir, "a", ".fusion"));
dbA.init();
dbB = new Database(join(tmpDir, "b", ".fusion"));
dbB.init();
upsertKnowledgePage(dbA, {
sourceKind: "task",
sourceId: "FN-A1",
title: "Add OAuth login flow",
content: "Implemented oauth login with token refresh in auth.ts",
tags: ["auth.ts"],
});
upsertKnowledgePage(dbB, {
sourceKind: "task",
sourceId: "FN-B1",
title: "Secret project-B widget",
content: "Project B only — confidential widget rendering",
tags: ["widget.ts"],
});
const storeA = storeFor(dbA, {
"FN-A2": {
id: "FN-A2",
title: "Refactor payment module",
description: "Cleaned up the stripe payment handler",
modifiedFiles: ["payment.ts"],
column: "done",
},
});
const storeB = storeFor(dbB);
app = buildApp({ "proj-a": storeA, "proj-b": storeB }, storeA);
});
afterEach(() => {
dbA.close();
dbB.close();
rmSync(tmpDir, { recursive: true, force: true });
});
it("returns relevant pages for a keyword query (fixture)", async () => {
const res = await request(app, "GET", "/api/knowledge/query?q=oauth&projectId=proj-a");
expect(res.status).toBe(200);
const body = res.body as QueryResponse;
expect(body.pages).toHaveLength(1);
expect(body.pages[0].sourceId).toBe("FN-A1");
expect(body.total).toBe(1);
});
it("returns empty for a non-matching keyword", async () => {
const res = await request(app, "GET", "/api/knowledge/query?q=kubernetes&projectId=proj-a");
expect(res.status).toBe(200);
expect((res.body as QueryResponse).pages).toHaveLength(0);
});
it("returns empty for a blank query rather than the whole index", async () => {
const res = await request(app, "GET", "/api/knowledge/query?q=&projectId=proj-a");
expect(res.status).toBe(200);
const body = res.body as QueryResponse;
expect(body.pages).toHaveLength(0);
expect(body.total).toBe(1);
});
it("project scoping — project-A query cannot read project-B pages", async () => {
// The project-B-only term must never surface for project A.
const leak = await request(app, "GET", "/api/knowledge/query?q=widget&projectId=proj-a");
expect(leak.status).toBe(200);
expect((leak.body as QueryResponse).pages).toHaveLength(0);
// ...but is visible to project B.
const ok = await request(app, "GET", "/api/knowledge/query?q=widget&projectId=proj-b");
expect(ok.status).toBe(200);
const okBody = ok.body as QueryResponse;
expect(okBody.pages).toHaveLength(1);
expect(okBody.pages[0].sourceId).toBe("FN-B1");
});
it("POST /refresh incrementally indexes a completed task, then it is queryable", async () => {
const refresh = await postJson(app, "/api/knowledge/refresh?projectId=proj-a", {
taskId: "FN-A2",
});
expect(refresh.status).toBe(200);
expect((refresh.body as RefreshResponse).page.sourceId).toBe("FN-A2");
const q = await request(app, "GET", "/api/knowledge/query?q=stripe&projectId=proj-a");
expect(q.status).toBe(200);
const qBody = q.body as QueryResponse;
expect(qBody.pages).toHaveLength(1);
expect(qBody.pages[0].sourceId).toBe("FN-A2");
});
it("POST /refresh returns 404 for an unknown task", async () => {
const res = await postJson(app, "/api/knowledge/refresh?projectId=proj-a", {
taskId: "does-not-exist",
});
expect(res.status).toBe(404);
});
it("POST /refresh requires a taskId", async () => {
const res = await postJson(app, "/api/knowledge/refresh?projectId=proj-a", {});
expect(res.status).toBe(400);
});
});

View File

@@ -37,6 +37,23 @@ export { rateLimit, RATE_LIMITS, type RateLimitOptions } from "./rate-limit.js";
export { GitHubPollingService, type GitHubPollingServiceOptions, type TaskWatchInput, type WatchedBadgeType } from "./github-poll.js";
export { GitHubIssueCommentService, DEFAULT_COMMENT_TEMPLATE } from "./github-issue-comment.js";
export { GitHubSourceIssueCloseService } from "./github-source-issue-close.js";
export {
upsertKnowledgePage,
queryKnowledgePages,
getKnowledgePage,
countKnowledgePages,
refreshKnowledgeForTask,
renderTaskPage,
buildSearchText,
tokenizeQuery,
KNOWLEDGE_QUERY_DEFAULT_LIMIT,
KNOWLEDGE_QUERY_MAX_LIMIT,
type KnowledgePage,
type KnowledgePageInput,
type KnowledgeSourceKind,
type KnowledgeQueryOptions,
} from "./knowledge-index.js";
export { KnowledgeIndexRefreshService } from "./knowledge-index-refresh.js";
export { GitHubTrackingCommentService, formatTrackingComment } from "./github-tracking-comments.js";
export { GitHubTrackingStateService, decideIssueAction } from "./github-tracking-state.js";
export { GitHubTrackingReconciler, RECONCILE_CONCURRENCY_LIMIT, RECONCILE_SCAN_LIMIT } from "./github-tracking-reconciler.js";

View File

@@ -0,0 +1,67 @@
import type { TaskStore } from "@fusion/core";
import { refreshKnowledgeForTask } from "./knowledge-index.js";
/**
* Task-completion refresh hook for the persistent knowledge index (U14).
*
* Listens for `task:moved` and, when a task reaches `done`, incrementally
* re-indexes just that task as a knowledge page (one upsert, never a full
* re-index). Mirrors the attach/detach/start/stop lifecycle of
* `GitHubSourceIssueCloseService` so it can be wired the same way alongside the
* other `task:moved` listeners. All refresh work is fail-soft (see
* {@link refreshKnowledgeForTask}) so it can never disrupt task completion.
*/
interface TaskMovedEvent {
task: { id: string };
// store's `task:moved` carries `ColumnId`; this handler only literal-compares
// legacy ids, so the widened string field is safe.
from: string;
to: string;
}
export class KnowledgeIndexRefreshService {
private readonly defaultStore: TaskStore;
private readonly listeners = new Map<TaskStore, { onTaskMoved: (event: TaskMovedEvent) => void }>();
private started = false;
constructor(store: TaskStore) {
this.defaultStore = store;
}
start(): void {
if (this.started) return;
this.started = true;
this.attach(this.defaultStore);
}
stop(): void {
if (!this.started) return;
this.started = false;
for (const store of this.listeners.keys()) {
this.detach(store);
}
}
attach(store: TaskStore): void {
if (this.listeners.has(store)) return;
const onTaskMoved = (event: TaskMovedEvent): void => {
void this.handleTaskMoved(store, event);
};
this.listeners.set(store, { onTaskMoved });
if (this.started) {
store.on("task:moved", onTaskMoved);
}
}
detach(store: TaskStore): void {
const handlers = this.listeners.get(store);
if (!handlers) return;
store.off("task:moved", handlers.onTaskMoved);
this.listeners.delete(store);
}
private async handleTaskMoved(store: TaskStore, event: TaskMovedEvent): Promise<void> {
if (event.to !== "done") return;
await refreshKnowledgeForTask(store, event.task.id);
}
}

View File

@@ -0,0 +1,386 @@
/**
* Persistent knowledge index (U14).
*
* A persistent, incrementally-refreshed knowledge layer that downstream agents
* can query. Each "page" captures the durable, queryable summary of one source
* (currently one page per completed task; PR-history pages share the same row
* shape). Pages are stored in the `knowledge_pages` SQLite table (schema +
* migration 119 in `packages/core/src/db.ts`).
*
* ## Delta over `insights` / `memoryView`
*
* This is intentionally NOT a second copy of the existing surfaces:
*
* - `InsightStore` / `insights-routes.ts` / `InsightsView` store **LLM-extracted
* durable project learnings** ("patterns/principles/pitfalls" mined from
* working memory by an agent run). `memoryView` renders the freeform working/
* insights **markdown memory files**. Both are *interpretation* layers and both
* require a model run to populate.
* - The knowledge index is a **deterministic, model-free, keyword-searchable
* index of concrete task/PR history** (title, description, modified files,
* commits, PR links). It is refreshed **incrementally on task completion** —
* one upsert per affected page, never a full re-index — and exposes a plain
* keyword **query API** an agent can call to recall "what work touched X".
*
* So the genuinely new capability is: (1) a persistent per-task/PR page store,
* (2) an incremental refresh hook on task completion, and (3) a keyword query
* API — none of which the insights/memory surfaces provide.
*
* ## Search
*
* Matching is plain keyword `LIKE` over a denormalized lowercased `searchText`
* column (AND-of-terms), deliberately avoiding SQLite FTS5 — FTS5 is not
* available on every SQLite build the engine runs on (see `probeFts5` in
* `db.ts`), and the plan scopes this unit to "SQLite full-text/keyword search,
* NOT an external embedding API".
*
* ## Security
*
* The query API is registered as an {@link ApiRouteRegistrar} (see
* `routes/register-knowledge-routes.ts`) so it inherits the dashboard's standard
* session/auth middleware AND resolves the database through `getScopedStore(req)`
* before reading — exactly like U9. The index holds sensitive repo/commit/PR
* content, so it is an information-disclosure surface, never an open endpoint.
*/
import type { Database, TaskStore } from "@fusion/core";
/** The kind of source a knowledge page was indexed from. */
export type KnowledgeSourceKind = "task" | "pr";
/** A knowledge page row as stored/read from `knowledge_pages`. */
export interface KnowledgePage {
id: number;
sourceKind: KnowledgeSourceKind;
sourceId: string;
/** Stable dedupe key (`<sourceKind>:<sourceId>`); upserts target this. */
sourceKey: string;
title: string;
summary: string | null;
content: string;
tags: string[];
createdAt: string;
updatedAt: string;
}
/** Input for {@link upsertKnowledgePage}. */
export interface KnowledgePageInput {
sourceKind: KnowledgeSourceKind;
sourceId: string;
title: string;
summary?: string | null;
content: string;
tags?: string[];
/** Injectable clock for deterministic tests. Defaults to now. */
now?: string;
}
interface KnowledgePageRow {
id: number;
sourceKind: string;
sourceId: string;
sourceKey: string;
title: string;
summary: string | null;
content: string;
tags: string | null;
searchText: string;
createdAt: string;
updatedAt: string;
}
/** Maximum number of pages a single keyword query returns. */
export const KNOWLEDGE_QUERY_DEFAULT_LIMIT = 20;
export const KNOWLEDGE_QUERY_MAX_LIMIT = 100;
function sourceKeyFor(kind: KnowledgeSourceKind, id: string): string {
return `${kind}:${id}`;
}
function rowToPage(row: KnowledgePageRow): KnowledgePage {
let tags: string[] = [];
if (row.tags) {
try {
const parsed = JSON.parse(row.tags) as unknown;
if (Array.isArray(parsed)) tags = parsed.filter((t): t is string => typeof t === "string");
} catch {
tags = [];
}
}
return {
id: row.id,
sourceKind: row.sourceKind as KnowledgeSourceKind,
sourceId: row.sourceId,
sourceKey: row.sourceKey,
title: row.title,
summary: row.summary,
content: row.content,
tags,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
}
/**
* Build the denormalized, lowercased search blob a page is matched against.
* Pure so it can be unit-tested independently of the DB.
*/
export function buildSearchText(input: {
title: string;
summary?: string | null;
content: string;
tags?: string[];
}): string {
return [
input.title,
input.summary ?? "",
input.content,
(input.tags ?? []).join(" "),
]
.join(" ")
.toLowerCase();
}
/**
* Tokenize a free-text query into lowercased keyword terms. Empty / whitespace
* input yields no terms (callers treat that as "match nothing", not "match all",
* to avoid returning the whole sensitive index for a blank query).
*/
export function tokenizeQuery(query: string): string[] {
return query
.toLowerCase()
.split(/[^a-z0-9_]+/i)
.map((t) => t.trim())
.filter((t) => t.length > 0);
}
/**
* Insert or update a knowledge page, keyed by `(sourceKind, sourceId)`.
*
* **Incremental by construction:** only the row for this source is touched, so a
* refresh of one task never rewrites (or re-timestamps) any other page. On an
* update, `createdAt` is preserved and only `updatedAt` advances.
*
* @returns the upserted page and whether it was newly created.
*/
export function upsertKnowledgePage(
db: Database,
input: KnowledgePageInput,
): { page: KnowledgePage; created: boolean } {
const now = input.now ?? new Date().toISOString();
const sourceKey = sourceKeyFor(input.sourceKind, input.sourceId);
const tags = input.tags ?? [];
const searchText = buildSearchText({
title: input.title,
summary: input.summary,
content: input.content,
tags,
});
const tagsJson = JSON.stringify(tags);
const existing = db
.prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?")
.get(sourceKey) as KnowledgePageRow | undefined;
if (existing) {
db.prepare(
`UPDATE knowledge_pages
SET title = ?, summary = ?, content = ?, tags = ?, searchText = ?, updatedAt = ?
WHERE sourceKey = ?`,
).run(input.title, input.summary ?? null, input.content, tagsJson, searchText, now, sourceKey);
const updated = db
.prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?")
.get(sourceKey) as KnowledgePageRow;
return { page: rowToPage(updated), created: false };
}
db.prepare(
`INSERT INTO knowledge_pages
(sourceKind, sourceId, sourceKey, title, summary, content, tags, searchText, createdAt, updatedAt)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
).run(
input.sourceKind,
input.sourceId,
sourceKey,
input.title,
input.summary ?? null,
input.content,
tagsJson,
searchText,
now,
now,
);
const inserted = db
.prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?")
.get(sourceKey) as KnowledgePageRow;
return { page: rowToPage(inserted), created: true };
}
/** Fetch a single page by its source identity, or `undefined`. */
export function getKnowledgePage(
db: Database,
sourceKind: KnowledgeSourceKind,
sourceId: string,
): KnowledgePage | undefined {
const row = db
.prepare("SELECT * FROM knowledge_pages WHERE sourceKey = ?")
.get(sourceKeyFor(sourceKind, sourceId)) as KnowledgePageRow | undefined;
return row ? rowToPage(row) : undefined;
}
/** Options for {@link queryKnowledgePages}. */
export interface KnowledgeQueryOptions {
query: string;
sourceKind?: KnowledgeSourceKind;
limit?: number;
}
/**
* Keyword search the index. Returns pages whose `searchText` contains **all**
* query terms (AND), most-recently-updated first. A blank/termless query returns
* an empty list rather than the whole index.
*/
export function queryKnowledgePages(db: Database, options: KnowledgeQueryOptions): KnowledgePage[] {
const terms = tokenizeQuery(options.query);
if (terms.length === 0) return [];
const limit = Math.min(
Math.max(1, options.limit ?? KNOWLEDGE_QUERY_DEFAULT_LIMIT),
KNOWLEDGE_QUERY_MAX_LIMIT,
);
const clauses: string[] = [];
const params: string[] = [];
for (const term of terms) {
clauses.push("searchText LIKE ? ESCAPE '\\'");
params.push(`%${escapeLike(term)}%`);
}
if (options.sourceKind) {
clauses.push("sourceKind = ?");
params.push(options.sourceKind);
}
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
const rows = db
.prepare(`SELECT * FROM knowledge_pages ${where} ORDER BY updatedAt DESC, id DESC LIMIT ?`)
.all(...params, limit) as KnowledgePageRow[];
return rows.map(rowToPage);
}
/** Escape SQLite `LIKE` wildcards in a term so user input can't inject them. */
function escapeLike(term: string): string {
return term.replace(/[\\%_]/g, (ch) => `\\${ch}`);
}
/** Total number of pages in the index. */
export function countKnowledgePages(db: Database): number {
const row = db.prepare("SELECT COUNT(*) AS count FROM knowledge_pages").get() as { count: number };
return row.count;
}
/**
* Render a completed task into a deterministic knowledge page body. Pure so the
* refresh hook is testable without a real store. Concatenates the durable,
* non-sensitive facts: title, description, modified files, associated commit
* subjects, and PR link if present.
*/
export function renderTaskPage(task: {
id: string;
title?: string;
description?: string;
modifiedFiles?: string[];
commitSubjects?: string[];
prUrl?: string | null;
column?: string;
}): KnowledgePageInput {
const title = (task.title ?? "").trim() || `Task ${task.id}`;
const lines: string[] = [];
if (task.description?.trim()) {
lines.push(task.description.trim());
}
if (task.modifiedFiles && task.modifiedFiles.length > 0) {
lines.push(`Files: ${task.modifiedFiles.join(", ")}`);
}
if (task.commitSubjects && task.commitSubjects.length > 0) {
lines.push(`Commits:\n${task.commitSubjects.map((s) => `- ${s}`).join("\n")}`);
}
if (task.prUrl) {
lines.push(`PR: ${task.prUrl}`);
}
const tags = (task.modifiedFiles ?? [])
.map((f) => f.split("/").pop() ?? f)
.filter((t) => t.length > 0);
return {
sourceKind: "task",
sourceId: task.id,
title,
summary: task.description?.trim().slice(0, 280) || null,
content: lines.join("\n\n") || title,
tags,
};
}
/**
* Incremental refresh hook: index (or re-index) a single task as a knowledge
* page. Intended to be invoked from the task-completion path (or by code that
* observes a task reaching `done`). It reads only the one task and upserts only
* its page, so unaffected pages are never touched.
*
* **Fail-soft:** any read/write error is logged and swallowed so a knowledge
* refresh can never break the task-completion flow that called it.
*
* @returns the upserted page, or `null` if the task could not be loaded/indexed.
*/
export async function refreshKnowledgeForTask(
store: TaskStore,
taskId: string,
options?: { now?: string },
): Promise<KnowledgePage | null> {
try {
const detail = await store.getTask(taskId);
if (!detail) return null;
let commitSubjects: string[] = [];
try {
const lineageId = (detail as { lineageId?: string }).lineageId ?? detail.id;
const rows = store
.getDatabase()
.prepare(
"SELECT commitSubject FROM task_commit_associations WHERE taskLineageId = ? ORDER BY authoredAt ASC",
)
.all(lineageId) as Array<{ commitSubject: string }>;
commitSubjects = rows.map((r) => r.commitSubject);
} catch {
commitSubjects = [];
}
const prUrl = extractPrUrl(detail);
const input = renderTaskPage({
id: detail.id,
title: detail.title,
description: detail.description,
modifiedFiles: (detail as { modifiedFiles?: string[] }).modifiedFiles,
commitSubjects,
prUrl,
column: detail.column,
});
if (options?.now) input.now = options.now;
const { page } = upsertKnowledgePage(store.getDatabase(), input);
return page;
} catch (err) {
console.warn(`[knowledge-index] refresh skipped for task ${taskId}:`, err);
return null;
}
}
/** Best-effort extraction of a PR URL from a task detail, tolerant of shape. */
function extractPrUrl(detail: unknown): string | null {
if (!detail || typeof detail !== "object") return null;
const d = detail as Record<string, unknown>;
if (typeof d.prUrl === "string" && d.prUrl) return d.prUrl;
const pr = d.pullRequest as Record<string, unknown> | undefined;
if (pr && typeof pr.url === "string" && pr.url) return pr.url;
if (pr && typeof pr.htmlUrl === "string" && pr.htmlUrl) return pr.htmlUrl;
return null;
}

View File

@@ -169,6 +169,7 @@ import { registerModelRoutes } from "./routes/register-model-routes.js";
import { registerCustomProviderRoutes } from "./routes/register-custom-provider-routes.js";
import { registerUsageRoutes } from "./routes/register-usage-routes.js";
import { registerCommandCenterRoutes } from "./routes/register-command-center-routes.js";
import { registerKnowledgeRoutes } from "./routes/register-knowledge-routes.js";
import { registerSignalRoutes } from "./routes/register-signal-routes.js";
import { registerAuthRoutes } from "./routes/register-auth-routes.js";
import { registerRuntimeProviderRoutes } from "./routes/register-runtime-provider-routes.js";
@@ -1994,6 +1995,11 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout
// U9 — Command Center analytics + live snapshot endpoints. Thin adapters over
// the core aggregators; inherit standard auth + getScopedStore project scoping.
registerCommandCenterRoutes(routeContext);
// U14 — persistent knowledge index query + incremental-refresh endpoints.
// Inherit standard auth + getScopedStore project scoping (same as U9); the
// index holds sensitive repo/PR content so no endpoint is unauthenticated or
// cross-project readable.
registerKnowledgeRoutes(routeContext);
// U11 — inbound external signal webhooks (Sentry/Datadog/PagerDuty/generic).
// Each route HMAC-verifies against a per-provider secret; never an
// unauthenticated task-creation endpoint.

View File

@@ -43,6 +43,7 @@ import { GitHubTrackingCommentService } from "../github-tracking-comments.js";
import { GitHubTrackingStateService } from "../github-tracking-state.js";
import { GitHubTrackingReconciler, RECONCILE_SCAN_LIMIT } from "../github-tracking-reconciler.js";
import { GitHubSourceIssueCloseService } from "../github-source-issue-close.js";
import { KnowledgeIndexRefreshService } from "../knowledge-index-refresh.js";
import { githubRateLimiter } from "../github-poll.js";
import * as projectStoreResolver from "../project-store-resolver.js";
import { generatePrMetadata } from "../pr-metadata-generator.js";
@@ -2485,6 +2486,12 @@ export function registerGitGitHubRoutes(ctx: ApiRoutesContext): void {
githubSourceIssueCloseService.start();
ctx.registerDispose(() => githubSourceIssueCloseService.stop());
// U14 — incremental knowledge-index refresh on task completion. Listens for
// task:moved → done and re-indexes just that task as a knowledge page.
const knowledgeIndexRefreshService = new KnowledgeIndexRefreshService(store);
knowledgeIndexRefreshService.start();
ctx.registerDispose(() => knowledgeIndexRefreshService.stop());
const githubTrackingStateService = new GitHubTrackingStateService(store);
const githubTrackingReconciler = new GitHubTrackingReconciler();
const reconcileScheduledStores = new WeakSet<TaskStore>();

View File

@@ -0,0 +1,95 @@
import { ApiError } from "../api-error.js";
import {
queryKnowledgePages,
countKnowledgePages,
refreshKnowledgeForTask,
KNOWLEDGE_QUERY_DEFAULT_LIMIT,
KNOWLEDGE_QUERY_MAX_LIMIT,
type KnowledgeSourceKind,
} from "../knowledge-index.js";
import type { ApiRouteRegistrar } from "./types.js";
/**
* Persistent knowledge-index API (U14).
*
* Thin HTTP adapter over the keyword index in `knowledge-index.ts`. Downstream
* agents call `GET /api/knowledge/query` to recall task/PR history.
*
* Security (same contract as U9 — `register-command-center-routes.ts`):
* - Every route inherits the dashboard's standard session/auth middleware via
* the {@link ApiRouteRegistrar} contract, so an unauthenticated request is
* rejected with 401 by the server-level auth middleware before reaching these
* handlers. No knowledge endpoint is unauthenticated.
* - Every endpoint resolves the database through `getScopedStore(req)` before
* reading/writing, so a project-A caller can never read project-B pages. The
* index holds sensitive repo/commit/PR content, so it is an information-
* disclosure surface, not an open endpoint.
*/
const VALID_SOURCE_KINDS: ReadonlySet<string> = new Set<KnowledgeSourceKind>(["task", "pr"]);
function resolveSourceKind(query: { sourceKind?: unknown }): KnowledgeSourceKind | undefined {
const raw = typeof query.sourceKind === "string" ? query.sourceKind : undefined;
return raw !== undefined && VALID_SOURCE_KINDS.has(raw)
? (raw as KnowledgeSourceKind)
: undefined;
}
function resolveLimit(query: { limit?: unknown }): number {
const raw = typeof query.limit === "string" ? Number.parseInt(query.limit, 10) : NaN;
if (!Number.isFinite(raw)) return KNOWLEDGE_QUERY_DEFAULT_LIMIT;
return Math.min(Math.max(1, raw), KNOWLEDGE_QUERY_MAX_LIMIT);
}
export const registerKnowledgeRoutes: ApiRouteRegistrar = (ctx) => {
const { router, getScopedStore, rethrowAsApiError } = ctx;
/**
* GET /api/knowledge/query?q=<keywords>&sourceKind=task|pr&limit=N
* Keyword search over the project-scoped knowledge index. Returns the matching
* pages (most-recently-updated first) and the total index size.
*/
router.get("/knowledge/query", async (req, res) => {
try {
const store = await getScopedStore(req);
const q = typeof req.query.q === "string" ? req.query.q : "";
const pages = queryKnowledgePages(store.getDatabase(), {
query: q,
sourceKind: resolveSourceKind(req.query),
limit: resolveLimit(req.query),
});
res.json({
query: q,
pages,
total: countKnowledgePages(store.getDatabase()),
});
} catch (err: unknown) {
if (err instanceof ApiError) throw err;
rethrowAsApiError(err, "Failed to query knowledge index");
}
});
/**
* POST /api/knowledge/refresh { taskId }
* Incrementally re-index a single task as a knowledge page. Exposes the
* task-completion refresh hook over HTTP so the completion path (or an
* operator) can trigger an incremental refresh without a full re-index.
*/
router.post("/knowledge/refresh", async (req, res) => {
try {
const store = await getScopedStore(req);
const taskId = typeof req.body?.taskId === "string" ? req.body.taskId.trim() : "";
if (!taskId) {
throw new ApiError(400, "taskId is required");
}
const page = await refreshKnowledgeForTask(store, taskId);
if (!page) {
throw new ApiError(404, `Task not found or could not be indexed: ${taskId}`);
}
res.json({ page });
} catch (err: unknown) {
if (err instanceof ApiError) throw err;
rethrowAsApiError(err, "Failed to refresh knowledge index");
}
});
};

View File

@@ -743,10 +743,10 @@ describe("RoadmapStore", () => {
});
describe("schema version", () => {
it("schema version is 118 after init", () => {
it("schema version is 119 after init", () => {
// Tracks @fusion/core's SCHEMA_VERSION (the roadmap store layers on core's
// Database). Bump this in lockstep when core adds a migration.
expect(db.getSchemaVersion()).toBe(118);
expect(db.getSchemaVersion()).toBe(119);
});
});