From 27cb2d2621fb30b9bd8ceede3cc5851a07eb6de3 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Mon, 10 Aug 2026 06:15:17 -0700 Subject: [PATCH] FN-8921: add deterministic knowledge graph tooling Add a committable, provenance-tagged knowledge graph layer with CLI generation and query support. - Extract TypeScript, Markdown, and FNXC knowledge into deterministic graph nodes and edges. - Persist recoverable graph artifacts outside ignored Fusion state and expose build/query CLI commands. - Document configuration and add core and CLI coverage for graph structure, serialization, and recovery. Files changed: .changeset/fn-8921-knowledge-graph.md | 7 + .gitattributes | 3 + AGENTS.md | 1 + docs/README.md | 2 + docs/cli-reference.md | 4 + docs/knowledge-graph.md | 37 +++++ docs/settings-reference.md | 4 + docs/storage.md | 2 + packages/cli/package.json | 3 +- .../__tests__/knowledge-graph-bundle-shape.test.ts | 4 + .../src/__tests__/knowledge-graph-command.test.ts | 115 ++++++++++++++ packages/cli/src/bin.ts | 19 +++ packages/cli/src/commands/knowledge-graph.ts | 79 ++++++++++ packages/cli/tsup.config.ts | 2 + packages/core/package.json | 4 +- packages/core/src/config/settings-schema.ts | 2 + packages/core/src/index.ts | 1 + .../__tests__/derive-modules.test.ts | 11 ++ .../__tests__/extract-file-composition.test.ts | 20 +++ .../knowledge-graph/__tests__/extract-fnxc.test.ts | 33 ++++ .../__tests__/extract-markdown.test.ts | 21 +++ .../__tests__/extract-typescript.test.ts | 30 ++++ .../__tests__/file-discovery.test.ts | 26 ++++ .../graph-artifact-not-gitignored.test.ts | 15 ++ .../__tests__/graph-builder-equivalence.test.ts | 76 ++++++++++ .../__tests__/graph-builder-incremental.test.ts | 52 +++++++ .../__tests__/graph-identity.test.ts | 11 ++ .../knowledge-graph/__tests__/graph-query.test.ts | 13 ++ .../__tests__/graph-serialization.test.ts | 29 ++++ .../__tests__/graph-store-recovery.test.ts | 106 +++++++++++++ .../__tests__/resolve-imports.test.ts | 10 ++ .../core/src/knowledge-graph/derive-modules.ts | 4 + packages/core/src/knowledge-graph/extract-file.ts | 6 + packages/core/src/knowledge-graph/extract-fnxc.ts | 168 +++++++++++++++++++++ .../core/src/knowledge-graph/extract-markdown.ts | 9 ++ .../core/src/knowledge-graph/extract-typescript.ts | 107 +++++++++++++ .../core/src/knowledge-graph/file-discovery.ts | 85 +++++++++++ packages/core/src/knowledge-graph/graph-builder.ts | 141 +++++++++++++++++ .../core/src/knowledge-graph/graph-manifest.ts | 4 + packages/core/src/knowledge-graph/graph-query.ts | 126 ++++++++++++++++ .../src/knowledge-graph/graph-serialization.ts | 134 ++++++++++++++++ packages/core/src/knowledge-graph/graph-store.ts | 97 ++++++++++++ packages/core/src/knowledge-graph/graph-types.ts | 54 +++++++ packages/core/src/knowledge-graph/index.ts | 14 ++ .../core/src/knowledge-graph/resolve-imports.ts | 4 + packages/core/src/types/settings/settings-scope.ts | 2 + pnpm-lock.yaml | 12 +- 47 files changed, 1700 insertions(+), 9 deletions(-) Fusion-Task-Id: FN-8921 Fusion-Task-Lineage: 7014d0f1-fc47-454b-afe5-5f0d9b229f33 Co-authored-by: Fusion (runfusion.ai) --- .changeset/fn-8921-knowledge-graph.md | 7 + .gitattributes | 3 + AGENTS.md | 1 + docs/README.md | 2 + docs/cli-reference.md | 4 + docs/knowledge-graph.md | 37 ++++ docs/settings-reference.md | 4 + docs/storage.md | 2 + packages/cli/package.json | 3 +- .../knowledge-graph-bundle-shape.test.ts | 4 + .../__tests__/knowledge-graph-command.test.ts | 115 ++++++++++++ packages/cli/src/bin.ts | 19 ++ packages/cli/src/commands/knowledge-graph.ts | 79 ++++++++ packages/cli/tsup.config.ts | 2 + packages/core/package.json | 4 +- packages/core/src/config/settings-schema.ts | 2 + packages/core/src/index.ts | 1 + .../__tests__/derive-modules.test.ts | 11 ++ .../extract-file-composition.test.ts | 20 +++ .../__tests__/extract-fnxc.test.ts | 33 ++++ .../__tests__/extract-markdown.test.ts | 21 +++ .../__tests__/extract-typescript.test.ts | 30 ++++ .../__tests__/file-discovery.test.ts | 26 +++ .../graph-artifact-not-gitignored.test.ts | 15 ++ .../graph-builder-equivalence.test.ts | 76 ++++++++ .../graph-builder-incremental.test.ts | 52 ++++++ .../__tests__/graph-identity.test.ts | 11 ++ .../__tests__/graph-query.test.ts | 13 ++ .../__tests__/graph-serialization.test.ts | 29 +++ .../__tests__/graph-store-recovery.test.ts | 106 +++++++++++ .../__tests__/resolve-imports.test.ts | 10 ++ .../src/knowledge-graph/derive-modules.ts | 4 + .../core/src/knowledge-graph/extract-file.ts | 6 + .../core/src/knowledge-graph/extract-fnxc.ts | 168 ++++++++++++++++++ .../src/knowledge-graph/extract-markdown.ts | 9 + .../src/knowledge-graph/extract-typescript.ts | 107 +++++++++++ .../src/knowledge-graph/file-discovery.ts | 85 +++++++++ .../core/src/knowledge-graph/graph-builder.ts | 141 +++++++++++++++ .../src/knowledge-graph/graph-manifest.ts | 4 + .../core/src/knowledge-graph/graph-query.ts | 126 +++++++++++++ .../knowledge-graph/graph-serialization.ts | 134 ++++++++++++++ .../core/src/knowledge-graph/graph-store.ts | 97 ++++++++++ .../core/src/knowledge-graph/graph-types.ts | 54 ++++++ packages/core/src/knowledge-graph/index.ts | 14 ++ .../src/knowledge-graph/resolve-imports.ts | 4 + .../core/src/types/settings/settings-scope.ts | 2 + pnpm-lock.yaml | 12 +- 47 files changed, 1700 insertions(+), 9 deletions(-) create mode 100644 .changeset/fn-8921-knowledge-graph.md create mode 100644 docs/knowledge-graph.md create mode 100644 packages/cli/src/__tests__/knowledge-graph-bundle-shape.test.ts create mode 100644 packages/cli/src/__tests__/knowledge-graph-command.test.ts create mode 100644 packages/cli/src/commands/knowledge-graph.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/derive-modules.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/extract-file-composition.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/extract-fnxc.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/extract-markdown.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/extract-typescript.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/file-discovery.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/graph-artifact-not-gitignored.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/graph-builder-equivalence.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/graph-builder-incremental.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/graph-identity.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/graph-query.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/graph-serialization.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/graph-store-recovery.test.ts create mode 100644 packages/core/src/knowledge-graph/__tests__/resolve-imports.test.ts create mode 100644 packages/core/src/knowledge-graph/derive-modules.ts create mode 100644 packages/core/src/knowledge-graph/extract-file.ts create mode 100644 packages/core/src/knowledge-graph/extract-fnxc.ts create mode 100644 packages/core/src/knowledge-graph/extract-markdown.ts create mode 100644 packages/core/src/knowledge-graph/extract-typescript.ts create mode 100644 packages/core/src/knowledge-graph/file-discovery.ts create mode 100644 packages/core/src/knowledge-graph/graph-builder.ts create mode 100644 packages/core/src/knowledge-graph/graph-manifest.ts create mode 100644 packages/core/src/knowledge-graph/graph-query.ts create mode 100644 packages/core/src/knowledge-graph/graph-serialization.ts create mode 100644 packages/core/src/knowledge-graph/graph-store.ts create mode 100644 packages/core/src/knowledge-graph/graph-types.ts create mode 100644 packages/core/src/knowledge-graph/index.ts create mode 100644 packages/core/src/knowledge-graph/resolve-imports.ts diff --git a/.changeset/fn-8921-knowledge-graph.md b/.changeset/fn-8921-knowledge-graph.md new file mode 100644 index 0000000000..28aebd905a --- /dev/null +++ b/.changeset/fn-8921-knowledge-graph.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": minor +--- + +summary: Add `fn knowledge-graph build` to generate a committable code knowledge graph. +category: feature +dev: Adds deterministic core graph extraction and tracked-artifact support; generated artifacts remain operator-committed. diff --git a/.gitattributes b/.gitattributes index da39829fb9..b96747000b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,3 +16,6 @@ demo/assets/*.gif filter=lfs diff=lfs merge=lfs -text demo/**/*.png filter=lfs diff=lfs merge=lfs -text docs/screenshots/**/*.png filter=lfs diff=lfs merge=lfs -text screenshots/**/*.png filter=lfs diff=lfs merge=lfs -text + +# FNXC:RepoAssets 2026-08-10-10:00: Graph JSON is byte-stable across platforms only when Git retains LF endings. +.fusion-knowledge/**/*.json text eol=lf diff --git a/AGENTS.md b/AGENTS.md index 9417fc4f8b..7b518931c7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -325,6 +325,7 @@ Scoped exception (FN-5819/FN-8823): while project auto-merge is On, shared-branc ## Reference docs (deeper detail) - `./docs/architecture.md` — lifecycle invariants, self-healing rules, reliability interaction backstops, run-audit internals. +- `./docs/knowledge-graph.md` — deterministic committable codebase structure graph. - `./docs/testing.md` — full testing lanes, worker fanout guidance, test taxonomy, weekly velocity baseline, and file organization. - `./docs/test-velocity-baseline.md` — weekly #leads-ready test feedback-loop velocity report generated by `scripts/test-velocity-baseline.mjs`. - `./docs/dashboard-guide.md` — dashboard behavior and **Styling Guide** details. User-facing docs for Merge Advance Notice and Smart Pull live here. diff --git a/docs/README.md b/docs/README.md index b36bf7adb2..f8909d82b9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -162,3 +162,5 @@ FN-7088 links previously-unlinked first-class testing and baseline docs here so - **Workflow author:** Dashboard Guide → Workflow Editor → Workflow Steps → Settings Reference - **Power user / automation owner:** Settings Reference → Workflow Steps → Agents → Planner Oversight (Settings Reference § Workflow Settings) - **Maintainer / contributor:** Architecture → Multi-Project → Contributing + +- [Knowledge graph](knowledge-graph.md) — deterministic committable codebase structure graph. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 9ad7402840..72314bde4d 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -1385,3 +1385,7 @@ For configuration details used by these commands, see [Settings Reference](./set collision mode defaults to `skip` and `suffix` creates deterministically named copies. Agent create/update payloads accept `roles` (a non-empty role-tag array) and optional `runtimeConfig.maxWorkflowSessions`. The legacy singular `role` input remains accepted for migration compatibility. + +## Knowledge graph + +`fn knowledge-graph build [--force] [--dir ] [--json]` refreshes the deterministic, committable structure graph. `--force` bypasses incremental reuse; `--dir` overrides the project `knowledgeGraphDir`; `--json` prints build statistics as JSON. diff --git a/docs/knowledge-graph.md b/docs/knowledge-graph.md new file mode 100644 index 0000000000..565d388d74 --- /dev/null +++ b/docs/knowledge-graph.md @@ -0,0 +1,37 @@ +# Knowledge graph + +`fn knowledge-graph build` creates a deterministic, committable structure graph for the FN-8920 memory epic. It is the embedding-free first layer: no LLM, vector recall, MCP API, inferred relationships, or capability bundle is included. + +## Artifact and configuration + +The default `knowledgeGraphDir` is `.fusion-knowledge/graph`, deliberately outside ignored `.fusion/`. It contains `nodes.json`, `edges.json`, and `manifest.json`; all use sorted, LF JSON. Nodes and edges are written before the manifest so a torn write safely triggers a full rebuild. The manifest records SHA-256 fingerprints and import references, enabling changed files only to be re-extracted and deleted file ownership to be pruned. + +The artifact is intentionally not committed by this change. Operators may review and commit it with `git add .fusion-knowledge/graph`; it is not ignored. + +## Model + +Nodes are `file`, `module`, `symbol`, `doc-concept`, or `rationale`. Edges are `contains`, `imports`, and `re-exports`, and always include source, owner (`file` or `derived`), and provenance (`extracted` or reserved `inferred`). IDs are path-derived (`file:path`, `module:dir`, `symbol:path#name`, `doc:path#slug~index`, and `rationale:path#area@stamp~index`) with reserved separators percent-escaped. + +TypeScript/TSX parsing is parser-only. Exported declarations become symbols; duplicate exports collapse to one earliest-position node with `declarationCount`, including invalid source. Syntax errors remain best-effort. `export *` records a re-export relationship but cannot expand names without a checker. Relative imports are resolved lexically using `.ts`, `.tsx`, and index candidates; package and tsconfig aliases are out of scope. + +Discovery includes package `src`, package scripts, `scripts`, plugins, docs, root policy documents, and `packages/dashboard/app`, whose UI is outside `src`. Modules are derived from direct TypeScript files every build. `queryNodes`, `neighbors`, and `shortestPath` expose the in-process API and retain complete edge provenance. + +FNXC rationale comes from TypeScript parser comment ranges and markdown HTML comments. Markdown fenced/indented code cannot open a rationale comment; once opened, a comment continues through indentation. Multiple stamped headers in one comment produce separate rationale nodes. + +## Extraction and recovery contract + +The dispatcher creates exactly one file node per discovered file. TypeScript and TSX are parsed with the TypeScript parser only; malformed source remains best-effort and never blocks a build. Symbol identity uses the exported name. Legal declaration merges and invalid redeclarations use the same deterministic collapse: the first source location and attributes win, while `declarationCount` and (when needed) `symbolKinds` preserve the fact of the collision. The only graph errors are invalid caller paths, an impossible internal cross-kind/owner collision, and artifact I/O failures. + +File-owned facts are replaced only when that file hash changes. Module nodes and module containment are derived from the final file set every build. Import references are persisted in the manifest and import/re-export edges are synthesized every build, so adding or deleting a target never requires parsing an unchanged importer. Relative resolution tries `.ts`, `.tsx`, then index candidates; it intentionally does not resolve aliases, packages, or expand `export *` names. + +Every real source position is recorded as a repository-relative path, line, and column. File and module-derived items use a `syntheticSource: "true"` attribute and a 1:1 anchor. Artifact payloads contain only content-derived values (including source FNXC stamps and hashes), never build time, host, or process metadata. Missing, malformed, mismatched, or inconsistent artifact files trigger a safe full rebuild; artifacts are written nodes, edges, then manifest so a torn write cannot validate incomplete output. + +## FNXC and query behavior + +FNXC rationale is first-class data. TypeScript-family comment ranges come from the parsed tree, not a raw scanner, which prevents strings, regexes, template text, and JSX text from becoming rationale. Markdown recognizes HTML comments outside fenced or narrowly defined indented code; code state gates the comment opener only, so an already-open multi-header comment is not truncated by indentation. Each stamped header starts a separate rationale node and runs to the next header in its comment. + +`queryNodes(filter)`, `neighbors(id, options)`, and `shortestPath(from, to)` are deterministic in-process APIs. Neighbor and path results retain complete edge objects, including source, ownership, and `extracted` provenance. `inferred` is reserved in the schema for the later memory-agent layer and is not emitted by this layer. + +## Non-goals + +This layer has no LLM calls, embeddings, vector recall, MCP tools, source-validity diagnostics, language support beyond TypeScript/TSX symbols, CommonMark parser, cross-rename identity, or capability-fabric bundle. The FR-29/FR-34 bundle format is deferred. diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 773abe7e12..ff261f4d99 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -1937,3 +1937,7 @@ Settings → Authentication can hold multiple named credential accounts for each ### Engine liveness heartbeat `engineLastActiveAt` is engine liveness bookkeeping. It is deliberately non-versioned and is preserved from the live settings object when a project configuration rollback restores a historic snapshot. + +### Knowledge graph artifact location + +`knowledgeGraphDir` is an optional project setting. Its default is `.fusion-knowledge/graph`; keep it outside `.fusion`, which is ignored. The directory is intentionally committable and is refreshed with `fn knowledge-graph build`. diff --git a/docs/storage.md b/docs/storage.md index 5679853874..7fc8766c30 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -833,3 +833,5 @@ Revision listing defaults to 100 rows and clamps `limit` to 1–500. The API acc ### `project.memory_recall_records` Project-scoped structured recall records for durable decisions, preferences, and solutions. The table uses the composite `(project_id, id)` key, row-level security, created-at indexes, and a named `(project_id, kind, content_hash)` exact-hash backstop. + +- Knowledge-graph artifact: `/.fusion-knowledge/graph/` (`nodes.json`, `edges.json`, and `manifest.json`). This is deliberately outside ignored `.fusion` and may be committed at the operator's discretion. diff --git a/packages/cli/package.json b/packages/cli/package.json index 0e6a3a0388..3547258e54 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -84,7 +84,8 @@ "react": "^19.2.0", "react-i18next": "^17.0.8", "ws": "^8.18.0", - "playwright-core": "^1.60.0" + "playwright-core": "^1.60.0", + "typescript": "^5.7.0" }, "optionalDependencies": { "sherpa-onnx-node": "1.13.4" diff --git a/packages/cli/src/__tests__/knowledge-graph-bundle-shape.test.ts b/packages/cli/src/__tests__/knowledge-graph-bundle-shape.test.ts new file mode 100644 index 0000000000..f0e143b9cd --- /dev/null +++ b/packages/cli/src/__tests__/knowledge-graph-bundle-shape.test.ts @@ -0,0 +1,4 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +describe("knowledge graph bundle contract",()=>{it("keeps TypeScript runtime-resolvable rather than inlining it",async()=>{const root=resolve(import.meta.dirname,"../..");const cli=JSON.parse(await readFile(resolve(root,"package.json"),"utf8"));const core=JSON.parse(await readFile(resolve(root,"../core/package.json"),"utf8"));const tsup=await readFile(resolve(root,"tsup.config.ts"),"utf8");expect(cli.dependencies.typescript).toBeDefined();expect(core.dependencies.typescript).toBeDefined();expect(tsup).toContain('"typescript"');});}); diff --git a/packages/cli/src/__tests__/knowledge-graph-command.test.ts b/packages/cli/src/__tests__/knowledge-graph-command.test.ts new file mode 100644 index 0000000000..370fc63dca --- /dev/null +++ b/packages/cli/src/__tests__/knowledge-graph-command.test.ts @@ -0,0 +1,115 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const build = vi.hoisted(() => vi.fn().mockResolvedValue({ + graph: { nodes: [], edges: [] }, + changed: false, + stats: { + parsedFiles: 1, + reusedFiles: 0, + prunedFiles: 0, + addedFiles: 1, + deletedFiles: 0, + synthesizedImportEdges: 2, + derivedModuleCount: 0, + recoveryReason: null, + }, +})); +const close = vi.hoisted(() => vi.fn().mockResolvedValue(undefined)); +const settings = vi.hoisted(() => vi.fn().mockResolvedValue({ knowledgeGraphDir: ".graph" })); +const resolveProject = vi.hoisted(() => vi.fn().mockResolvedValue({ + projectPath: "/project", + store: { getSettings: settings }, +})); + +class TestLockRetryExhaustedError extends Error {} + +vi.mock("@fusion/core", () => ({ + buildKnowledgeGraph: build, + resolveKnowledgeGraphDir: (root: string, dir?: string) => + `${root}/${dir ?? ".fusion-knowledge/graph"}`, +})); +vi.mock("../project-context.js", () => ({ + resolveProject, + closeProjectStore: close, + createLocalStore: vi.fn(), + asLocalProjectContext: vi.fn(), +})); +vi.mock("../lock-retry.js", () => ({ + retryOnLock: (run: () => unknown) => run(), + LockRetryExhaustedError: TestLockRetryExhaustedError, +})); + +const { runKnowledgeGraphBuild } = await import("../commands/knowledge-graph.js"); + +describe("knowledge graph CLI command", () => { + beforeEach(() => { + build.mockReset().mockResolvedValue({ + graph: { nodes: [], edges: [] }, + changed: false, + stats: { + parsedFiles: 1, + reusedFiles: 0, + prunedFiles: 0, + addedFiles: 1, + deletedFiles: 0, + synthesizedImportEdges: 2, + derivedModuleCount: 0, + recoveryReason: null, + }, + }); + close.mockClear(); + settings.mockReset().mockResolvedValue({ knowledgeGraphDir: ".graph" }); + resolveProject.mockClear(); + }); + + it("uses an explicit directory over settings and emits complete JSON", async () => { + const output = vi.spyOn(console, "log").mockImplementation(() => undefined); + + await runKnowledgeGraphBuild({ dir: ".custom", json: true }); + + expect(build).toHaveBeenCalledWith(expect.objectContaining({ graphDir: "/project/.custom" })); + expect(close).toHaveBeenCalledTimes(1); + expect(JSON.parse(output.mock.calls[0]![0] as string)).toMatchObject({ + recoveryReason: null, + synthesizedImportEdges: 2, + changed: false, + }); + output.mockRestore(); + }); + + it("uses the setting before the default directory", async () => { + await runKnowledgeGraphBuild(); + expect(build).toHaveBeenCalledWith(expect.objectContaining({ graphDir: "/project/.graph" })); + }); + + it("reports all operator-facing build counters in human output", async () => { + const output = vi.spyOn(console, "log").mockImplementation(() => undefined); + build.mockResolvedValueOnce({ + graph: { nodes: [{}, {}], edges: [{}] }, + changed: true, + stats: { parsedFiles: 2, reusedFiles: 3, prunedFiles: 4, addedFiles: 2, deletedFiles: 4, synthesizedImportEdges: 5, derivedModuleCount: 6, recoveryReason: "forced" }, + }); + + await runKnowledgeGraphBuild(); + + expect(output).toHaveBeenCalledWith(expect.stringContaining("pruned 4")); + expect(output).toHaveBeenCalledWith(expect.stringContaining("synthesized imports 5")); + expect(output).toHaveBeenCalledWith(expect.stringContaining("derived modules 6")); + expect(output).toHaveBeenCalledWith(expect.stringContaining("recovery=forced")); + output.mockRestore(); + }); + + it("closes the store when the settings retry is exhausted", async () => { + settings.mockRejectedValueOnce(new TestLockRetryExhaustedError("locked")); + + await expect(runKnowledgeGraphBuild()).rejects.toThrow("locked"); + expect(close).toHaveBeenCalledTimes(1); + }); + + it("closes the store when building fails", async () => { + build.mockRejectedValueOnce(new Error("failed")); + + await expect(runKnowledgeGraphBuild()).rejects.toThrow("failed"); + expect(close).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/cli/src/bin.ts b/packages/cli/src/bin.ts index fe09099e8c..12a086f078 100644 --- a/packages/cli/src/bin.ts +++ b/packages/cli/src/bin.ts @@ -134,6 +134,7 @@ async function loadCommandHandlers() { const { runBackupCreate, runBackupList, runBackupRestore, runBackupCleanup } = await import("./commands/backup.js"); const { runDbVacuum, runDbMigrate } = await import("./commands/db.js"); const { runMemoryBackupCreate, runMemoryBackupList, runMemoryBackupRestore } = await import("./commands/memory-backup.js"); + const { runKnowledgeGraphBuild } = await import("./commands/knowledge-graph.js"); const { runMissionCreate, runMissionList, runMissionShow, runMissionDelete, runMissionActivateSlice, runMissionLinkGoal, runMissionUnlinkGoal, runMissionGoals } = await import("./commands/mission.js"); const { runGoalsList, runGoalsCreate, runGoalsArchive, runGoalsCitations } = await import("./commands/goals.js"); const { runProjectList, runProjectAdd, runProjectRemove, runProjectShow, runProjectInfo, runProjectSetDefault, runProjectDetect } = await import("./commands/project.js"); @@ -228,6 +229,7 @@ async function loadCommandHandlers() { runMemoryBackupCreate, runMemoryBackupList, runMemoryBackupRestore, + runKnowledgeGraphBuild, runMissionCreate, runMissionList, runMissionShow, @@ -466,6 +468,8 @@ PR: fn memory-backup --list List all memory backups fn memory-backup --restore Restore memory from a backup directory snapshot + fn knowledge-graph build [--force] [--dir ] [--json] + Build the deterministic knowledge graph fn plugin list | ls List installed plugins fn plugin install [--ai-scan] Install a plugin from path or package fn plugin add Alias for plugin install @@ -779,6 +783,7 @@ async function main() { runMemoryBackupCreate, runMemoryBackupList, runMemoryBackupRestore, + runKnowledgeGraphBuild, runMissionCreate, runMissionList, runMissionShow, @@ -2006,6 +2011,20 @@ async function main() { break; } + case "knowledge-graph": { + const usage = "Usage: fn knowledge-graph build [--force] [--dir ] [--json]"; + const dirIndex = args.indexOf("--dir"); + const allowed = new Set(["build", "--force", "--dir", "--json"]); + const hasUnknownArgument = args.slice(1).some((arg, index) => !allowed.has(arg) + && !(dirIndex >= 0 && index === dirIndex)); + if (args[1] !== "build" || hasUnknownArgument || (dirIndex >= 0 && !args[dirIndex + 1])) { + console.error(usage); + process.exit(1); + } + await runKnowledgeGraphBuild({ projectName, force: args.includes("--force"), json: args.includes("--json"), dir: dirIndex >= 0 ? args[dirIndex + 1] : undefined }); + break; + } + case "memory-backup": { const create = args.includes("--create"); const list = args.includes("--list"); diff --git a/packages/cli/src/commands/knowledge-graph.ts b/packages/cli/src/commands/knowledge-graph.ts new file mode 100644 index 0000000000..1223f020fe --- /dev/null +++ b/packages/cli/src/commands/knowledge-graph.ts @@ -0,0 +1,79 @@ +import { + buildKnowledgeGraph, + resolveKnowledgeGraphDir, + type ProjectSettings, +} from "@fusion/core"; +import { + asLocalProjectContext, + closeProjectStore, + createLocalStore, + resolveProject, + type ProjectContext, +} from "../project-context.js"; +import { retryOnLock } from "../lock-retry.js"; + +export interface KnowledgeGraphBuildOptions { + projectName?: string; + force?: boolean; + dir?: string; + json?: boolean; +} + +/** + * FNXC:CliBoardMutation 2026-08-10-11:28: + * The graph command resolves a project store solely to read its configured artifact directory. + * Always close that store before returning or propagating an error so the FN-7739 handle-leak + * class cannot keep a one-shot CLI process alive after a settings retry or graph build failure. + */ +export async function runKnowledgeGraphBuild( + options: KnowledgeGraphBuildOptions = {}, +): Promise { + let context: ProjectContext | undefined; + + try { + try { + context = await resolveProject(options.projectName); + } catch { + const store = await createLocalStore(process.cwd()); + context = asLocalProjectContext(store); + } + + const activeContext = context; + if (!activeContext) throw new Error("Knowledge graph project context was not resolved"); + + const settings = await retryOnLock( + () => activeContext.store.getSettings(), + { id: "knowledge-graph-settings", action: "read settings" }, + ) as ProjectSettings; + const graphDir = resolveKnowledgeGraphDir( + activeContext.projectPath, + options.dir ?? settings.knowledgeGraphDir, + ); + const result = await buildKnowledgeGraph({ + projectRoot: activeContext.projectPath, + graphDir, + force: options.force, + }); + const output = { + ...result.stats, + changed: result.changed, + nodes: result.graph.nodes.length, + edges: result.graph.edges.length, + }; + + if (options.json) { + console.log(JSON.stringify(output)); + return; + } + + const recovery = output.recoveryReason ? `; recovery=${output.recoveryReason}` : ""; + console.log( + `Knowledge graph: ${output.nodes} nodes, ${output.edges} edges; ` + + `parsed ${output.parsedFiles}, reused ${output.reusedFiles}, pruned ${output.prunedFiles}; ` + + `synthesized imports ${output.synthesizedImportEdges}, derived modules ${output.derivedModuleCount}; ` + + `changed=${output.changed}${recovery}`, + ); + } finally { + if (context) await closeProjectStore(context); + } +} diff --git a/packages/cli/tsup.config.ts b/packages/cli/tsup.config.ts index 7a93f3b0b4..913e4875b8 100644 --- a/packages/cli/tsup.config.ts +++ b/packages/cli/tsup.config.ts @@ -440,6 +440,8 @@ const cliBuildConfig = { esbuild cannot resolve, while the published CLI installs this direct runtime dep. */ "playwright-core", + // FNXC:KnowledgeGraph 2026-08-10-10:00: TypeScript 5.9.3 (sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==) stays runtime external. + "typescript", ], splitting: false, // Keep clean disabled so the dedicated plugin-sdk tsup config can emit into diff --git a/packages/core/package.json b/packages/core/package.json index e633d9d896..db6e64fc88 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -61,7 +61,6 @@ "@types/dockerode": "^3.3.41", "@types/node": "^25.5.0", "@vitest/coverage-v8": "^4.1.10", - "typescript": "^5.7.0", "vitest": "^4.1.10" }, "engines": { @@ -78,7 +77,8 @@ "extract-zip": "^2.0.1", "postgres": "^3.4.9", "tar": "^7.5.13", - "yaml": "^2.8.3" + "yaml": "^2.8.3", + "typescript": "^5.7.0" }, "optionalDependencies": { "keytar": "^7.9.0" diff --git a/packages/core/src/config/settings-schema.ts b/packages/core/src/config/settings-schema.ts index 6ec23b353d..ece70f35ab 100644 --- a/packages/core/src/config/settings-schema.ts +++ b/packages/core/src/config/settings-schema.ts @@ -728,6 +728,8 @@ export const DEFAULT_PROJECT_SETTINGS = { memoryBackupSchedule: "0 3 * * *", memoryBackupRetention: 14, memoryBackupDir: ".fusion/backups/memory", + // FNXC:KnowledgeGraph 2026-08-10-10:00: The graph must stay outside gitignored .fusion so operators may commit it. + knowledgeGraphDir: ".fusion-knowledge/graph", memoryBackupScope: "all" as const, autoSummarizeTitles: false, /* diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index fe12f8134d..bd781a03ea 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1607,6 +1607,7 @@ export { syncMemoryBackupRoutine, } from "./memory/memory-backup.js"; export type { MemoryBackupInfo, MemoryBackupOptions } from "./memory/memory-backup.js"; +export * from "./knowledge-graph/index.js"; export { exportSettings, importSettings, diff --git a/packages/core/src/knowledge-graph/__tests__/derive-modules.test.ts b/packages/core/src/knowledge-graph/__tests__/derive-modules.test.ts new file mode 100644 index 0000000000..df3a27b82f --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/derive-modules.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vitest"; +import { deriveModules } from "../derive-modules.js"; +const file=(path:string) => ({id:`file:${path}`,kind:"file" as const,name:path,owner:"file" as const,ownerPath:path,source:{path,line:1,column:1},attributes:{syntheticSource:"true"}}); +describe("module derivation", () => { + it("derives direct TypeScript directories with synthetic owning anchors", () => { + const result = deriveModules([file("src/index.ts"), file("src/child/a.ts"), file("docs/a.md")]); + expect(result.nodes.map(node => node.id)).toEqual(["module:src", "module:src/child"]); + expect(result.nodes.every(node => node.attributes.syntheticSource === "true" && node.source.line === 1)).toBe(true); + expect(result.edges.every(edge => edge.owner === "derived" && edge.attributes.syntheticSource === "true")).toBe(true); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/extract-file-composition.test.ts b/packages/core/src/knowledge-graph/__tests__/extract-file-composition.test.ts new file mode 100644 index 0000000000..0b264629ec --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/extract-file-composition.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it, vi } from "vitest"; +import { extractFile } from "../extract-file.js"; +import { extractTypeScript } from "../extract-typescript.js"; + +describe("per-file extractor composition", () => { + it("owns exactly one file node and combines TypeScript and FNXC output", () => { + const result = extractFile({ relPath: "src/a.ts", content: "/* FNXC:Area 2026-01-01-00:00: rationale */ export const a = 1;" }); + expect(result.nodes.filter(node => node.kind === "file")).toHaveLength(1); + expect(result.nodes.map(node => node.kind)).toEqual(expect.arrayContaining(["symbol", "rationale"])); + expect(result.edges.every(edge => edge.owner === "file" && edge.ownerPath === "src/a.ts" && edge.provenance === "extracted")).toBe(true); + expect(result.edges.some(edge => edge.kind === "imports" || edge.kind === "re-exports")).toBe(false); + }); + + it("threads a test-only TypeScript extractor without changing output", () => { + const input = { relPath: "src/a.ts", content: "export const a = 1;" }; + const spy = vi.fn(extractTypeScript); + expect(extractFile(input, { typescript: spy })).toEqual(extractFile(input)); + expect(spy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/extract-fnxc.test.ts b/packages/core/src/knowledge-graph/__tests__/extract-fnxc.test.ts new file mode 100644 index 0000000000..9ebc50a308 --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/extract-fnxc.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { extractFile } from "../extract-file.js"; +const rationale = (path: string, content: string) => extractFile({ relPath: path, content }).nodes.filter(node => node.kind === "rationale"); +describe("FNXC rationale extraction", () => { + it("extracts headers from real comments but never literals, regexes, or JSX text", () => { + const nodes = rationale("src/a.tsx", "const s = 'FNXC:Fake 2026-01-01-00:00'; const r = /[//]FNXC:Regex 2026-01-01-00:00/; /* FNXC:Real 2026-01-01-00:00: reason */ export const view =

// FNXC:Jsx 2026-01-01-00:00

;"); + expect(nodes.map(node => node.attributes.fnxcArea)).toEqual(["Real"]); + }); + it("extracts a real comment inside a TSX JSX expression", () => { + const nodes = rationale("src/view.tsx", "export const View = () =>
{/* FNXC:InlineRationale 2026-01-01-00:00: preserve this behavior */}
;"); + expect(nodes).toHaveLength(1); + expect(nodes[0]?.attributes).toMatchObject({ fnxcArea: "InlineRationale", fnxcText: "preserve this behavior" }); + }); + it("splits multiple headers in one block and preserves occurrence identity", () => { + const nodes = rationale("src/a.ts", "/*\n * FNXC:Area 2026-01-01-00:00: first\n * body\n * FNXC:Area 2026-01-01-00:00: second\n */"); + expect(nodes).toHaveLength(2); + expect(nodes.map(node => node.attributes.fnxcText)).toEqual(["first body", "second"]); + expect(nodes.map(node => node.id.endsWith("~0") || node.id.endsWith("~1"))).toEqual([true, true]); + }); + it("suppresses fenced and indented markdown examples while retaining HTML comments", () => { + const nodes = rationale("docs/a.md", "```md\n\n```\n\n \n"); + expect(nodes.map(node => node.attributes.fnxcArea)).toEqual(["Real"]); + }); + it("does not truncate an opened markdown comment at indented or fenced-looking body lines", () => { + const nodes = rationale("docs/a.md", ""); + expect(nodes.map(node => node.attributes.fnxcArea).sort()).toEqual(["One", "Three", "Two"].sort()); + expect(nodes.find(node => node.attributes.fnxcArea === "Two")?.attributes.fnxcText).toContain("indented continuation"); + }); + it("groups consecutive line comments into a single header-delimited unit", () => { + const nodes = rationale("src/a.ts", "// FNXC:One 2026-01-01-00:00: first\n// continuation\n// FNXC:Two 2026-01-01-00:00: second"); + expect(nodes.map(node => node.attributes.fnxcText)).toEqual(["first continuation", "second"]); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/extract-markdown.test.ts b/packages/core/src/knowledge-graph/__tests__/extract-markdown.test.ts new file mode 100644 index 0000000000..15127b4735 --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/extract-markdown.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { extractFile } from "../extract-file.js"; +describe("markdown extraction", () => { + it("extracts nested headings and ignores fenced and indented examples", () => { + const graph = extractFile({ relPath: "docs/a.md", content: "# Root\n## Child\n```md\n# ignored\n```\n\n # also ignored" }); + const docs = graph.nodes.filter(node => node.kind === "doc-concept"); + expect(docs.map(node => node.name)).toEqual(["Child", "Root"]); + expect(graph.edges.some(edge => edge.from === docs.find(node => node.name === "Root")!.id && edge.to === docs.find(node => node.name === "Child")!.id)).toBe(true); + }); + + it("does not emit headings inside an HTML comment after prose", () => { + const graph = extractFile({ relPath: "docs/a.md", content: "intro\n\n# visible" }); + expect(graph.nodes.filter(node => node.kind === "doc-concept").map(node => node.name)).toEqual(["visible"]); + }); + + it("keeps an opened comment opaque through its close while recognizing a later same-line unit", () => { + const graph = extractFile({ relPath: "docs/a.md", content: " \n# visible" }); + expect(graph.nodes.filter(node => node.kind === "rationale").map(node => node.attributes.fnxcArea)).toEqual(["One", "Two"]); + expect(graph.nodes.filter(node => node.kind === "doc-concept").map(node => node.name)).toEqual(["visible"]); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/extract-typescript.test.ts b/packages/core/src/knowledge-graph/__tests__/extract-typescript.test.ts new file mode 100644 index 0000000000..8f4d160224 --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/extract-typescript.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { extractFile } from "../extract-file.js"; + +const symbols = (text: string) => extractFile({ relPath: "src/example.ts", content: text }).nodes.filter(node => node.kind === "symbol"); +describe("TypeScript knowledge-graph extraction", () => { + it("extracts exported declarations and filters local declarations", () => { + expect(symbols("const local = 1; export function run() {} export interface Api {} export type Name = string; export enum E { A }").map(node => node.name)).toEqual(["Api", "E", "Name", "run"]); + }); + it("collapses legal and invalid duplicate exports without aborting", () => { + const nodes = symbols("export function f(): void; export function f() {} export class f {} export const a = 1; export const a = 2;"); + const f = nodes.find(node => node.name === "f")!; + expect(f.attributes).toMatchObject({ declarationCount: "3", symbolKind: "function", symbolKinds: "class,function" }); + expect(nodes.find(node => node.name === "a")?.attributes.declarationCount).toBe("2"); + }); + it("extracts destructured values, aliases, defaults, and relative import references", () => { + const result = extractFile({ relPath: "src/example.ts", content: "export const { a, b: renamed, ...rest } = value; export { x as y } from './other'; export default function named() {}" }); + expect(result.nodes.filter(node => node.kind === "symbol").map(node => node.name)).toEqual(["a", "default", "renamed", "rest", "y"]); + expect(result.nodes.find(node => node.name === "default")?.attributes).toMatchObject({ defaultExport: "true", localName: "named" }); + expect(result.importRefs).toEqual([expect.objectContaining({ kind: "re-exports", specifier: "./other" })]); + expect(result.edges.some(edge => edge.kind === "imports" || edge.kind === "re-exports")).toBe(false); + }); + it("models export assignments as default exports and keeps export= distinct", () => { + const result = extractFile({ relPath: "src/defaults.ts", content: "export default 1; export = legacy;" }); + expect(result.nodes.filter(node => node.kind === "symbol").map(node => [node.name, node.attributes.defaultExport, node.attributes.localName])) + .toEqual([["default", "true", undefined], ["export=", undefined, "legacy"]]); + }); + it("is best effort for malformed content", () => { + expect(() => extractFile({ relPath: "src/broken.ts", content: "export const a = ; /*" })).not.toThrow(); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/file-discovery.test.ts b/packages/core/src/knowledge-graph/__tests__/file-discovery.test.ts new file mode 100644 index 0000000000..13f130af4a --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/file-discovery.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { fileURLToPath } from "node:url"; +import { dirname, resolve } from "node:path"; +import { DEFAULT_EXCLUDED_DIRECTORIES, DEFAULT_MARKDOWN_ROOTS, DEFAULT_SOURCE_ROOTS, discoverFiles } from "../file-discovery.js"; + +describe("knowledge graph file discovery", () => { + it("covers every required repository source surface without generated paths", async () => { + const root = resolve(dirname(fileURLToPath(import.meta.url)), "../../../../.."); + expect(DEFAULT_SOURCE_ROOTS).toContain("packages/dashboard/app"); + expect(DEFAULT_MARKDOWN_ROOTS).toEqual(expect.arrayContaining(["docs", "AGENTS.md", "CONCEPTS.md"])); + const paths = await discoverFiles(root); + expect(paths).toEqual([...paths].sort()); + expect(paths).toEqual(expect.arrayContaining([ + expect.stringMatching(/^packages\/dashboard\/app\/.+\.(ts|tsx)$/), + expect.stringMatching(/^packages\/core\/src\/.+\.ts$/), + expect.stringMatching(/^packages\/engine\/src\/.+\.ts$/), + expect.stringMatching(/^packages\/cli\/src\/.+\.ts$/), + "scripts/check-fnxc-future-dates.mjs", + "AGENTS.md", + "CONCEPTS.md", + ])); + expect(paths.some(path => path.startsWith("docs/") && path.endsWith(".md"))).toBe(true); + expect(paths.some(path => path.endsWith(".d.ts"))).toBe(false); + for (const directory of DEFAULT_EXCLUDED_DIRECTORIES) expect(paths.some(path => path.split("/").includes(directory))).toBe(false); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/graph-artifact-not-gitignored.test.ts b/packages/core/src/knowledge-graph/__tests__/graph-artifact-not-gitignored.test.ts new file mode 100644 index 0000000000..1b01bd5a17 --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/graph-artifact-not-gitignored.test.ts @@ -0,0 +1,15 @@ +import { execFileSync } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +describe("tracked knowledge graph artifact", () => { + it("is not covered by the repository ignore rules", async () => { + const root = fileURLToPath(new URL("../../../../../", import.meta.url)); + const ignored = await readFile(join(root, ".gitignore"), "utf8"); + expect(ignored).not.toContain(".fusion-knowledge"); + expect(() => execFileSync("git", ["check-ignore", "-q", ".fusion-knowledge/graph/nodes.json"], { cwd: root, stdio: "ignore" })).toThrow(); + expect(() => execFileSync("git", ["check-ignore", "-q", ".fusion/x.json"], { cwd: root, stdio: "ignore" })).not.toThrow(); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/graph-builder-equivalence.test.ts b/packages/core/src/knowledge-graph/__tests__/graph-builder-equivalence.test.ts new file mode 100644 index 0000000000..396fc90984 --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/graph-builder-equivalence.test.ts @@ -0,0 +1,76 @@ +import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildKnowledgeGraph } from "../graph-builder.js"; + +const roots: string[] = []; +const discovery = { sourceRoots: ["src"], markdownRoots: [] }; +async function bytes(root: string): Promise { + const dir = join(root, ".fusion-knowledge/graph"); + return Promise.all(["nodes.json", "edges.json", "manifest.json"].map(file => readFile(join(dir, file), "utf8"))); +} +async function fixture(): Promise { + const root = await mkdtemp(join(tmpdir(), "kg-equivalence-")); + roots.push(root); + await mkdir(join(root, "src", "m"), { recursive: true }); + await writeFile(join(root, "src", "a.ts"), "import { b } from './b'; import { value } from './m'; export const a = b + value;"); + await writeFile(join(root, "src", "b.ts"), "export const b = 1;"); + await writeFile(join(root, "src", "m", "index.ts"), "export const value = 1;"); + return root; +} +async function equalFull(root: string): Promise { + const incremental = await bytes(root); + await buildKnowledgeGraph({ projectRoot: root, discovery, force: true }); + expect(await bytes(root)).toEqual(incremental); +} +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); + +describe("incremental/full graph equivalence", () => { + it("matches a forced full rebuild across edit, add, delete, restore, and rename", async () => { + const root = await fixture(); + await buildKnowledgeGraph({ projectRoot: root, discovery }); + await writeFile(join(root, "src", "b.ts"), "export const b = 2;"); + await buildKnowledgeGraph({ projectRoot: root, discovery }); await equalFull(root); + await writeFile(join(root, "src", "m.ts"), "export const value = 2;"); + await buildKnowledgeGraph({ projectRoot: root, discovery }); await equalFull(root); + await rm(join(root, "src", "b.ts")); + await buildKnowledgeGraph({ projectRoot: root, discovery }); await equalFull(root); + await writeFile(join(root, "src", "b.ts"), "export const b = 3;"); + await buildKnowledgeGraph({ projectRoot: root, discovery }); await equalFull(root); + await rename(join(root, "src", "b.ts"), join(root, "src", "c.ts")); + await buildKnowledgeGraph({ projectRoot: root, discovery }); await equalFull(root); + await writeFile(join(root, "src", "a.ts"), "export const duplicate = 1; export const duplicate = 2;"); + await buildKnowledgeGraph({ projectRoot: root, discovery }); await equalFull(root); + await writeFile(join(root, "src", "a.ts"), "export const duplicate = 1;"); + await buildKnowledgeGraph({ projectRoot: root, discovery }); await equalFull(root); + await writeFile(join(root, "src", "a.ts"), "import { value } from './m'; export const a = value;"); + await buildKnowledgeGraph({ projectRoot: root, discovery }); + await writeFile(join(root, "src", "m.ts"), "export const value = 3;"); + await buildKnowledgeGraph({ projectRoot: root, discovery }); await equalFull(root); + }); + + it("fully rebuilds every recovery-matrix corruption with clean-build bytes", async () => { + const root = await fixture(); + const dir = join(root, ".fusion-knowledge/graph"); + const corruptions: Array<[string, string]> = [ + ["manifest.json", "{"], + ["nodes.json", "{"], + ["edges.json", JSON.stringify({ schemaVersion: 999, edges: [] })], + ["manifest.json", JSON.stringify({ schemaVersion: 2, extractorVersion: 1, files: { "src/a.ts": { hash: "a".repeat(64) } } })], + ]; + for (const [file, content] of corruptions) { + await buildKnowledgeGraph({ projectRoot: root, discovery, force: true }); + await writeFile(join(dir, file), content); + const recovered = await buildKnowledgeGraph({ projectRoot: root, discovery }); + expect(recovered.stats.parsedFiles).toBe(3); + expect(recovered.stats.recoveryReason).toMatch(/invalid-artifact|version-mismatch|inconsistent-artifact/); + await equalFull(root); + } + await buildKnowledgeGraph({ projectRoot: root, discovery, force: true }); + await rm(join(dir, "manifest.json")); + const missing = await buildKnowledgeGraph({ projectRoot: root, discovery }); + expect(missing.stats).toMatchObject({ parsedFiles: 3, recoveryReason: "missing-artifact" }); + await equalFull(root); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/graph-builder-incremental.test.ts b/packages/core/src/knowledge-graph/__tests__/graph-builder-incremental.test.ts new file mode 100644 index 0000000000..d90ea5c798 --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/graph-builder-incremental.test.ts @@ -0,0 +1,52 @@ +import { mkdtemp, readFile, rm, writeFile, mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildKnowledgeGraph } from "../graph-builder.js"; +import { extractFile as realExtractFile } from "../extract-file.js"; +import { extractTypeScript as realTypeScript } from "../extract-typescript.js"; +const roots:string[]=[]; +const discovery = { sourceRoots: ["src"], markdownRoots: [] }; +async function fixture(){ const root=await mkdtemp(join(tmpdir(), "kg-fixture-")); roots.push(root); await mkdir(join(root,"src")); await writeFile(join(root,"src","a.ts"),"import { b } from './b'; export const a = b;"); await writeFile(join(root,"src","b.ts"),"export const b = 1;"); return root; } +afterEach(async()=>{await Promise.all(roots.splice(0).map(path=>rm(path,{recursive:true,force:true})));}); +describe("incremental graph builder", () => { + it("writes deterministic artifacts and only reparses changed files", async () => { + const root=await fixture(), dir=join(root,".fusion-knowledge/graph"); + await buildKnowledgeGraph({projectRoot:root,graphDir:dir,discovery}); + const before=await Promise.all(["nodes.json","edges.json","manifest.json"].map(file=>readFile(join(dir,file),"utf8"))); + const typeScript=vi.fn(realTypeScript), extract=vi.fn(realExtractFile); + const noChange=await buildKnowledgeGraph({projectRoot:root,graphDir:dir,extractFile:extract,deps:{typescript:typeScript},discovery}); + expect(noChange.changed).toBe(false); expect(extract).not.toHaveBeenCalled(); + await writeFile(join(root,"src","b.ts"),"export const b = 2;"); + const changed=await buildKnowledgeGraph({projectRoot:root,graphDir:dir,extractFile:extract,deps:{typescript:typeScript},discovery}); + expect(changed.stats).toMatchObject({parsedFiles:1,reusedFiles:1}); expect(extract).toHaveBeenCalledTimes(1); expect(typeScript).toHaveBeenCalledTimes(1); + await writeFile(join(root,"src","b.ts"),"export const b = 1;"); + await buildKnowledgeGraph({projectRoot:root,graphDir:dir,discovery}); + const after=await Promise.all(["nodes.json","edges.json","manifest.json"].map(file=>readFile(join(dir,file),"utf8"))); + expect(after).toEqual(before); + }); + it("uses the tracked graph directory when callers omit graphDir", async () => { + const root=await fixture(); + await buildKnowledgeGraph({projectRoot:root,discovery}); + await expect(readFile(join(root,".fusion-knowledge/graph/nodes.json"),"utf8")).resolves.toContain("file:src/a.ts"); + }); + it("prunes deleted files and dangling import edges without reparsing importers", async () => { + const root=await fixture(), dir=join(root,".fusion-knowledge/graph"); await buildKnowledgeGraph({projectRoot:root,graphDir:dir,discovery}); + await rm(join(root,"src","b.ts")); const extract=vi.fn(realExtractFile); const result=await buildKnowledgeGraph({projectRoot:root,graphDir:dir,extractFile:extract,discovery}); + expect(extract).not.toHaveBeenCalled(); expect(result.graph.nodes.some(node=>node.id==="file:src/b.ts")).toBe(false); expect(result.graph.edges.some(edge=>edge.kind==="imports")).toBe(false); + }); + it("rebuilds rather than reusing a cache with forged synthetic file provenance", async () => { + const root = await fixture(), dir = join(root, ".fusion-knowledge/graph"); + await buildKnowledgeGraph({ projectRoot: root, graphDir: dir, discovery }); + const nodesPath = join(dir, "nodes.json"); + const artifact = JSON.parse(await readFile(nodesPath, "utf8")) as { nodes: Array<{ id: string; source: { line: number } }> }; + artifact.nodes.find(node => node.id === "file:src/a.ts")!.source.line = 2; + await writeFile(nodesPath, `${JSON.stringify(artifact)}\n`); + + const extract = vi.fn(realExtractFile); + const result = await buildKnowledgeGraph({ projectRoot: root, graphDir: dir, extractFile: extract, discovery }); + expect(result.stats.recoveryReason).toBe("inconsistent-artifact"); + expect(extract).toHaveBeenCalledTimes(2); + expect(result.graph.nodes.find(node => node.id === "file:src/a.ts")?.source).toMatchObject({ line: 1, column: 1 }); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/graph-identity.test.ts b/packages/core/src/knowledge-graph/__tests__/graph-identity.test.ts new file mode 100644 index 0000000000..40365fcee3 --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/graph-identity.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vitest"; +import { edgeId, escapeIdSegment, normalizeRelPath, symbolNodeId } from "../graph-types.js"; + +describe("knowledge graph identity", () => { + it("normalizes paths and escapes reserved identity separators", () => { + expect(normalizeRelPath("src\\a.ts")).toBe("src/a.ts"); + expect(escapeIdSegment("a#b@c~d|e")).toBe("a%23b%40c%7Ed%7Ce"); + expect(symbolNodeId("src/a.ts", "x#y")).toBe("symbol:src/a.ts#x%23y"); + expect(edgeId("contains", "file:a", "symbol:a#x%7Cy")).toBe("contains|file:a|symbol:a#x%7Cy"); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/graph-query.test.ts b/packages/core/src/knowledge-graph/__tests__/graph-query.test.ts new file mode 100644 index 0000000000..ff5de6a003 --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/graph-query.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { neighbors, queryNodes, shortestPath } from "../graph-query.js"; +import type { KnowledgeGraph } from "../graph-types.js"; +const graph: KnowledgeGraph = { schemaVersion: 2, nodes: ["a", "b", "c"].map(id => ({id,kind:"file",name:id,owner:"file",ownerPath:id,source:{path:id,line:1,column:1},attributes:{}})), edges: ["a-b", "b-c"].map((id,index) => ({id,kind:"contains" as const,from:index ? "b" : "a",to:index ? "c" : "b",provenance:"extracted" as const,owner:"file" as const,ownerPath:"a",source:{path:"a",line:1,column:1},attributes:{}})) }; +describe("knowledge graph queries", () => { + it("filters, traverses, and exposes edge provenance", () => { + expect(queryNodes(graph, { idPrefix: "b" }).map(node => node.id)).toEqual(["b"]); + expect(neighbors(graph, "a", { depth: 2 }).map(result => result.node.id)).toEqual(["b", "c"]); + expect(neighbors(graph, "a", { direction: "both", depth: 2 })[0]!.edges[0]!.provenance).toBe("extracted"); + expect(shortestPath(graph, "a", "c")?.nodes.map(node => node.id)).toEqual(["a", "b", "c"]); + expect(() => shortestPath(graph, "a", "missing")).toThrow(); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/graph-serialization.test.ts b/packages/core/src/knowledge-graph/__tests__/graph-serialization.test.ts new file mode 100644 index 0000000000..07b1acf35d --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/graph-serialization.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { deserializeArtifacts, serializeGraph, serializeManifest } from "../graph-serialization.js"; +import type { GraphManifest, KnowledgeGraph } from "../graph-types.js"; + +const graph: KnowledgeGraph = { + schemaVersion: 2, + nodes: [{ id: "file:src/a.ts", kind: "file", name: "a", owner: "file", ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: { syntheticSource: "true", ext: ".ts" } }], + edges: [{ id: "contains|file:src/a.ts|symbol:src/a.ts#a", kind: "contains", from: "file:src/a.ts", to: "symbol:src/a.ts#a", provenance: "extracted", owner: "file", ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 2, column: 1 }, attributes: {} }], +}; +const manifest: GraphManifest = { schemaVersion: 2, extractorVersion: 1, files: { "src/a.ts": { hash: "a".repeat(64), importRefs: [{ kind: "imports", specifier: "./b", candidates: ["src/b.ts", "src/b.tsx"], line: 1, column: 1, typeOnly: false }] } } }; + +describe("knowledge graph serialization", () => { + it("emits schemaVersion first and preserves ordered import candidates", () => { + const bytes = serializeGraph(graph); + const manifestBytes = serializeManifest(manifest); + expect(bytes.nodes).toMatch(/^\{\n "schemaVersion"/); + expect(bytes.edges).toMatch(/^\{\n "schemaVersion"/); + expect(manifestBytes).toMatch(/^\{\n "schemaVersion"/); + expect(deserializeArtifacts(bytes.nodes, bytes.edges, manifestBytes)).toEqual({ ok: true, graph, manifest }); + expect(manifestBytes.indexOf("src/b.ts")).toBeLessThan(manifestBytes.indexOf("src/b.tsx")); + }); + + it("returns validation results rather than throwing for broken or stale artifacts", () => { + const bytes = serializeGraph(graph); + expect(deserializeArtifacts("{", bytes.edges, serializeManifest(manifest))).toEqual({ ok: false, reason: "invalid-artifact" }); + expect(bytes.edges).toContain('"id": "contains|file:src/a.ts|symbol:src/a.ts#a"'); + expect(deserializeArtifacts(bytes.nodes.replace("\"schemaVersion\": 2", "\"schemaVersion\": 3"), bytes.edges, serializeManifest(manifest))).toEqual({ ok: false, reason: "version-mismatch" }); + }); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/graph-store-recovery.test.ts b/packages/core/src/knowledge-graph/__tests__/graph-store-recovery.test.ts new file mode 100644 index 0000000000..6a6a0bf3cc --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/graph-store-recovery.test.ts @@ -0,0 +1,106 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; +import { loadArtifacts, writeArtifacts } from "../graph-store.js"; +import type { GraphManifest, KnowledgeGraph } from "../graph-types.js"; +const dirs:string[]=[]; const graph:KnowledgeGraph={schemaVersion:2,nodes:[],edges:[]}; const manifest:GraphManifest={schemaVersion:2,extractorVersion:1,files:{}}; +afterEach(async()=>{await Promise.all(dirs.splice(0).map(dir=>rm(dir,{recursive:true,force:true})));}); +describe("knowledge graph artifact recovery",()=>{ + it("treats missing and corrupt artifact sets as full rebuild candidates",async()=>{const dir=await mkdtemp(join(tmpdir(),"kg-store-"));dirs.push(dir);expect(await loadArtifacts(dir)).toMatchObject({ok:false,recoveryReason:"missing-artifact"});await writeArtifacts(dir,graph,manifest);await writeFile(join(dir,"nodes.json"),"{");expect(await loadArtifacts(dir)).toMatchObject({ok:false,recoveryReason:"invalid-artifact"});}); + it("rejects structurally corrupted graph and manifest records", async () => { + const dir = await mkdtemp(join(tmpdir(), "kg-store-")); + dirs.push(dir); + await writeArtifacts(dir, graph, manifest); + await writeFile(join(dir, "manifest.json"), JSON.stringify({ schemaVersion: 2, extractorVersion: 1, files: { "src/a.ts": { hash: "not-a-hash", importRefs: [{ kind: "imports" }] } } })); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "invalid-artifact" }); + await writeArtifacts(dir, { schemaVersion: 2, nodes: [{ id: "file:src/a.ts", kind: "file", name: "a", owner: "file", ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: {} }], edges: [] }, { schemaVersion: 2, extractorVersion: 1, files: { "src/a.ts": { hash: "a".repeat(64) } } }); + await writeFile(join(dir, "nodes.json"), JSON.stringify({ schemaVersion: 2, nodes: [{ id: "file:src/a.ts", kind: "not-a-kind", name: "a" }] })); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "invalid-artifact" }); + await writeFile(join(dir, "nodes.json"), JSON.stringify({ nodes: [] })); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "invalid-artifact" }); + }); + it("rejects a retained edge whose file owner is absent from the manifest", async () => { + const dir = await mkdtemp(join(tmpdir(), "kg-store-")); + dirs.push(dir); + const graphWithEdge: KnowledgeGraph = { + schemaVersion: 2, + nodes: [ + { id: "file:src/a.ts", kind: "file", name: "a", owner: "file", ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: {} }, + { id: "symbol:src/a.ts#a", kind: "symbol", name: "a", owner: "file", ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: {} }, + ], + edges: [{ id: "contains|file:src/a.ts|symbol:src/a.ts#a", kind: "contains", from: "file:src/a.ts", to: "symbol:src/a.ts#a", provenance: "extracted", owner: "file", ownerPath: "src/missing.ts", source: { path: "src/missing.ts", line: 1, column: 1 }, attributes: {} }], + }; + await writeArtifacts(dir, graphWithEdge, { schemaVersion: 2, extractorVersion: 1, files: { "src/a.ts": { hash: "a".repeat(64) } } }); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "inconsistent-artifact" }); + }); + it("rejects duplicate or malformed persisted roots rather than trusting a partial graph", async () => { + const dir = await mkdtemp(join(tmpdir(), "kg-store-")); + dirs.push(dir); + await writeArtifacts(dir, graph, manifest); + await writeFile(join(dir, "nodes.json"), JSON.stringify({ schemaVersion: 2, nodes: [], unexpected: true })); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "invalid-artifact" }); + await writeArtifacts(dir, { schemaVersion: 2, nodes: [ + { id: "file:src/a.ts", kind: "file", name: "a", owner: "file", ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: {} }, + { id: "file:src/a.ts", kind: "file", name: "a", owner: "file", ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: {} }, + ], edges: [] }, { schemaVersion: 2, extractorVersion: 1, files: { "src/a.ts": { hash: "a".repeat(64) } } }); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "inconsistent-artifact" }); + await writeArtifacts(dir, { schemaVersion: 2, nodes: [ + { id: "file:rogue", kind: "file", name: "rogue", owner: "file", ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: {} }, + { id: "file:src/a.ts", kind: "file", name: "a", owner: "file", ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: {} }, + ], edges: [] }, { schemaVersion: 2, extractorVersion: 1, files: { "src/a.ts": { hash: "a".repeat(64) } } }); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "inconsistent-artifact" }); + await writeFile(join(dir, "manifest.json"), JSON.stringify({ schemaVersion: 2, extractorVersion: 1, files: { "./src/a.ts": { hash: "a".repeat(64) } } })); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "invalid-artifact" }); + }); + it("rejects malformed non-file identities instead of reusing a corrupt no-op artifact", async () => { + const dir = await mkdtemp(join(tmpdir(), "kg-store-")); + dirs.push(dir); + await writeArtifacts(dir, { schemaVersion: 2, nodes: [ + { id: "file:src/a.ts", kind: "file", name: "a", owner: "file", ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: {} }, + { id: "symbol:src/other.ts#a", kind: "symbol", name: "a", owner: "file", ownerPath: "src/a.ts", source: { path: "src/other.ts", line: 1, column: 1 }, attributes: { symbolKind: "variable" } }, + ], edges: [{ id: "corrupt", kind: "contains", from: "file:src/a.ts", to: "symbol:src/other.ts#a", provenance: "extracted", owner: "file", ownerPath: "src/a.ts", source: { path: "src/other.ts", line: 1, column: 1 }, attributes: {} }] }, { schemaVersion: 2, extractorVersion: 1, files: { "src/a.ts": { hash: "a".repeat(64) } } }); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "inconsistent-artifact" }); + }); + it("rejects invalid ownership classes and duplicate edge identities", async () => { + const dir = await mkdtemp(join(tmpdir(), "kg-store-")); + dirs.push(dir); + const file = { id: "file:src/a.ts", kind: "file" as const, name: "a", owner: "file" as const, ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: {} }; + const derivedSymbol = { id: "symbol:src/a.ts#a", kind: "symbol" as const, name: "a", owner: "derived" as const, ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: { symbolKind: "variable" } }; + const validManifest = { schemaVersion: 2, extractorVersion: 1, files: { "src/a.ts": { hash: "a".repeat(64) } } }; + await writeArtifacts(dir, { schemaVersion: 2, nodes: [file, derivedSymbol], edges: [] }, validManifest); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "inconsistent-artifact" }); + + const edge = { id: "contains|file:src/a.ts|file:src/a.ts", kind: "contains" as const, from: file.id, to: file.id, provenance: "extracted" as const, owner: "file" as const, ownerPath: "src/a.ts", source: file.source, attributes: {} }; + await writeArtifacts(dir, { schemaVersion: 2, nodes: [file], edges: [edge, { ...edge }] }, validManifest); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "inconsistent-artifact" }); + }); + it("rejects forged synthetic anchors and synthetic markers on parsed facts", async () => { + const dir = await mkdtemp(join(tmpdir(), "kg-store-")); + dirs.push(dir); + const file = { id: "file:src/a.ts", kind: "file" as const, name: "a", owner: "file" as const, ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 1, column: 1 }, attributes: { ext: ".ts", syntheticSource: "true" } }; + const module = { id: "module:src", kind: "module" as const, name: "src", owner: "derived" as const, ownerPath: "src", source: { path: "src", line: 1, column: 1 }, attributes: { directFileCount: "1", barrel: "false", syntheticSource: "true" } }; + const symbol = { id: "symbol:src/a.ts#a", kind: "symbol" as const, name: "a", owner: "file" as const, ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 2, column: 1 }, attributes: { symbolKind: "variable", declarationCount: "1" } }; + const moduleEdge = { id: "contains|module:src|file:src/a.ts", kind: "contains" as const, from: module.id, to: file.id, provenance: "extracted" as const, owner: "derived" as const, ownerPath: "src", source: module.source, attributes: { syntheticSource: "true" } }; + const parentHeading = { id: "doc:src/a.ts#parent~0", kind: "doc-concept" as const, name: "parent", owner: "file" as const, ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 3, column: 1 }, attributes: { headingLevel: "1" } }; + const childHeading = { id: "doc:src/a.ts#child~0", kind: "doc-concept" as const, name: "child", owner: "file" as const, ownerPath: "src/a.ts", source: { path: "src/a.ts", line: 4, column: 1 }, attributes: { headingLevel: "2" } }; + const headingEdge = { id: "contains|doc:src/a.ts#parent~0|doc:src/a.ts#child~0", kind: "contains" as const, from: parentHeading.id, to: childHeading.id, provenance: "extracted" as const, owner: "file" as const, ownerPath: "src/a.ts", source: childHeading.source, attributes: {} }; + const validGraph = { schemaVersion: 2, nodes: [file, module, symbol, parentHeading, childHeading], edges: [moduleEdge, headingEdge] }; + const validManifest = { schemaVersion: 2, extractorVersion: 1, files: { "src/a.ts": { hash: "a".repeat(64) } } }; + await writeArtifacts(dir, validGraph, validManifest); + expect(await loadArtifacts(dir)).toMatchObject({ ok: true }); + + const invalidGraphs = [ + { ...validGraph, nodes: [{ ...file, source: { ...file.source, line: 2 } }, module, symbol] }, + { ...validGraph, nodes: [{ ...file, attributes: { ext: ".ts" } }, module, symbol] }, + { ...validGraph, nodes: [file, { ...module, attributes: { directFileCount: "1", barrel: "false" } }, symbol] }, + { ...validGraph, nodes: [file, module, { ...symbol, attributes: { ...symbol.attributes, syntheticSource: "true" } }] }, + { ...validGraph, edges: [{ ...moduleEdge, source: { path: "src/a.ts", line: 1, column: 1 } }] }, + ]; + for (const invalidGraph of invalidGraphs) { + await writeArtifacts(dir, invalidGraph, validManifest); + expect(await loadArtifacts(dir)).toMatchObject({ ok: false, recoveryReason: "inconsistent-artifact" }); + } + }); + it("does not rewrite identical bytes",async()=>{const dir=await mkdtemp(join(tmpdir(),"kg-store-"));dirs.push(dir);expect((await writeArtifacts(dir,graph,manifest)).changed).toBe(true);expect((await writeArtifacts(dir,graph,manifest)).changed).toBe(false);}); +}); diff --git a/packages/core/src/knowledge-graph/__tests__/resolve-imports.test.ts b/packages/core/src/knowledge-graph/__tests__/resolve-imports.test.ts new file mode 100644 index 0000000000..5490fa4d73 --- /dev/null +++ b/packages/core/src/knowledge-graph/__tests__/resolve-imports.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from "vitest"; +import { importCandidates, selectImportTarget } from "../resolve-imports.js"; +describe("import resolution", () => { + it("uses lexical TypeScript candidate priority", () => { + expect(importCandidates("src/a.ts", "./m")).toEqual(["src/m.ts", "src/m.tsx", "src/m/index.ts", "src/m/index.tsx"]); + expect(importCandidates("src/a.ts", "./m.js")).toEqual(["src/m.ts", "src/m.tsx"]); + expect(importCandidates("src/a.ts", "pkg")).toEqual([]); + expect(selectImportTarget(["a.ts", "a.tsx"], new Set(["a.tsx"]))).toBe("a.tsx"); + }); +}); diff --git a/packages/core/src/knowledge-graph/derive-modules.ts b/packages/core/src/knowledge-graph/derive-modules.ts new file mode 100644 index 0000000000..eb96306052 --- /dev/null +++ b/packages/core/src/knowledge-graph/derive-modules.ts @@ -0,0 +1,4 @@ +import { dirname, basename } from "node:path"; +import { edgeId, fileNodeId, moduleCanonicalPath, moduleNodeId, type GraphEdge, type GraphNode } from "./graph-types.js"; +/** Modules are recomputed from files: assigning a multi-file module to one owner would make pruning unsound. */ +export function deriveModules(files:GraphNode[]):{nodes:GraphNode[];edges:GraphEdge[]}{const paths=files.filter(n=>n.kind==="file"&&/\.tsx?$/.test(n.ownerPath)).map(n=>n.ownerPath),dirs=[...new Set(paths.map(dirname).filter(dir=>dir!=="."))].sort(),nodes:GraphNode[]=[],edges:GraphEdge[]=[];for(const dir of dirs){const direct=paths.filter(p=>dirname(p)===dir),canonical=moduleCanonicalPath(dir,paths),id=moduleNodeId(dir),attributes={directFileCount:String(direct.length),barrel:String(paths.includes(`${dir}/index.ts`)||paths.includes(`${dir}/index.tsx`)),syntheticSource:"true"};nodes.push({id,kind:"module",name:basename(dir),owner:"derived",ownerPath:dir,source:{path:canonical,line:1,column:1},attributes});for(const file of direct)edges.push({id:edgeId("contains",id,fileNodeId(file)),kind:"contains",from:id,to:fileNodeId(file),provenance:"extracted",owner:"derived",ownerPath:dir,source:{path:canonical,line:1,column:1},attributes:{syntheticSource:"true"}});const parent=dirs.filter(d=>dir.startsWith(`${d}/`)).sort((a,b)=>b.length-a.length)[0];if(parent){const parentNode=moduleNodeId(parent),parentCanonical=moduleCanonicalPath(parent,paths);edges.push({id:edgeId("contains",parentNode,id),kind:"contains",from:parentNode,to:id,provenance:"extracted",owner:"derived",ownerPath:parent,source:{path:parentCanonical,line:1,column:1},attributes:{syntheticSource:"true"}});}}return{nodes:nodes.sort((a,b)=>a.id.localeCompare(b.id)),edges:edges.sort((a,b)=>a.id.localeCompare(b.id))};} diff --git a/packages/core/src/knowledge-graph/extract-file.ts b/packages/core/src/knowledge-graph/extract-file.ts new file mode 100644 index 0000000000..71d13e2018 --- /dev/null +++ b/packages/core/src/knowledge-graph/extract-file.ts @@ -0,0 +1,6 @@ +import { extname } from "node:path"; +import { collapseDuplicateSymbolNodes, edgeId, fileNodeId, normalizeRelPath, type ExtractorDependencies, type ExtractorOutput } from "./graph-types.js"; +import { extractTypeScript } from "./extract-typescript.js"; +import { extractMarkdown } from "./extract-markdown.js"; +import { extractFnxc } from "./extract-fnxc.js"; +export function extractFile(input:{relPath:string;content:string},deps:ExtractorDependencies={}):ExtractorOutput{const relPath=normalizeRelPath(input.relPath),id=fileNodeId(relPath),ext=extname(relPath),base={relPath,content:input.content,fileNodeId:id},outputs:ExtractorOutput[]=[];if(ext===".ts"||ext===".tsx"){outputs.push((deps.typescript??extractTypeScript)(base),(deps.fnxc??extractFnxc)(base));}else if(ext===".md")outputs.push((deps.markdown??extractMarkdown)(base),(deps.fnxc??extractFnxc)(base));else if([".js",".mjs",".cjs"].includes(ext))outputs.push((deps.fnxc??extractFnxc)(base));const nodes=collapseDuplicateSymbolNodes([{id,kind:"file",name:relPath,owner:"file",ownerPath:relPath,source:{path:relPath,line:1,column:1},attributes:{ext,syntheticSource:"true"}},...outputs.flatMap(x=>x.nodes).map(node=>({...node,owner:"file" as const,ownerPath:relPath}))]);const edges=new Map();for(const edge of outputs.flatMap(x=>x.edges)){const normalized={...edge,id:edgeId(edge.kind,edge.from,edge.to),owner:"file" as const,ownerPath:relPath,provenance:"extracted" as const};if(!edges.has(normalized.id))edges.set(normalized.id,normalized);}return{nodes:nodes.sort((a,b)=>a.id.localeCompare(b.id)),edges:[...edges.values()].sort((a,b)=>a.id.localeCompare(b.id)),importRefs:outputs.flatMap(x=>x.importRefs)};} diff --git a/packages/core/src/knowledge-graph/extract-fnxc.ts b/packages/core/src/knowledge-graph/extract-fnxc.ts new file mode 100644 index 0000000000..9d6d7ca4bc --- /dev/null +++ b/packages/core/src/knowledge-graph/extract-fnxc.ts @@ -0,0 +1,168 @@ +import ts from "typescript"; +import { edgeId, rationaleNodeId, type ExtractorInput, type ExtractorOutput, type GraphNode } from "./graph-types.js"; + +export const FNXC_STAMP_SOURCE = "FNXC:([A-Za-z0-9_-]+)\\s+(\\d{4}-\\d{2}-\\d{2}(?:-\\d{2}:\\d{2})?)"; +const header = new RegExp(`^${FNXC_STAMP_SOURCE}:?\\s*(.*)$`); + +type CommentUnit = { text: string; pos: number }; + +/** + * FNXC:KnowledgeGraph 2026-08-10-11:54: + * TS-family rationale uses parser-derived trivia rather than a scanner: scanner-only tokenization + * misreads regex literals and JSX text as comments. Markdown code state gates only an HTML comment + * opener; applying it to continuation lines would truncate multi-header rationale. + */ +function scriptKind(path: string): ts.ScriptKind { + return path.endsWith(".tsx") ? ts.ScriptKind.TSX : path.endsWith(".ts") ? ts.ScriptKind.TS : ts.ScriptKind.JS; +} + +function isTemplateText(node: ts.Node): boolean { + return [ts.SyntaxKind.TemplateHead, ts.SyntaxKind.TemplateMiddle, ts.SyntaxKind.TemplateTail, ts.SyntaxKind.NoSubstitutionTemplateLiteral].includes(node.kind); +} + +function tsCommentUnits(input: ExtractorInput): CommentUnit[] { + const source = ts.createSourceFile(input.relPath, input.content, ts.ScriptTarget.Latest, true, scriptKind(input.relPath)); + const ranges = new Map(); + const visitJsxExpressions = (node: ts.Node): void => { + if (node.kind === ts.SyntaxKind.JsxExpression) return visit(node); + for (const child of node.getChildren(source)) visitJsxExpressions(child); + }; + const visit = (node: ts.Node): void => { + /* + FNXC:KnowledgeGraph 2026-08-10-11:54: + JSX expression containers preserve parser-comment extraction because rationale in an expression + comment is code. Raw JSX children are excluded because their visible text is not trivia. + */ + const jsxContainer = node.kind === ts.SyntaxKind.JsxElement || node.kind === ts.SyntaxKind.JsxFragment || node.kind === ts.SyntaxKind.JsxSelfClosingElement; + if (!jsxContainer && node.kind !== ts.SyntaxKind.JsxText && !isTemplateText(node)) { + for (const range of [...(ts.getLeadingCommentRanges(input.content, node.getFullStart()) ?? []), ...(ts.getTrailingCommentRanges(input.content, node.getEnd()) ?? [])]) ranges.set(`${range.pos}:${range.end}`, range); + } + for (const child of node.getChildren(source)) { + if (jsxContainer) visitJsxExpressions(child); + else visit(child); + } + }; + visit(source); + const ordered = [...ranges.values()].sort((left, right) => left.pos - right.pos); + const units: CommentUnit[] = []; + for (const range of ordered) { + const previous = units.at(-1); + const canJoin = range.kind === ts.SyntaxKind.SingleLineCommentTrivia && previous && /^\s*$/.test(input.content.slice(previous.pos + previous.text.length, range.pos)); + if (canJoin) previous.text += input.content.slice(previous.pos + previous.text.length, range.end); + else units.push({ text: input.content.slice(range.pos, range.end), pos: range.pos }); + } + return units; +} + +/** + * Return only real markdown HTML-comment units. The scanner deliberately evaluates fences and + * narrow indented-code state before an opener, then treats an opened unit as opaque until `-->`. + */ +export function markdownHtmlCommentUnits(content: string): CommentUnit[] { + const units: CommentUnit[] = []; + const lines = content.match(/.*(?:\n|$)/g) ?? []; + let offset = 0; + let fence: { marker: string; length: number } | undefined; + let previousIndented = false; + let previousBlank = true; + let openStart: number | undefined; + + for (const lineWithNewline of lines) { + if (!lineWithNewline) continue; + const line = lineWithNewline.replace(/\r?\n$/, ""); + let cursor = 0; + + if (openStart !== undefined) { + const close = line.indexOf("-->"); + if (close < 0) { + offset += lineWithNewline.length; + continue; + } + units.push({ pos: openStart, text: content.slice(openStart, offset + close + 3) }); + openStart = undefined; + cursor = close + 3; + } else { + const trimmed = line.trim(); + const fenceMatch = /^(?:`{3,}|~{3,})/.exec(trimmed); + if (fence) { + if (fenceMatch && fenceMatch[0]![0] === fence.marker && fenceMatch[0]!.length >= fence.length && trimmed === fenceMatch[0]) fence = undefined; + offset += lineWithNewline.length; + previousIndented = false; + previousBlank = false; + continue; + } + if (fenceMatch) { + fence = { marker: fenceMatch[0]![0]!, length: fenceMatch[0]!.length }; + offset += lineWithNewline.length; + previousIndented = false; + previousBlank = false; + continue; + } + const indented = /^(?: {4}|\t)/.test(line) && (previousIndented || previousBlank); + if (indented) { + offset += lineWithNewline.length; + previousIndented = true; + previousBlank = line.trim() === ""; + continue; + } + } + + while (cursor < line.length) { + const start = line.indexOf("", start + 4); + if (close >= 0) { + units.push({ pos: offset + start, text: content.slice(offset + start, offset + close + 3) }); + cursor = close + 3; + } else { + openStart = offset + start; + break; + } + } + offset += lineWithNewline.length; + previousIndented = false; + previousBlank = line.trim() === ""; + } + if (openStart !== undefined) units.push({ pos: openStart, text: content.slice(openStart) }); + return units; +} + +function strippedLine(value: string): string { + return value.replace(/^\s*(?:\/\/|\/\*|/g, "").replace(/^\s*\*\s?/, "").trim(); +} + +function make(input: ExtractorInput, units: CommentUnit[]): ExtractorOutput { + const nodes: GraphNode[] = []; + const edges: ExtractorOutput["edges"] = []; + const seen = new Map(); + for (const unit of units) { + let lineOffset = unit.pos; + let active: { area: string; stamp: string; text: string[]; offset: number } | undefined; + const flush = (): void => { + if (!active) return; + const occurrence = seen.get(`${active.area}\0${active.stamp}`) ?? 0; + seen.set(`${active.area}\0${active.stamp}`, occurrence + 1); + const prefix = input.content.slice(0, active.offset); + const line = prefix.split("\n").length; + const column = active.offset - (prefix.lastIndexOf("\n") + 1) + 1; + const node: GraphNode = { id: rationaleNodeId(input.relPath, active.area, active.stamp, occurrence), kind: "rationale", name: active.area, owner: "file", ownerPath: input.relPath, source: { path: input.relPath, line, column }, attributes: { fnxcArea: active.area, fnxcStamp: active.stamp, fnxcText: active.text.map(value => value.trim()).filter(Boolean).join(" ") } }; + nodes.push(node); + edges.push({ id: edgeId("contains", input.fileNodeId, node.id), kind: "contains", from: input.fileNodeId, to: node.id, provenance: "extracted", owner: "file", ownerPath: input.relPath, source: node.source, attributes: {} }); + }; + for (const raw of unit.text.split(/(?<=\n)/)) { + const clean = strippedLine(raw.replace(/\r?\n$/, "")); + const match = header.exec(clean); + if (match) { + flush(); + active = { area: match[1]!, stamp: match[2]!, text: match[3] ? [match[3]] : [], offset: lineOffset + Math.max(0, raw.indexOf("FNXC:")) }; + } else active?.text.push(clean); + lineOffset += raw.length; + } + flush(); + } + return { nodes, edges, importRefs: [] }; +} + +export function extractFnxc(input: ExtractorInput): ExtractorOutput { + return make(input, input.relPath.endsWith(".md") ? markdownHtmlCommentUnits(input.content) : tsCommentUnits(input)); +} diff --git a/packages/core/src/knowledge-graph/extract-markdown.ts b/packages/core/src/knowledge-graph/extract-markdown.ts new file mode 100644 index 0000000000..821099ee59 --- /dev/null +++ b/packages/core/src/knowledge-graph/extract-markdown.ts @@ -0,0 +1,9 @@ +import { docConceptNodeId, edgeId, type ExtractorInput, type ExtractorOutput, type GraphNode } from "./graph-types.js"; +import { markdownHtmlCommentUnits } from "./extract-fnxc.js"; +const slug=(s:string)=>s.toLowerCase().trim().replace(/[^a-z0-9]+/g,"-").replace(/^-|-$/g,"")||"heading"; +export function extractMarkdown(input:ExtractorInput):ExtractorOutput{const nodes:GraphNode[]=[];const edges:ExtractorOutput["edges"]=[], stack:{level:number;node:GraphNode}[]=[];const counts=new Map();const htmlRanges=markdownHtmlCommentUnits(input.content).map(unit=>[unit.pos,unit.pos+unit.text.length] as const);let fence:{marker:string;length:number}|undefined, previousIndented=false, previousBlank=true,offset=0;for(const [i,line] of input.content.split(/\r?\n/).entries()){const trim=line.trim(), fenceMatch=/^(`{3,}|~{3,})/.exec(trim);/* +FNXC:KnowledgeGraph 2026-08-10-11:54: +A markdown HTML comment may open after the line's first character. Range overlap keeps every +heading on that comment-bearing line out of the doc-concept graph. +*/ +const inHtml=htmlRanges.some(([start,end])=>startoffset);if(!inHtml&&fenceMatch){if(!fence)fence={marker:fenceMatch[0]![0]!,length:fenceMatch[0]!.length};else if(fence.marker===fenceMatch[0]![0]&&fenceMatch[0]!.length>=fence.length&&trim===fenceMatch[0])fence=undefined;offset+=line.length+1;previousIndented=false;previousBlank=false;continue;}const indented: boolean=/^(?: {4}|\t)/.test(line)&&(previousIndented||previousBlank);if(fence||inHtml||indented){offset+=line.length+1;previousIndented=indented;previousBlank=line.trim()==="";continue;}const match=/^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line);if(!match){offset+=line.length+1;previousIndented=false;previousBlank=line.trim()==="";continue;}const level=match[1]!.length,name=match[2]!, key=slug(name),index=counts.get(key)??0;counts.set(key,index+1);const node:GraphNode={id:docConceptNodeId(input.relPath,key,index),kind:"doc-concept",name,owner:"file",ownerPath:input.relPath,source:{path:input.relPath,line:i+1,column:line.indexOf("#")+1},attributes:{headingLevel:String(level)}};nodes.push(node);while(stack.length&&stack.at(-1)!.level>=level)stack.pop();const parent=stack.at(-1)?.node;edges.push({id:edgeId("contains",parent?.id??input.fileNodeId,node.id),kind:"contains",from:parent?.id??input.fileNodeId,to:node.id,provenance:"extracted",owner:"file",ownerPath:input.relPath,source:node.source,attributes:{}});stack.push({level,node});offset+=line.length+1;previousIndented=false;previousBlank=line.trim()==="";}return{nodes,edges,importRefs:[]};} diff --git a/packages/core/src/knowledge-graph/extract-typescript.ts b/packages/core/src/knowledge-graph/extract-typescript.ts new file mode 100644 index 0000000000..ee72e7eeef --- /dev/null +++ b/packages/core/src/knowledge-graph/extract-typescript.ts @@ -0,0 +1,107 @@ +import ts from "typescript"; +import { + edgeId, + symbolNodeId, + type ExtractorInput, + type ExtractorOutput, + type GraphNode, + type ImportRef, + type SymbolKind, +} from "./graph-types.js"; +import { importCandidates } from "./resolve-imports.js"; + +const scriptKind = (path: string): ts.ScriptKind => path.endsWith(".tsx") ? ts.ScriptKind.TSX : path.endsWith(".ts") ? ts.ScriptKind.TS : ts.ScriptKind.JS; +const kindOf = (node: ts.Node): SymbolKind => ts.isFunctionDeclaration(node) ? "function" + : ts.isClassDeclaration(node) || ts.isClassExpression(node) ? "class" + : ts.isInterfaceDeclaration(node) ? "interface" + : ts.isTypeAliasDeclaration(node) ? "type-alias" + : ts.isEnumDeclaration(node) ? "enum" + : ts.isModuleDeclaration(node) ? "namespace" + : ts.isVariableDeclaration(node) ? "variable" : "alias"; +const isExported = (node: ts.Node): boolean => !!ts.getModifiers(node as ts.HasModifiers)?.some(modifier => modifier.kind === ts.SyntaxKind.ExportKeyword); +const isDefault = (node: ts.Node): boolean => !!ts.getModifiers(node as ts.HasModifiers)?.some(modifier => modifier.kind === ts.SyntaxKind.DefaultKeyword); + +function bindingNames(name: ts.BindingName): string[] { + if (ts.isIdentifier(name)) return [name.text]; + if (ts.isObjectBindingPattern(name) || ts.isArrayBindingPattern(name)) { + return name.elements.flatMap(element => ts.isBindingElement(element) ? bindingNames(element.name) : []); + } + return []; +} + +/** Parser-only TypeScript extraction intentionally reports no validity diagnostics. */ +export function extractTypeScript(input: ExtractorInput): ExtractorOutput { + try { + const sourceFile = ts.createSourceFile(input.relPath, input.content, ts.ScriptTarget.Latest, true, scriptKind(input.relPath)); + const nodes: GraphNode[] = []; + const edges: ExtractorOutput["edges"] = []; + const importRefs: ImportRef[] = []; + const source = (node: ts.Node) => { + const position = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)); + return { path: input.relPath, line: position.line + 1, column: position.character + 1 }; + }; + const add = (name: string, node: ts.Node, symbolKind: SymbolKind, attributes: Record = {}) => { + const location = source(node); + const symbol: GraphNode = { + id: symbolNodeId(input.relPath, name), kind: "symbol", name, owner: "file", ownerPath: input.relPath, + source: location, attributes: { symbolKind, exported: "true", ...attributes }, + }; + nodes.push(symbol); + edges.push({ + id: edgeId("contains", input.fileNodeId, symbol.id), kind: "contains", from: input.fileNodeId, to: symbol.id, + provenance: "extracted", owner: "file", ownerPath: input.relPath, source: location, attributes: {}, + }); + }; + const addReference = (node: ts.Node, specifier: string, kind: ImportRef["kind"], typeOnly = false) => { + const candidates = importCandidates(input.relPath, specifier); + if (candidates.length === 0) return; + const location = source(node); + importRefs.push({ kind, specifier, candidates, line: location.line, column: location.column, typeOnly }); + }; + const visit = (node: ts.Node): void => { + if (ts.isImportDeclaration(node) && ts.isStringLiteral(node.moduleSpecifier)) { + const clause = node.importClause; + const namedBindings = clause?.namedBindings; + const namedTypeOnly = namedBindings && ts.isNamedImports(namedBindings) + && namedBindings.elements.length > 0 + && namedBindings.elements.every(element => element.isTypeOnly); + addReference(node, node.moduleSpecifier.text, "imports", !!clause?.isTypeOnly || !!namedTypeOnly); + } else if (ts.isImportEqualsDeclaration(node) && ts.isExternalModuleReference(node.moduleReference) && ts.isStringLiteral(node.moduleReference.expression)) { + addReference(node, node.moduleReference.expression.text, "imports"); + } else if (ts.isExportDeclaration(node)) { + if (node.moduleSpecifier && ts.isStringLiteral(node.moduleSpecifier)) addReference(node, node.moduleSpecifier.text, "re-exports", !!node.isTypeOnly); + if (node.exportClause && ts.isNamedExports(node.exportClause)) { + for (const element of node.exportClause.elements) { + add(element.name.text, element, "alias", { + localName: (element.propertyName ?? element.name).text, + ...(node.isTypeOnly || element.isTypeOnly ? { typeOnly: "true" } : {}), + ...(node.moduleSpecifier ? { reExportSpecifier: element.getText(sourceFile) } : {}), + }); + } + } else if (node.exportClause && ts.isNamespaceExport(node.exportClause)) { + add(node.exportClause.name.text, node.exportClause, "alias", node.moduleSpecifier ? { reExportSpecifier: node.exportClause.getText(sourceFile) } : {}); + } + } else if (ts.isExportAssignment(node)) { + if (node.isExportEquals) add("export=", node, "alias", { localName: node.expression.getText(sourceFile) }); + else add("default", node, ts.isClassExpression(node.expression) ? "class" : "variable", { defaultExport: "true" }); + } else if (isExported(node)) { + if ((ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node)) && isDefault(node)) { + add("default", node, kindOf(node), { defaultExport: "true", ...(node.name ? { localName: node.name.text } : {}) }); + } else if ((ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node) || ts.isInterfaceDeclaration(node) + || ts.isTypeAliasDeclaration(node) || ts.isEnumDeclaration(node) || ts.isModuleDeclaration(node)) && node.name) { + add(node.name.text, node, kindOf(node)); + } else if (ts.isVariableStatement(node)) { + for (const declaration of node.declarationList.declarations) { + for (const name of bindingNames(declaration.name)) add(name, declaration, "variable", ts.isIdentifier(declaration.name) ? {} : { binding: "destructured" }); + } + } + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + return { nodes, edges, importRefs }; + } catch { + // Source text is never a graph-build error; the file node survives at the dispatcher boundary. + return { nodes: [], edges: [], importRefs: [] }; + } +} diff --git a/packages/core/src/knowledge-graph/file-discovery.ts b/packages/core/src/knowledge-graph/file-discovery.ts new file mode 100644 index 0000000000..2161bbf2ef --- /dev/null +++ b/packages/core/src/knowledge-graph/file-discovery.ts @@ -0,0 +1,85 @@ +import { readdir, stat } from "node:fs/promises"; +import { join, relative } from "node:path"; +import { normalizeRelPath } from "./graph-types.js"; + +export const DEFAULT_EXCLUDED_DIRECTORIES = ["node_modules", "dist", "dist-electron", "coverage", ".git", ".fusion", ".fusion-knowledge", ".worktrees", ".pi", ".history", ".gate-bundle", "build", "android", "public", "locales"]; +export const DEFAULT_SOURCE_ROOTS = ["packages/*/src", "packages/dashboard/app", "packages/*/scripts", "scripts", "plugins"]; +export const DEFAULT_MARKDOWN_ROOTS = ["docs", "AGENTS.md", "CONCEPTS.md"]; +const sourceExtensions = new Set([".ts", ".tsx", ".mjs", ".cjs", ".js"]); +const markdownExtensions = new Set([".md"]); + +/* +FNXC:KnowledgeGraph 2026-08-10-11:15: +Discovery is rooted in the explicit source-root contract rather than the repository root. The +Dashboard keeps first-party React source in packages/dashboard/app, outside src; omitting it would +silently lose that component and FNXC-rationale surface from the deterministic graph. +*/ +export interface FileDiscoveryOptions { + excludedDirectories?: string[]; + sourceRoots?: string[]; + markdownRoots?: string[]; +} + +async function existingDirectories(root: string, patterns: readonly string[]): Promise { + const results = new Set(); + for (const pattern of patterns) { + if (!pattern.includes("*")) { + try { if ((await stat(join(root, pattern))).isDirectory()) results.add(join(root, pattern)); } catch { /* Optional roots are absent in small fixture repositories. */ } + continue; + } + const [prefix, suffix] = pattern.split("*"); + const container = join(root, prefix ?? ""); + try { + for (const entry of await readdir(container, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const candidate = join(container, entry.name, suffix ?? ""); + try { if ((await stat(candidate)).isDirectory()) results.add(candidate); } catch { /* Pattern entry has no matching child root. */ } + } + } catch { /* Optional root container is absent. */ } + } + return [...results].sort(); +} + +async function packageEntryFiles(root: string): Promise { + const entries: string[] = []; + try { + for (const pkg of await readdir(join(root, "packages"), { withFileTypes: true })) { + if (!pkg.isDirectory()) continue; + const packageRoot = join(root, "packages", pkg.name); + for (const name of await readdir(packageRoot)) { + if (name === "index.ts" || name === "bin.mjs" || name === "build.ts" || name.endsWith(".config.ts")) { + const candidate = join(packageRoot, name); + try { if ((await stat(candidate)).isFile()) entries.push(candidate); } catch { /* Directory entries can race a fixture mutation. */ } + } + } + } + } catch { /* A fixture may intentionally omit packages. */ } + return entries.sort(); +} + +/** Deterministically discover only the documented graph inputs, never generated or unrelated files. */ +export async function discoverFiles(root: string, options: FileDiscoveryOptions = {}): Promise { + const excluded = new Set(options.excludedDirectories ?? DEFAULT_EXCLUDED_DIRECTORIES); + const output = new Set(); + const add = (absolute: string) => output.add(normalizeRelPath(relative(root, absolute))); + const walk = async (absolute: string, allowedExtensions: Set): Promise => { + let entries; + try { entries = await readdir(absolute, { withFileTypes: true }); } catch { return; } + for (const entry of entries) { + const candidate = join(absolute, entry.name); + if (entry.isDirectory()) { + if (!excluded.has(entry.name)) await walk(candidate, allowedExtensions); + } else if (entry.isFile() && allowedExtensions.has(entry.name.slice(entry.name.lastIndexOf("."))) && !entry.name.endsWith(".d.ts")) { + add(candidate); + } + } + }; + + for (const directory of await existingDirectories(root, options.sourceRoots ?? DEFAULT_SOURCE_ROOTS)) await walk(directory, sourceExtensions); + for (const directory of await existingDirectories(root, options.markdownRoots ?? DEFAULT_MARKDOWN_ROOTS.filter(path => path === "docs"))) await walk(directory, markdownExtensions); + for (const file of options.markdownRoots ?? DEFAULT_MARKDOWN_ROOTS.filter(path => path !== "docs")) { + try { if ((await stat(join(root, file))).isFile()) add(join(root, file)); } catch { /* Optional root document is absent. */ } + } + for (const file of await packageEntryFiles(root)) add(file); + return [...output].sort(); +} diff --git a/packages/core/src/knowledge-graph/graph-builder.ts b/packages/core/src/knowledge-graph/graph-builder.ts new file mode 100644 index 0000000000..6d3b0b91dc --- /dev/null +++ b/packages/core/src/knowledge-graph/graph-builder.ts @@ -0,0 +1,141 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { deriveModules } from "./derive-modules.js"; +import { discoverFiles, type FileDiscoveryOptions } from "./file-discovery.js"; +import { extractFile } from "./extract-file.js"; +import { fingerprintContent } from "./graph-manifest.js"; +import { + assertNoInternalDuplicateNodes, + edgeId, + fileNodeId, + KNOWLEDGE_GRAPH_SCHEMA_VERSION, + KNOWLEDGE_GRAPH_EXTRACTOR_VERSION, + type ExtractorDependencies, + type GraphEdge, + type GraphManifest, + type KnowledgeGraph, +} from "./graph-types.js"; +import { selectImportTarget } from "./resolve-imports.js"; +import { loadArtifacts, resolveKnowledgeGraphDir, writeArtifacts, type RecoveryReason } from "./graph-store.js"; + +export interface BuildKnowledgeGraphOptions { + projectRoot: string; + /** Defaults to the tracked artifact location under projectRoot. */ + graphDir?: string; + force?: boolean; + deps?: ExtractorDependencies; + extractFile?: typeof extractFile; + logger?: (message: string) => void; + discovery?: FileDiscoveryOptions; +} + +/** + * Rebuild the content-pure file layer incrementally, then regenerate cross-file and derived facts. + * Import edges are deliberately synthesized from persisted references on every run: an added or + * removed target must not require parsing an otherwise unchanged importer. + */ +/* +FNXC:KnowledgeGraph 2026-08-10-11:07: +The programmatic builder defaults to the tracked graph location just like the CLI. Callers may +supply another contained directory for tests or project policy, but an omitted option must not +turn a normal build into an undefined-path I/O failure. +*/ +export async function buildKnowledgeGraph(options: BuildKnowledgeGraphOptions) { + const graphDir = resolveKnowledgeGraphDir(options.projectRoot, options.graphDir); + const files = await discoverFiles(options.projectRoot, options.discovery); + const prior = options.force + ? { ok: false as const, recoveryReason: "forced" as RecoveryReason } + : await loadArtifacts(graphDir, options.logger); + const graph: KnowledgeGraph = prior.ok + ? { + schemaVersion: prior.graph.schemaVersion, + nodes: prior.graph.nodes.filter(node => node.owner !== "derived"), + edges: prior.graph.edges.filter(edge => edge.owner !== "derived" && edge.kind !== "imports" && edge.kind !== "re-exports"), + } + : { schemaVersion: KNOWLEDGE_GRAPH_SCHEMA_VERSION, nodes: [], edges: [] }; + const manifest: GraphManifest = prior.ok + ? { ...prior.manifest, files: { ...prior.manifest.files } } + : { schemaVersion: KNOWLEDGE_GRAPH_SCHEMA_VERSION, extractorVersion: KNOWLEDGE_GRAPH_EXTRACTOR_VERSION, files: {} }; + const oldPaths = new Set(Object.keys(manifest.files)); + let parsedFiles = 0; + let reusedFiles = 0; + let addedFiles = 0; + + for (const relPath of files) { + const bytes = await readFile(resolve(options.projectRoot, relPath)); + const content = bytes.toString("utf8"); + const hash = fingerprintContent(bytes); + const previous = manifest.files[relPath]; + if (prior.ok && previous?.hash === hash) { + reusedFiles++; + continue; + } + if (!previous) addedFiles++; + parsedFiles++; + graph.nodes = graph.nodes.filter(node => node.ownerPath !== relPath); + graph.edges = graph.edges.filter(edge => edge.ownerPath !== relPath); + const result = (options.extractFile ?? extractFile)({ relPath, content }, options.deps); + graph.nodes.push(...result.nodes); + graph.edges.push(...result.edges); + manifest.files[relPath] = { hash, ...(result.importRefs.length > 0 ? { importRefs: result.importRefs } : {}) }; + } + + let deletedFiles = 0; + for (const relPath of oldPaths) { + if (files.includes(relPath)) continue; + deletedFiles++; + delete manifest.files[relPath]; + graph.nodes = graph.nodes.filter(node => node.ownerPath !== relPath); + graph.edges = graph.edges.filter(edge => edge.ownerPath !== relPath); + } + + const discovered = new Set(files); + for (const relPath of files) { + for (const reference of manifest.files[relPath]?.importRefs ?? []) { + const target = selectImportTarget(reference.candidates, discovered); + if (!target) continue; + const edge: GraphEdge = { + id: edgeId(reference.kind, fileNodeId(relPath), fileNodeId(target)), + kind: reference.kind, + from: fileNodeId(relPath), + to: fileNodeId(target), + provenance: "extracted", + owner: "file", + ownerPath: relPath, + source: { path: relPath, line: reference.line, column: reference.column }, + attributes: { specifier: reference.specifier, ...(reference.typeOnly ? { typeOnly: "true" } : {}) }, + }; + if (!graph.edges.some(existing => existing.id === edge.id)) graph.edges.push(edge); + } + } + + const derived = deriveModules(graph.nodes); + graph.nodes.push(...derived.nodes); + graph.edges.push(...derived.edges); + const nodeIds = new Set(graph.nodes.map(node => node.id)); + graph.edges = graph.edges.filter(edge => nodeIds.has(edge.from) && nodeIds.has(edge.to)); + + // Validate the deliberately narrow internal-invariant throw surface before any defensive + // normalization can hide a cross-kind or cross-owner collision. Same-owner symbol duplicates + // are collapsed by extractFile, so source content cannot reach this check as a build failure. + assertNoInternalDuplicateNodes(graph.nodes); + graph.nodes.sort((left, right) => left.id.localeCompare(right.id)); + graph.edges.sort((left, right) => left.from.localeCompare(right.from) || left.to.localeCompare(right.to) || left.kind.localeCompare(right.kind) || left.id.localeCompare(right.id)); + const written = await writeArtifacts(graphDir, graph, manifest); + + return { + graph, + manifest, + changed: written.changed, + stats: { + parsedFiles, + reusedFiles, + prunedFiles: deletedFiles, + addedFiles, + deletedFiles, + synthesizedImportEdges: graph.edges.filter(edge => edge.kind === "imports" || edge.kind === "re-exports").length, + derivedModuleCount: derived.nodes.length, + recoveryReason: prior.ok ? null : prior.recoveryReason, + }, + }; +} diff --git a/packages/core/src/knowledge-graph/graph-manifest.ts b/packages/core/src/knowledge-graph/graph-manifest.ts new file mode 100644 index 0000000000..0fcd424b64 --- /dev/null +++ b/packages/core/src/knowledge-graph/graph-manifest.ts @@ -0,0 +1,4 @@ +import { createHash } from "node:crypto"; + +/** Hash raw file bytes; the manifest must not depend on platform text decoding. */ +export const fingerprintContent = (content: string | Uint8Array): string => createHash("sha256").update(content).digest("hex"); diff --git a/packages/core/src/knowledge-graph/graph-query.ts b/packages/core/src/knowledge-graph/graph-query.ts new file mode 100644 index 0000000000..1a05e5c0ca --- /dev/null +++ b/packages/core/src/knowledge-graph/graph-query.ts @@ -0,0 +1,126 @@ +import { + KnowledgeGraphError, + type EdgeKind, + type GraphEdge, + type GraphNode, + type GraphOwner, + type KnowledgeGraph, + type SymbolKind, +} from "./graph-types.js"; + +export interface NodeFilter { + kinds?: GraphNode["kind"][]; + pathPrefix?: string; + idPrefix?: string; + namePattern?: string | RegExp; + fnxcArea?: string; + symbolKind?: SymbolKind; + owner?: GraphOwner; + limit?: number; +} + +export interface NeighborOptions { + direction?: "out" | "in" | "both"; + edgeKinds?: EdgeKind[]; + depth?: number; +} + +export interface NeighborResult { + node: GraphNode; + distance: number; + /** Every connecting edge is retained so callers can inspect provenance and source anchors. */ + edges: GraphEdge[]; +} + +type Adjacency = { nodes: Map; out: Map; in: Map }; +const adjacencyCache = new WeakMap(); + +/** Build the deterministic adjacency index once per graph rather than scanning all edges per hop. */ +function adjacency(graph: KnowledgeGraph): Adjacency { + const cached = adjacencyCache.get(graph); + if (cached) return cached; + const index: Adjacency = { nodes: new Map(graph.nodes.map(node => [node.id, node])), out: new Map(), in: new Map() }; + for (const edge of [...graph.edges].sort((left, right) => left.id.localeCompare(right.id))) { + const out = index.out.get(edge.from) ?? []; + out.push(edge); + index.out.set(edge.from, out); + const incoming = index.in.get(edge.to) ?? []; + incoming.push(edge); + index.in.set(edge.to, incoming); + } + adjacencyCache.set(graph, index); + return index; +} + +export function queryNodes(graph: KnowledgeGraph, filter: NodeFilter = {}): GraphNode[] { + const pattern = typeof filter.namePattern === "string" ? new RegExp(filter.namePattern, "i") : filter.namePattern; + return graph.nodes + .filter(node => (!filter.kinds || filter.kinds.includes(node.kind)) + && (!filter.pathPrefix || node.ownerPath.startsWith(filter.pathPrefix)) + && (!filter.idPrefix || node.id.startsWith(filter.idPrefix)) + && (!pattern || (pattern.lastIndex = 0, pattern.test(node.name))) + && (!filter.fnxcArea || node.attributes.fnxcArea === filter.fnxcArea) + && (!filter.symbolKind || node.attributes.symbolKind === filter.symbolKind) + && (!filter.owner || node.owner === filter.owner)) + .sort((left, right) => left.id.localeCompare(right.id)) + .slice(0, filter.limit); +} + +/** + * FNXC:KnowledgeGraph 2026-08-10-10:53: + * Query results expose complete edges so downstream recall and MCP layers can distinguish parser + * extracted structure from the inferred provenance reserved for the memory agent. + */ +export function neighbors(graph: KnowledgeGraph, nodeId: string, options: NeighborOptions = {}): NeighborResult[] { + const index = adjacency(graph); + const direction = options.direction ?? "out"; + const maximumDepth = Math.max(0, options.depth ?? 1); + const visited = new Set([nodeId]); + const queue: Array<{ id: string; distance: number }> = [{ id: nodeId, distance: 0 }]; + const results = new Map(); + + while (queue.length > 0) { + const current = queue.shift()!; + if (current.distance >= maximumDepth) continue; + const candidates = [ + ...(direction === "in" ? [] : index.out.get(current.id) ?? []), + ...(direction === "out" ? [] : index.in.get(current.id) ?? []), + ].filter(edge => !options.edgeKinds || options.edgeKinds.includes(edge.kind)).sort((left, right) => left.id.localeCompare(right.id)); + for (const edge of candidates) { + const next = edge.from === current.id ? edge.to : edge.from; + const node = index.nodes.get(next); + if (!node) continue; + const existing = results.get(next); + if (existing && existing.distance === current.distance + 1) existing.edges.push(edge); + if (visited.has(next)) continue; + visited.add(next); + results.set(next, { node, distance: current.distance + 1, edges: [edge] }); + queue.push({ id: next, distance: current.distance + 1 }); + } + } + return [...results.values()] + .map(result => ({ ...result, edges: result.edges.sort((left, right) => left.id.localeCompare(right.id)) })) + .sort((left, right) => left.distance - right.distance || left.node.id.localeCompare(right.node.id)); +} + +export function shortestPath(graph: KnowledgeGraph, fromId: string, toId: string): { nodes: GraphNode[]; edges: GraphEdge[] } | null { + const index = adjacency(graph); + if (!index.nodes.has(fromId) || !index.nodes.has(toId)) throw new KnowledgeGraphError("Unknown graph node"); + if (fromId === toId) return { nodes: [index.nodes.get(fromId)!], edges: [] }; + const queue: Array<{ id: string; nodes: string[]; edges: GraphEdge[] }> = [{ id: fromId, nodes: [fromId], edges: [] }]; + const visited = new Set([fromId]); + while (queue.length > 0) { + const current = queue.shift()!; + const candidates = [...(index.out.get(current.id) ?? []), ...(index.in.get(current.id) ?? [])].sort((left, right) => left.id.localeCompare(right.id)); + for (const edge of candidates) { + const next = edge.from === current.id ? edge.to : edge.from; + if (visited.has(next) || !index.nodes.has(next)) continue; + const nodeIds = [...current.nodes, next]; + const edges = [...current.edges, edge]; + if (next === toId) return { nodes: nodeIds.map(id => index.nodes.get(id)!), edges }; + visited.add(next); + queue.push({ id: next, nodes: nodeIds, edges }); + } + } + return null; +} diff --git a/packages/core/src/knowledge-graph/graph-serialization.ts b/packages/core/src/knowledge-graph/graph-serialization.ts new file mode 100644 index 0000000000..71208722f3 --- /dev/null +++ b/packages/core/src/knowledge-graph/graph-serialization.ts @@ -0,0 +1,134 @@ +import { + KNOWLEDGE_GRAPH_EXTRACTOR_VERSION, + KNOWLEDGE_GRAPH_SCHEMA_VERSION, + type EdgeKind, + type GraphEdge, + type GraphManifest, + type GraphManifestEntry, + type GraphNode, + type GraphNodeKind, + type GraphOwner, + type ImportRef, + type KnowledgeGraph, + normalizeRelPath, +} from "./graph-types.js"; + +function ordered(value: unknown): unknown { + if (Array.isArray(value)) return value.map(ordered); + if (value && typeof value === "object") { + const entries = Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)); + /* + FNXC:KnowledgeGraph 2026-08-10-11:38: + Artifact roots are self-describing, so schemaVersion remains the first serialized key while + nested maps stay recursively sorted for byte-stable commits. + */ + const schema = entries.find(([key]) => key === "schemaVersion"); + const rootOrdered = schema ? [schema, ...entries.filter(([key]) => key !== "schemaVersion")] : entries; + return Object.fromEntries(rootOrdered.map(([key, entry]) => [key, ordered(entry)])); + } + return value; +} + +export const canonicalJson = (value: unknown) => `${JSON.stringify(ordered(value), null, 2)}\n`; + +export function serializeGraph(graph: KnowledgeGraph): { nodes: string; edges: string } { + return { + nodes: canonicalJson({ schemaVersion: KNOWLEDGE_GRAPH_SCHEMA_VERSION, nodes: [...graph.nodes].sort((a, b) => a.id.localeCompare(b.id)) }), + edges: canonicalJson({ schemaVersion: KNOWLEDGE_GRAPH_SCHEMA_VERSION, edges: [...graph.edges].sort((a, b) => a.from.localeCompare(b.from) || a.to.localeCompare(b.to) || a.kind.localeCompare(b.kind) || a.id.localeCompare(b.id)) }), + }; +} + +export const serializeManifest = (manifest: GraphManifest) => canonicalJson({ + schemaVersion: KNOWLEDGE_GRAPH_SCHEMA_VERSION, + extractorVersion: KNOWLEDGE_GRAPH_EXTRACTOR_VERSION, + files: manifest.files, +}); + +const owners = new Set(["file", "derived"]); +const nodeKinds = new Set(["file", "module", "symbol", "doc-concept", "rationale"]); +const edgeKinds = new Set(["contains", "imports", "re-exports"]); +const hasStringMap = (value: unknown): value is Record => !!value && typeof value === "object" && !Array.isArray(value) && Object.values(value).every(entry => typeof entry === "string"); +const validPath = (value: unknown) => { + if (typeof value !== "string" || value.length === 0) return false; + try { + /* + FNXC:KnowledgeGraph 2026-08-10-12:06: + Persisted identities must already be canonical; normalizing on load could silently reuse a + foreign owner rather than triggering the full-rebuild recovery path. + */ + return normalizeRelPath(value) === value; + } catch { + return false; + } +}; +const validSource = (value: unknown) => !!value && typeof value === "object" && validPath((value as GraphNode["source"]).path) && Number.isInteger((value as GraphNode["source"]).line) && (value as GraphNode["source"]).line >= 1 && Number.isInteger((value as GraphNode["source"]).column) && (value as GraphNode["source"]).column >= 1; + +function validNode(value: unknown): value is GraphNode { + if (!value || typeof value !== "object") return false; + const node = value as GraphNode; + return typeof node.id === "string" && node.id.length > 0 && nodeKinds.has(node.kind) && typeof node.name === "string" + && owners.has(node.owner) && validPath(node.ownerPath) && validSource(node.source) && hasStringMap(node.attributes); +} + +function validEdge(value: unknown): value is GraphEdge { + if (!value || typeof value !== "object") return false; + const edge = value as GraphEdge; + return typeof edge.id === "string" && edge.id.length > 0 && edgeKinds.has(edge.kind) && typeof edge.from === "string" && typeof edge.to === "string" + && (edge.provenance === "extracted" || edge.provenance === "inferred") && owners.has(edge.owner) && validPath(edge.ownerPath) + && validSource(edge.source) && hasStringMap(edge.attributes); +} + +function validImportRef(value: unknown): value is ImportRef { + if (!value || typeof value !== "object") return false; + const ref = value as ImportRef; + return (ref.kind === "imports" || ref.kind === "re-exports") && typeof ref.specifier === "string" && Array.isArray(ref.candidates) + && ref.candidates.every(validPath) && Number.isInteger(ref.line) && ref.line >= 1 && Number.isInteger(ref.column) && ref.column >= 1 && typeof ref.typeOnly === "boolean"; +} + +function validManifestEntry(value: unknown): value is GraphManifestEntry { + if (!value || typeof value !== "object") return false; + const entry = value as GraphManifestEntry; + return typeof entry.hash === "string" && /^[a-f0-9]{64}$/.test(entry.hash) + && (entry.importRefs === undefined || (Array.isArray(entry.importRefs) && entry.importRefs.every(validImportRef))); +} + +function hasExactKeys(value: object, keys: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + return actual.length === keys.length && actual.every((key, index) => key === [...keys].sort()[index]); +} + +function validManifest(value: unknown): value is GraphManifest { + if (!value || typeof value !== "object") return false; + const manifest = value as GraphManifest; + return hasExactKeys(manifest, ["schemaVersion", "extractorVersion", "files"]) + && manifest.schemaVersion === KNOWLEDGE_GRAPH_SCHEMA_VERSION && manifest.extractorVersion === KNOWLEDGE_GRAPH_EXTRACTOR_VERSION + && !!manifest.files && typeof manifest.files === "object" && !Array.isArray(manifest.files) + && Object.entries(manifest.files).every(([path, entry]) => validPath(path) && validManifestEntry(entry)); +} + +/** Invalid persisted shapes return a recovery result; corrupted artifacts never become trusted graph state. */ +export function deserializeArtifacts(nodesText: string, edgesText: string, manifestText: string): { ok: true; graph: KnowledgeGraph; manifest: GraphManifest } | { ok: false; reason: "invalid-artifact" | "version-mismatch" } { + try { + const nodesPayload: unknown = JSON.parse(nodesText); + const edgesPayload: unknown = JSON.parse(edgesText); + const manifestPayload: unknown = JSON.parse(manifestText); + // A missing version is a malformed payload, not an older compatible artifact. Reserve the + // version-mismatch recovery reason for otherwise-shaped payloads that explicitly declare a + // different version, so operators can distinguish corruption from an intentional upgrade. + const versionMismatch = [nodesPayload, edgesPayload, manifestPayload].some(value => !!value && typeof value === "object" + && "schemaVersion" in value && (value as { schemaVersion?: unknown }).schemaVersion !== KNOWLEDGE_GRAPH_SCHEMA_VERSION) + || !!manifestPayload && typeof manifestPayload === "object" && "extractorVersion" in manifestPayload + && (manifestPayload as { extractorVersion?: unknown }).extractorVersion !== KNOWLEDGE_GRAPH_EXTRACTOR_VERSION; + if (versionMismatch) return { ok: false, reason: "version-mismatch" }; + if (!nodesPayload || typeof nodesPayload !== "object" || !edgesPayload || typeof edgesPayload !== "object" + || !hasExactKeys(nodesPayload, ["schemaVersion", "nodes"]) || !hasExactKeys(edgesPayload, ["schemaVersion", "edges"]) + || !Array.isArray((nodesPayload as { nodes?: unknown }).nodes) || !Array.isArray((edgesPayload as { edges?: unknown }).edges) + || !(nodesPayload as { nodes: unknown[] }).nodes.every(validNode) || !(edgesPayload as { edges: unknown[] }).edges.every(validEdge) || !validManifest(manifestPayload)) { + return { ok: false, reason: "invalid-artifact" }; + } + return { ok: true, graph: { schemaVersion: KNOWLEDGE_GRAPH_SCHEMA_VERSION, nodes: (nodesPayload as { nodes: GraphNode[] }).nodes, edges: (edgesPayload as { edges: GraphEdge[] }).edges }, manifest: manifestPayload }; + } catch { + return { ok: false, reason: "invalid-artifact" }; + } +} diff --git a/packages/core/src/knowledge-graph/graph-store.ts b/packages/core/src/knowledge-graph/graph-store.ts new file mode 100644 index 0000000000..c69f5556b4 --- /dev/null +++ b/packages/core/src/knowledge-graph/graph-store.ts @@ -0,0 +1,97 @@ +import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; +import { randomUUID } from "node:crypto"; +import { deserializeArtifacts, serializeGraph, serializeManifest } from "./graph-serialization.js"; +import { edgeId, fileNodeId, KnowledgeGraphError, moduleCanonicalPath, moduleNodeId, type GraphEdge, type GraphManifest, type GraphNode, type KnowledgeGraph } from "./graph-types.js"; + +/* +FNXC:KnowledgeGraph 2026-08-10-10:00: +The graph is review-visible and committable, while .fusion is ignored, so artifacts live under +.fusion-knowledge/graph. FR-29/FR-34 capability bundles do not have a format yet and are deferred. +*/ +export const DEFAULT_KNOWLEDGE_GRAPH_DIR = ".fusion-knowledge/graph"; +export type RecoveryReason="missing-artifact"|"invalid-artifact"|"version-mismatch"|"inconsistent-artifact"|"forced"; +export type ArtifactLoadResult={ok:true;graph:KnowledgeGraph;manifest:GraphManifest;recoveryReason:null}|{ok:false;recoveryReason:RecoveryReason}; +export function resolveKnowledgeGraphDir(projectRoot:string, configured?:string):string { const root=resolve(projectRoot), target=resolve(root,configured||DEFAULT_KNOWLEDGE_GRAPH_DIR), rel=relative(root,target); if(isAbsolute(rel)||rel===".."||rel.startsWith(`..${sep}`)) throw new KnowledgeGraphError("Knowledge graph directory escapes project root");return target; } +function isMissing(error:unknown){return (error as NodeJS.ErrnoException).code==="ENOENT";} +function consistent(graph: KnowledgeGraph, manifest: GraphManifest): boolean { + const allFileNodes = graph.nodes.filter(node => node.owner === "file" && node.kind === "file"); + const fileNodes = new Map(allFileNodes.map(node => [node.ownerPath, node])); + /* + FNXC:KnowledgeGraph 2026-08-10-12:06: + A Map alone hides a second, differently-id'd file root with the same ownership path. Reject it + so a malformed artifact always takes the safe full-rebuild path instead of reusing rogue nodes. + */ + if (allFileNodes.length !== fileNodes.size || fileNodes.size !== Object.keys(manifest.files).length) return false; + if ([...fileNodes].some(([path, node]) => node.id !== `file:${path}` || !manifest.files[path])) return false; + if (graph.nodes.some(node => node.owner === "file" && !manifest.files[node.ownerPath])) return false; + + // Artifact JSON is an untrusted incremental cache. Checking only ownerPath permits a corrupt + // symbol/doc/rationale id (or a cross-file source anchor) to survive every no-op rebuild. + const expectedNodeId = (node: GraphNode): string | undefined => { + switch (node.kind) { + case "file": return `file:${node.ownerPath}`; + case "module": return `module:${node.ownerPath}`; + case "symbol": return node.id.startsWith(`symbol:${node.ownerPath}#`) ? node.id : undefined; + case "doc-concept": return node.id.startsWith(`doc:${node.ownerPath}#`) ? node.id : undefined; + case "rationale": return node.id.startsWith(`rationale:${node.ownerPath}#`) ? node.id : undefined; + } + }; + /* + FNXC:KnowledgeGraph 2026-08-10-12:38: + Persisted artifacts are an untrusted incremental cache. Validate ownership classes as well as ids: + only module facts are derived, and derived edges are module-scoped contains edges. Otherwise a + malformed cache can survive an incremental reuse path with a node that pruning can never own. + */ + const moduleFiles = allFileNodes + .map(node => node.ownerPath) + .filter(path => /\.tsx?$/.test(path)); + const hasSyntheticMarker = (item: GraphNode | GraphEdge) => "syntheticSource" in item.attributes; + const isSyntheticAnchor = (item: GraphNode | GraphEdge, path: string) => item.attributes.syntheticSource === "true" + && item.source.path === path && item.source.line === 1 && item.source.column === 1; + /* + FNXC:KnowledgeGraph 2026-08-10-12:48: + A persisted graph is an untrusted incremental cache, so synthetic anchors must be validated before + reuse. File/module facts and module-scoped edges have no parsed text position; accepting a fake + location or a synthetic marker on a real symbol/comment edge would preserve false provenance. + */ + if (graph.nodes.some(node => { + if (expectedNodeId(node) !== node.id) return true; + if (node.kind === "file") return node.owner !== "file" || !isSyntheticAnchor(node, node.ownerPath); + if (node.kind === "module") return node.owner !== "derived" + || !isSyntheticAnchor(node, moduleCanonicalPath(node.ownerPath, moduleFiles)); + return node.owner !== "file" || node.source.path !== node.ownerPath || hasSyntheticMarker(node); + })) return false; + if (graph.edges.some(edge => { + if (edge.id !== edgeId(edge.kind, edge.from, edge.to)) return true; + if (edge.owner === "derived") return edge.kind !== "contains" + || edge.from !== moduleNodeId(edge.ownerPath) + || !isSyntheticAnchor(edge, moduleCanonicalPath(edge.ownerPath, moduleFiles)); + /* + FNXC:KnowledgeGraph 2026-08-10-12:56: + Documentation hierarchy is file-owned but links a heading to its child heading. Only import + references must originate at the owning file node; rejecting the hierarchy would force a full + rebuild of every otherwise-valid graph artifact. + */ + return edge.source.path !== edge.ownerPath || hasSyntheticMarker(edge) + || ((edge.kind === "imports" || edge.kind === "re-exports") && edge.from !== fileNodeId(edge.ownerPath)); + })) return false; + + const ids = new Set(); + for (const node of graph.nodes) { + if (ids.has(node.id)) return false; + ids.add(node.id); + } + const edgeIds = new Set(); + for (const edge of graph.edges) { + if (edgeIds.has(edge.id)) return false; + edgeIds.add(edge.id); + } + return !graph.edges + .filter(edge => edge.owner !== "derived" && edge.kind !== "imports" && edge.kind !== "re-exports") + .some(edge => !manifest.files[edge.ownerPath] || !ids.has(edge.from) || !ids.has(edge.to)); +} +export async function loadArtifacts(graphDir:string, logger:(message:string)=>void=()=>{}):Promise{const paths=["nodes.json","edges.json","manifest.json"].map(x=>resolve(graphDir,x));let text:string[];try{text=await Promise.all(paths.map(p=>readFile(p,"utf8")));}catch(error){if(isMissing(error))return{ok:false,recoveryReason:"missing-artifact"};throw new KnowledgeGraphError(`Unable to read knowledge graph artifacts: ${error instanceof Error?error.message:String(error)}`);}const decoded=deserializeArtifacts(text[0]!,text[1]!,text[2]!);if(!decoded.ok){logger(`Knowledge graph artifact recovery: ${decoded.reason}`);return{ok:false,recoveryReason:decoded.reason};}if(!consistent(decoded.graph,decoded.manifest)){logger("Knowledge graph artifact recovery: inconsistent-artifact");return{ok:false,recoveryReason:"inconsistent-artifact"};}return{ok:true,graph:decoded.graph,manifest:decoded.manifest,recoveryReason:null};} +async function writeIfChanged(path:string, content:string):Promise{try{if(await readFile(path,"utf8")===content)return false;}catch(error){if(!isMissing(error))throw error;}await mkdir(dirname(path),{recursive:true});const temp=`${path}.${randomUUID()}.tmp`;try{await writeFile(temp,content,"utf8");await rename(temp,path);return true;}catch(error){await unlink(temp).catch(()=>{});throw error;}} +/** Manifest is last: a torn write can only force a safe full rebuild, never validate unwritten nodes. */ +export async function writeArtifacts(graphDir:string,graph:KnowledgeGraph,manifest:GraphManifest):Promise<{nodes:boolean;edges:boolean;manifest:boolean;changed:boolean}>{const bytes=serializeGraph(graph),nodes=await writeIfChanged(resolve(graphDir,"nodes.json"),bytes.nodes),edges=await writeIfChanged(resolve(graphDir,"edges.json"),bytes.edges),manifestChanged=await writeIfChanged(resolve(graphDir,"manifest.json"),serializeManifest(manifest));return{nodes,edges,manifest:manifestChanged,changed:nodes||edges||manifestChanged};} diff --git a/packages/core/src/knowledge-graph/graph-types.ts b/packages/core/src/knowledge-graph/graph-types.ts new file mode 100644 index 0000000000..e1747aed61 --- /dev/null +++ b/packages/core/src/knowledge-graph/graph-types.ts @@ -0,0 +1,54 @@ +import { basename, posix } from "node:path"; + +/* +FNXC:KnowledgeGraph 2026-08-10-10:00: +FN-8921 establishes a deterministic, committable structure graph for the memory epic. Part 1 records +only parser-extracted facts; inferred edges, embeddings, MCP tools, and capability bundles are deferred. +*/ +/* +FNXC:KnowledgeGraph 2026-08-10-12:24: +Schema version 2 records edge identities as their kind and already-canonical node ids joined by pipes. +Re-escaping whole node ids changed structural `#` separators and produced a different persisted grammar. +*/ +export const KNOWLEDGE_GRAPH_SCHEMA_VERSION = 2; +export const KNOWLEDGE_GRAPH_EXTRACTOR_VERSION = 1; +export type EdgeProvenance = "extracted" | "inferred"; +export type GraphOwner = "file" | "derived"; +export type GraphNodeKind = "file" | "module" | "symbol" | "doc-concept" | "rationale"; +export type SymbolKind = "function" | "class" | "interface" | "type-alias" | "enum" | "variable" | "namespace" | "alias"; +export type EdgeKind = "contains" | "imports" | "re-exports"; +export interface SourceLocation { path: string; line: number; column: number } +export interface GraphNode { id: string; kind: GraphNodeKind; name: string; owner: GraphOwner; ownerPath: string; source: SourceLocation; attributes: Record } +export interface GraphEdge { id: string; kind: EdgeKind; from: string; to: string; provenance: EdgeProvenance; owner: GraphOwner; ownerPath: string; source: SourceLocation; attributes: Record } +export interface KnowledgeGraph { schemaVersion: number; nodes: GraphNode[]; edges: GraphEdge[] } +export interface ImportRef { kind: "imports" | "re-exports"; specifier: string; candidates: string[]; line: number; column: number; typeOnly: boolean } +export interface GraphManifestEntry { hash: string; importRefs?: ImportRef[] } +export interface GraphManifest { schemaVersion: number; extractorVersion: number; files: Record } +export type TypeScriptExtractor = (input: ExtractorInput) => ExtractorOutput; +export type MarkdownExtractor = (input: ExtractorInput) => ExtractorOutput; +export type FnxcExtractor = (input: ExtractorInput) => ExtractorOutput; +export interface ExtractorDependencies { typescript?: TypeScriptExtractor; markdown?: MarkdownExtractor; fnxc?: FnxcExtractor } +export interface ExtractorInput { relPath: string; content: string; fileNodeId: string } +export interface ExtractorOutput { nodes: GraphNode[]; edges: GraphEdge[]; importRefs: ImportRef[] } +/** Only invalid caller paths, impossible internal duplicate invariants, and load IO faults may throw this error; source content never may. */ +export class KnowledgeGraphError extends Error { constructor(message: string) { super(message); this.name = "KnowledgeGraphError"; } } +/** Normalize ids to repo-relative POSIX paths and reject caller paths which can escape the repository. */ +export function normalizeRelPath(value: string): string { const normalized=value.replace(/\\/g,"/"); if (!normalized || normalized.startsWith("/") || /^[A-Za-z]:\//.test(normalized) || normalized.split("/").includes("..")) throw new KnowledgeGraphError(`Invalid relative path: ${value}`); const result=posix.normalize(normalized).replace(/^\.\//,"").replace(/\/$/,""); if (!result || result==="." || result.startsWith("../")) throw new KnowledgeGraphError(`Invalid relative path: ${value}`); return result; } +export function escapeIdSegment(value: string): string { return value.replace(/[%#@~|\n]/g, c => ({"%":"%25","#":"%23","@":"%40","~":"%7E","|":"%7C","\n":"%0A"})[c]!); } +export const fileNodeId=(path:string)=>`file:${normalizeRelPath(path)}`; +export const moduleNodeId=(path:string)=>`module:${normalizeRelPath(path)}`; +export const symbolNodeId=(path:string,name:string)=>`symbol:${normalizeRelPath(path)}#${escapeIdSegment(name)}`; +export const docConceptNodeId=(path:string,slug:string,index:number)=>`doc:${normalizeRelPath(path)}#${escapeIdSegment(slug)}~${index}`; +export const rationaleNodeId=(path:string,area:string,stamp:string,index:number)=>`rationale:${normalizeRelPath(path)}#${escapeIdSegment(area)}@${escapeIdSegment(stamp)}~${index}`; +/** + * FNXC:KnowledgeGraph 2026-08-10-12:24: + * Compose the persisted edge identity as `||`. + * Node ids have already escaped their variable segments, including `|`; escaping a complete node id + * here would rewrite structural separators such as `#` and make persisted ids diverge from the contract. + */ +export const edgeId=(kind:EdgeKind,from:string,to:string)=>`${kind}|${from}|${to}`; +export function moduleCanonicalPath(dir:string, files: readonly string[]): string { const d=normalizeRelPath(dir); return files.includes(`${d}/index.ts`)?`${d}/index.ts`:files.includes(`${d}/index.tsx`)?`${d}/index.tsx`:d; } +/** Collapse every same-file symbol duplicate in source-position order, including malformed redeclarations. */ +export function collapseDuplicateSymbolNodes(nodes: GraphNode[]): GraphNode[] { const groups=new Map(); for(const n of nodes){if(n.kind!=="symbol"){groups.set(`${n.id}\0${groups.size}`, [n]);continue;} const a=groups.get(n.id)??[];a.push(n);groups.set(n.id,a);} const out:GraphNode[]=[];for(const list of groups.values()){list.sort((a,b)=>a.source.path.localeCompare(b.source.path)||a.source.line-b.source.line||a.source.column-b.source.column); const first=list[0]!; if(first.kind!=="symbol"){out.push(first);continue;} const kinds=[...new Set(list.map(x=>x.attributes.symbolKind).filter(Boolean))].sort(); const attributes: Record={...first.attributes,declarationCount:String(list.length)}; if(kinds.length>1) attributes.symbolKinds=kinds.join(","); else delete attributes.symbolKinds; out.push({...first,attributes});} return out.sort((a,b)=>a.id.localeCompare(b.id)); } +export function assertNoInternalDuplicateNodes(nodes: GraphNode[]): void { const seen=new Map(); for(const node of nodes){const prior=seen.get(node.id); if(prior && (prior.kind!==node.kind || prior.ownerPath!==node.ownerPath)) throw new KnowledgeGraphError(`Internal duplicate node invariant: ${node.id}`); seen.set(node.id,node);} } +export const moduleName=(dir:string)=>basename(dir); diff --git a/packages/core/src/knowledge-graph/index.ts b/packages/core/src/knowledge-graph/index.ts new file mode 100644 index 0000000000..de73f379fe --- /dev/null +++ b/packages/core/src/knowledge-graph/index.ts @@ -0,0 +1,14 @@ +/** FNXC:CodeOrganization 2026-08-10-10:00: Public deterministic knowledge-graph domain barrel. */ +export * from "./graph-types.js"; +export * from "./graph-serialization.js"; +export * from "./graph-store.js"; +export * from "./graph-manifest.js"; +export * from "./file-discovery.js"; +export * from "./extract-file.js"; +export * from "./extract-typescript.js"; +export * from "./extract-markdown.js"; +export * from "./extract-fnxc.js"; +export * from "./resolve-imports.js"; +export * from "./derive-modules.js"; +export * from "./graph-builder.js"; +export * from "./graph-query.js"; diff --git a/packages/core/src/knowledge-graph/resolve-imports.ts b/packages/core/src/knowledge-graph/resolve-imports.ts new file mode 100644 index 0000000000..ab0e0abee6 --- /dev/null +++ b/packages/core/src/knowledge-graph/resolve-imports.ts @@ -0,0 +1,4 @@ +import { posix } from "node:path"; +import { normalizeRelPath } from "./graph-types.js"; +export function importCandidates(fromRelPath:string,specifier:string):string[]{if(!specifier.startsWith("./")&&!specifier.startsWith("../"))return[];let base:string;try{base=normalizeRelPath(posix.join(posix.dirname(normalizeRelPath(fromRelPath)),specifier));}catch{return[];}if(/\.tsx?$/.test(specifier))return[base];if(/\.(?:jsx?|mjs|cjs)$/.test(specifier))return[base.replace(/\.(?:jsx?|mjs|cjs)$/, ".ts"),base.replace(/\.(?:jsx?|mjs|cjs)$/, ".tsx")];if(specifier.endsWith("/"))return[`${base}/index.ts`,`${base}/index.tsx`];return[`${base}.ts`,`${base}.tsx`,`${base}/index.ts`,`${base}/index.tsx`];} +export const selectImportTarget=(candidates:readonly string[],discovered:ReadonlySet)=>candidates.find(x=>discovered.has(x))??candidates[0]; diff --git a/packages/core/src/types/settings/settings-scope.ts b/packages/core/src/types/settings/settings-scope.ts index 62982408f0..0bdf4bae3f 100644 --- a/packages/core/src/types/settings/settings-scope.ts +++ b/packages/core/src/types/settings/settings-scope.ts @@ -2097,6 +2097,8 @@ export interface ProjectSettings { /** Directory for memory backup snapshots, relative to project root. * Default: ".fusion/backups/memory". */ memoryBackupDir?: string; + /** Directory for the committable deterministic knowledge graph. */ + knowledgeGraphDir?: string; /** Scope of memory backup snapshots. * - "project": backups `.fusion/memory` only * - "agents": backups `.fusion/agent-memory` only diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c0a130a929..d4bf77306b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -109,6 +109,9 @@ importers: react-i18next: specifier: ^17.0.8 version: 17.0.8(i18next@26.3.6(typescript@5.9.3))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(typescript@5.9.3) + typescript: + specifier: ^5.7.0 + version: 5.9.3 ws: specifier: ^8.18.0 version: 8.21.1 @@ -161,9 +164,6 @@ importers: typebox: specifier: ^1.0.0 version: 1.1.38 - typescript: - specifier: ^5.7.0 - version: 5.9.3 vitest: specifier: ^4.1.10 version: 4.1.10(@opentelemetry/api@1.9.0)(@types/node@25.9.5)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.1)(jsdom@29.1.1(@noble/hashes@2.3.0))(vite@6.4.1(@types/node@25.9.5)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0)) @@ -210,6 +210,9 @@ importers: tar: specifier: ^7.5.13 version: 7.5.13 + typescript: + specifier: ^5.7.0 + version: 5.9.3 yaml: specifier: ^2.8.3 version: 2.9.0 @@ -226,9 +229,6 @@ importers: '@vitest/coverage-v8': specifier: ^4.1.10 version: 4.1.10(vitest@4.1.10) - typescript: - specifier: ^5.7.0 - version: 5.9.3 vitest: specifier: ^4.1.10 version: 4.1.10(@opentelemetry/api@1.9.0)(@types/node@25.9.5)(@vitest/coverage-v8@4.1.10)(happy-dom@20.10.1)(jsdom@29.1.1(@noble/hashes@2.3.0))(vite@6.4.1(@types/node@25.9.5)(jiti@2.7.0)(tsx@4.21.0)(yaml@2.9.0))