From 29b27a7ebb301f3f490d6399e17d18155610d85f Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Wed, 17 Jun 2026 21:45:50 -0700 Subject: [PATCH] FN-6620: categorize Fusion tool analytics Improve Command Center tool analytics so Fusion tool calls land in meaningful buckets. - categorize Fusion task, planning, research, agent, skills, secrets, workflow, GitHub, and memory tool families - re-bucket historical tool_call rows whose stored category was missing or `other` while preserving explicit custom categories - cover categorization and aggregation behavior with core tests - add a patch changeset for the published CLI package Files changed: .changeset/fn-6620-tool-categories.md | 5 ++ packages/core/src/__tests__/tool-analytics.test.ts | 33 +++++++++++ packages/core/src/__tests__/usage-events.test.ts | 69 +++++++++++++++++++--- packages/core/src/tool-analytics.ts | 22 +++++-- packages/core/src/usage-events.ts | 66 ++++++++++++++++++++- 5 files changed, 178 insertions(+), 17 deletions(-) Fusion-Task-Id: FN-6620 Fusion-Task-Lineage: a8f2525e-7aff-4576-89e2-6b142792f380 --- .changeset/fn-6620-tool-categories.md | 5 ++ .../core/src/__tests__/tool-analytics.test.ts | 33 +++++++++ .../core/src/__tests__/usage-events.test.ts | 69 ++++++++++++++++--- packages/core/src/tool-analytics.ts | 22 ++++-- packages/core/src/usage-events.ts | 66 +++++++++++++++++- 5 files changed, 178 insertions(+), 17 deletions(-) create mode 100644 .changeset/fn-6620-tool-categories.md diff --git a/.changeset/fn-6620-tool-categories.md b/.changeset/fn-6620-tool-categories.md new file mode 100644 index 0000000000..59e4471e1b --- /dev/null +++ b/.changeset/fn-6620-tool-categories.md @@ -0,0 +1,5 @@ +--- +"@runfusion/fusion": patch +--- + +Improve Command Center tool analytics by categorizing Fusion tool families and re-bucketing historical `other` rows. diff --git a/packages/core/src/__tests__/tool-analytics.test.ts b/packages/core/src/__tests__/tool-analytics.test.ts index ac8dbbd378..ce6fb880a7 100644 --- a/packages/core/src/__tests__/tool-analytics.test.ts +++ b/packages/core/src/__tests__/tool-analytics.test.ts @@ -65,6 +65,39 @@ describe("tool-analytics", () => { ]); }); + it("re-buckets historical other tool calls by tool name while preserving explicit categories", () => { + const ts = "2026-03-01T00:00:00.000Z"; + emitUsageEvent(db, { kind: "tool_call", toolName: "fn_task_create", category: "other", ts }); + emitUsageEvent(db, { kind: "tool_call", toolName: "fn_research_run", category: "other", ts }); + emitUsageEvent(db, { kind: "tool_call", toolName: "fn_memory_append", category: null, ts }); + emitUsageEvent(db, { kind: "tool_call", toolName: "fn_mission_show", category: "other", ts }); + emitUsageEvent(db, { kind: "tool_call", toolName: "fn_skills_search", category: "other", ts }); + emitUsageEvent(db, { kind: "tool_call", toolName: "Read", category: "other", ts }); + emitUsageEvent(db, { kind: "tool_call", toolName: "Bash", category: "other", ts }); + emitUsageEvent(db, { kind: "tool_call", toolName: "Unknown", category: "other", ts }); + emitUsageEvent(db, { kind: "tool_call", toolName: null, category: "other", ts }); + emitUsageEvent(db, { kind: "tool_call", toolName: "fn_task_update", category: "custom", ts }); + + const result = aggregateToolAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + const byCategory = new Map(result.byCategory.map((row) => [row.category, row.count])); + + expect(result.toolCalls).toBe(10); + expect(byCategory).toEqual( + new Map([ + ["other", 2], + ["custom", 1], + ["edit", 1], + ["execute", 1], + ["memory", 1], + ["planning", 1], + ["read", 1], + ["research", 1], + ["skills", 1], + ]), + ); + expect(result.byCategory[0]).toEqual({ category: "other", count: 2 }); + }); + it("autonomy denominator counts a USER steer + an approval but NOT an agent steer", () => { insertTaskWithSteers(db, "task-1", [ { id: "s1", text: "do X", createdAt: "2026-03-02T00:00:00.000Z", author: "user" }, diff --git a/packages/core/src/__tests__/usage-events.test.ts b/packages/core/src/__tests__/usage-events.test.ts index 6ae70ac961..f17c2aabbe 100644 --- a/packages/core/src/__tests__/usage-events.test.ts +++ b/packages/core/src/__tests__/usage-events.test.ts @@ -88,15 +88,66 @@ describe("usage_events", () => { }); it("categorizes tool names into coarse buckets", () => { - expect(categorizeToolName("Read")).toBe("read"); - expect(categorizeToolName("Grep")).toBe("read"); - expect(categorizeToolName("Edit")).toBe("edit"); - expect(categorizeToolName("Write")).toBe("edit"); - expect(categorizeToolName("Bash")).toBe("execute"); - expect(categorizeToolName("WebFetch")).toBe("network"); - expect(categorizeToolName("Unknown")).toBe("other"); - expect(categorizeToolName(undefined)).toBe("other"); - expect(categorizeToolName(null)).toBe("other"); + const cases: Array<[string | null | undefined, string]> = [ + ["Read", "read"], + ["Grep", "read"], + ["Glob", "read"], + ["ls", "read"], + ["semantic_search", "read"], + ["fn_task_list", "read"], + ["fn_task_show", "read"], + ["fn_task_get", "read"], + ["fn_task_search", "read"], + ["fn_list_agents", "read"], + ["fn_agent_org_chart", "read"], + ["fn_task_document_read", "read"], + ["fn_research_list", "research"], + ["Edit", "edit"], + ["Write", "edit"], + ["MultiEdit", "edit"], + ["NotebookEdit", "edit"], + ["fn_task_create", "edit"], + ["fn_task_update", "edit"], + ["fn_task_attach", "edit"], + ["fn_task_archive", "edit"], + ["fn_task_document_write", "edit"], + ["Bash", "execute"], + ["execute_command", "execute"], + ["terminal", "execute"], + ["WebFetch", "network"], + ["fn_web_fetch", "network"], + ["http_request", "network"], + ["fn_mission_show", "planning"], + ["fn_milestone_add", "planning"], + ["fn_slice_activate", "planning"], + ["fn_feature_link_task", "planning"], + ["fn_goal_create", "planning"], + ["fn_task_plan", "planning"], + ["fn_research_run", "research"], + ["fn_insight_show", "research"], + ["fn_experiment_finalize", "research"], + ["fn_memory_append", "memory"], + ["fn_agent_create", "agents"], + ["fn_delegate_task", "agents"], + ["fn_skills_search", "skills"], + ["fn_secret_get", "secrets"], + ["fn_task_import_github", "github"], + ["fn_task_import_github_issue", "github"], + ["fn_task_browse_github_issues", "github"], + ["fn_workflow_create", "workflow"], + ["fn_review_spec", "workflow"], + ["mcp__server__search", "read"], + ["mcp__server__tool", "other"], + ["Unknown", "other"], + ["", "other"], + [" ", "other"], + [undefined, "other"], + [null, "other"], + ]; + + for (const [toolName, expected] of cases) { + expect(categorizeToolName(toolName), String(toolName)).toBe(expected); + } }); it("rejects a meta payload over the byte cap at write (event skipped, nothing inserted)", () => { diff --git a/packages/core/src/tool-analytics.ts b/packages/core/src/tool-analytics.ts index e1135648bd..6d1ac96ff9 100644 --- a/packages/core/src/tool-analytics.ts +++ b/packages/core/src/tool-analytics.ts @@ -1,4 +1,5 @@ import type { Database } from "./db.js"; +import { categorizeToolName } from "./usage-events.js"; import type { SteeringComment } from "./types.js"; /** @@ -75,6 +76,7 @@ interface CountRow { } interface CategoryRow { + toolName: string | null; category: string | null; count: number; } @@ -175,17 +177,27 @@ export function aggregateToolAnalytics( .get(...eventParams) as CountRow ).count; + /** + * FNXC:CommandCenter 2026-06-17-21:43: + * Historical usage rows were logged with `category = "other"` before Fusion tool families were mapped, so aggregation must re-derive those buckets from `toolName`. + * Preserve explicit non-`other` categories because external callers may already provide a deliberate custom bucket. + */ const categoryRows = db .prepare( - `SELECT category AS category, COUNT(*) AS count + `SELECT toolName AS toolName, category AS category, COUNT(*) AS count FROM usage_events WHERE kind = 'tool_call' ${rangeWhere} - GROUP BY category`, + GROUP BY toolName, category`, ) .all(...eventParams) as CategoryRow[]; - const byCategory: ToolCategoryCount[] = categoryRows - .map((r) => ({ category: r.category ?? "other", count: r.count })) - .sort((a, b) => b.count - a.count); + const categoryCounts = new Map(); + for (const row of categoryRows) { + const category = row.category && row.category !== "other" ? row.category : categorizeToolName(row.toolName); + categoryCounts.set(category, (categoryCounts.get(category) ?? 0) + row.count); + } + const byCategory: ToolCategoryCount[] = [...categoryCounts.entries()] + .map(([category, count]) => ({ category, count })) + .sort((a, b) => b.count - a.count || a.category.localeCompare(b.category)); const sessions = ( db diff --git a/packages/core/src/usage-events.ts b/packages/core/src/usage-events.ts index a2532df2a8..fbce4c90e1 100644 --- a/packages/core/src/usage-events.ts +++ b/packages/core/src/usage-events.ts @@ -99,14 +99,74 @@ interface UsageEventRow { /** * Coarse tool category derived from a tool name, for the Tools analytics area. * Pure and side-effect free; callers may also pass an explicit `category`. + * + * FNXC:CommandCenter 2026-06-17-21:35: + * Fusion agents mostly call namespaced `fn_*` tools, so Command Center analytics must bucket those families meaningfully instead of letting the Tools chart collapse into `other`. + * Keep this mapping pure and lowercase-normalized because it is used both at log-write time and when re-bucketing historical rows. */ export function categorizeToolName(toolName: string | null | undefined): string { if (!toolName) return "other"; - const name = toolName.toLowerCase(); - if (name === "read" || name === "grep" || name === "glob" || name === "ls" || name.includes("search")) { + const name = toolName.trim().toLowerCase(); + if (!name) return "other"; + + if (name.startsWith("fn_task_import_github") || name.startsWith("fn_task_browse_github")) { + return "github"; + } + if (name === "fn_web_fetch") return "network"; + if (name.startsWith("fn_secret_")) return "secrets"; + if (name.startsWith("fn_skills_")) return "skills"; + if (name.startsWith("fn_memory_")) return "memory"; + if (name === "fn_list_agents" || name === "fn_agent_org_chart") return "read"; + if (name.startsWith("fn_agent_") || name === "fn_delegate_task") return "agents"; + if ( + name.startsWith("fn_mission_") || + name.startsWith("fn_milestone_") || + name.startsWith("fn_slice_") || + name.startsWith("fn_feature_") || + name.startsWith("fn_goal_") || + name === "fn_task_plan" + ) { + return "planning"; + } + if (name.startsWith("fn_research_") || name.startsWith("fn_insight_") || name.startsWith("fn_experiment_")) { + return "research"; + } + if (name.startsWith("fn_workflow_") || name === "fn_review_spec") return "workflow"; + + if ( + name === "read" || + name === "grep" || + name === "glob" || + name === "ls" || + name.includes("search") || + name === "fn_list_agents" || + name === "fn_agent_org_chart" || + name === "fn_task_document_read" || + name.endsWith("_list") || + name.endsWith("_show") || + name.endsWith("_get") || + name.endsWith("_search") + ) { return "read"; } - if (name === "edit" || name === "write" || name === "multiedit" || name.includes("notebook")) { + if ( + name === "edit" || + name === "write" || + name === "multiedit" || + name.includes("notebook") || + name === "fn_task_create" || + name === "fn_task_update" || + name === "fn_task_attach" || + name === "fn_task_pause" || + name === "fn_task_unpause" || + name === "fn_task_retry" || + name === "fn_task_duplicate" || + name === "fn_task_refine" || + name === "fn_task_archive" || + name === "fn_task_unarchive" || + name === "fn_task_delete" || + name === "fn_task_document_write" + ) { return "edit"; } if (name === "bash" || name.includes("exec") || name.includes("command") || name.includes("terminal")) {