FN-6620: categorize Fusion tool analytics
Improve Command Center tool analytics so Fusion tool calls land in meaningful buckets. - categorize Fusion task, planning, research, agent, skills, secrets, workflow, GitHub, and memory tool families - re-bucket historical tool_call rows whose stored category was missing or `other` while preserving explicit custom categories - cover categorization and aggregation behavior with core tests - add a patch changeset for the published CLI package Files changed: .changeset/fn-6620-tool-categories.md | 5 ++ packages/core/src/__tests__/tool-analytics.test.ts | 33 +++++++++++ packages/core/src/__tests__/usage-events.test.ts | 69 +++++++++++++++++++--- packages/core/src/tool-analytics.ts | 22 +++++-- packages/core/src/usage-events.ts | 66 ++++++++++++++++++++- 5 files changed, 178 insertions(+), 17 deletions(-) Fusion-Task-Id: FN-6620 Fusion-Task-Lineage: a8f2525e-7aff-4576-89e2-6b142792f380
This commit is contained in:
5
.changeset/fn-6620-tool-categories.md
Normal file
5
.changeset/fn-6620-tool-categories.md
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
---
|
||||||
|
"@runfusion/fusion": patch
|
||||||
|
---
|
||||||
|
|
||||||
|
Improve Command Center tool analytics by categorizing Fusion tool families and re-bucketing historical `other` rows.
|
||||||
@@ -65,6 +65,39 @@ describe("tool-analytics", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("re-buckets historical other tool calls by tool name while preserving explicit categories", () => {
|
||||||
|
const ts = "2026-03-01T00:00:00.000Z";
|
||||||
|
emitUsageEvent(db, { kind: "tool_call", toolName: "fn_task_create", category: "other", ts });
|
||||||
|
emitUsageEvent(db, { kind: "tool_call", toolName: "fn_research_run", category: "other", ts });
|
||||||
|
emitUsageEvent(db, { kind: "tool_call", toolName: "fn_memory_append", category: null, ts });
|
||||||
|
emitUsageEvent(db, { kind: "tool_call", toolName: "fn_mission_show", category: "other", ts });
|
||||||
|
emitUsageEvent(db, { kind: "tool_call", toolName: "fn_skills_search", category: "other", ts });
|
||||||
|
emitUsageEvent(db, { kind: "tool_call", toolName: "Read", category: "other", ts });
|
||||||
|
emitUsageEvent(db, { kind: "tool_call", toolName: "Bash", category: "other", ts });
|
||||||
|
emitUsageEvent(db, { kind: "tool_call", toolName: "Unknown", category: "other", ts });
|
||||||
|
emitUsageEvent(db, { kind: "tool_call", toolName: null, category: "other", ts });
|
||||||
|
emitUsageEvent(db, { kind: "tool_call", toolName: "fn_task_update", category: "custom", ts });
|
||||||
|
|
||||||
|
const result = aggregateToolAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||||
|
const byCategory = new Map(result.byCategory.map((row) => [row.category, row.count]));
|
||||||
|
|
||||||
|
expect(result.toolCalls).toBe(10);
|
||||||
|
expect(byCategory).toEqual(
|
||||||
|
new Map([
|
||||||
|
["other", 2],
|
||||||
|
["custom", 1],
|
||||||
|
["edit", 1],
|
||||||
|
["execute", 1],
|
||||||
|
["memory", 1],
|
||||||
|
["planning", 1],
|
||||||
|
["read", 1],
|
||||||
|
["research", 1],
|
||||||
|
["skills", 1],
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
expect(result.byCategory[0]).toEqual({ category: "other", count: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
it("autonomy denominator counts a USER steer + an approval but NOT an agent steer", () => {
|
it("autonomy denominator counts a USER steer + an approval but NOT an agent steer", () => {
|
||||||
insertTaskWithSteers(db, "task-1", [
|
insertTaskWithSteers(db, "task-1", [
|
||||||
{ id: "s1", text: "do X", createdAt: "2026-03-02T00:00:00.000Z", author: "user" },
|
{ id: "s1", text: "do X", createdAt: "2026-03-02T00:00:00.000Z", author: "user" },
|
||||||
|
|||||||
@@ -88,15 +88,66 @@ describe("usage_events", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("categorizes tool names into coarse buckets", () => {
|
it("categorizes tool names into coarse buckets", () => {
|
||||||
expect(categorizeToolName("Read")).toBe("read");
|
const cases: Array<[string | null | undefined, string]> = [
|
||||||
expect(categorizeToolName("Grep")).toBe("read");
|
["Read", "read"],
|
||||||
expect(categorizeToolName("Edit")).toBe("edit");
|
["Grep", "read"],
|
||||||
expect(categorizeToolName("Write")).toBe("edit");
|
["Glob", "read"],
|
||||||
expect(categorizeToolName("Bash")).toBe("execute");
|
["ls", "read"],
|
||||||
expect(categorizeToolName("WebFetch")).toBe("network");
|
["semantic_search", "read"],
|
||||||
expect(categorizeToolName("Unknown")).toBe("other");
|
["fn_task_list", "read"],
|
||||||
expect(categorizeToolName(undefined)).toBe("other");
|
["fn_task_show", "read"],
|
||||||
expect(categorizeToolName(null)).toBe("other");
|
["fn_task_get", "read"],
|
||||||
|
["fn_task_search", "read"],
|
||||||
|
["fn_list_agents", "read"],
|
||||||
|
["fn_agent_org_chart", "read"],
|
||||||
|
["fn_task_document_read", "read"],
|
||||||
|
["fn_research_list", "research"],
|
||||||
|
["Edit", "edit"],
|
||||||
|
["Write", "edit"],
|
||||||
|
["MultiEdit", "edit"],
|
||||||
|
["NotebookEdit", "edit"],
|
||||||
|
["fn_task_create", "edit"],
|
||||||
|
["fn_task_update", "edit"],
|
||||||
|
["fn_task_attach", "edit"],
|
||||||
|
["fn_task_archive", "edit"],
|
||||||
|
["fn_task_document_write", "edit"],
|
||||||
|
["Bash", "execute"],
|
||||||
|
["execute_command", "execute"],
|
||||||
|
["terminal", "execute"],
|
||||||
|
["WebFetch", "network"],
|
||||||
|
["fn_web_fetch", "network"],
|
||||||
|
["http_request", "network"],
|
||||||
|
["fn_mission_show", "planning"],
|
||||||
|
["fn_milestone_add", "planning"],
|
||||||
|
["fn_slice_activate", "planning"],
|
||||||
|
["fn_feature_link_task", "planning"],
|
||||||
|
["fn_goal_create", "planning"],
|
||||||
|
["fn_task_plan", "planning"],
|
||||||
|
["fn_research_run", "research"],
|
||||||
|
["fn_insight_show", "research"],
|
||||||
|
["fn_experiment_finalize", "research"],
|
||||||
|
["fn_memory_append", "memory"],
|
||||||
|
["fn_agent_create", "agents"],
|
||||||
|
["fn_delegate_task", "agents"],
|
||||||
|
["fn_skills_search", "skills"],
|
||||||
|
["fn_secret_get", "secrets"],
|
||||||
|
["fn_task_import_github", "github"],
|
||||||
|
["fn_task_import_github_issue", "github"],
|
||||||
|
["fn_task_browse_github_issues", "github"],
|
||||||
|
["fn_workflow_create", "workflow"],
|
||||||
|
["fn_review_spec", "workflow"],
|
||||||
|
["mcp__server__search", "read"],
|
||||||
|
["mcp__server__tool", "other"],
|
||||||
|
["Unknown", "other"],
|
||||||
|
["", "other"],
|
||||||
|
[" ", "other"],
|
||||||
|
[undefined, "other"],
|
||||||
|
[null, "other"],
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const [toolName, expected] of cases) {
|
||||||
|
expect(categorizeToolName(toolName), String(toolName)).toBe(expected);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects a meta payload over the byte cap at write (event skipped, nothing inserted)", () => {
|
it("rejects a meta payload over the byte cap at write (event skipped, nothing inserted)", () => {
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import type { Database } from "./db.js";
|
import type { Database } from "./db.js";
|
||||||
|
import { categorizeToolName } from "./usage-events.js";
|
||||||
import type { SteeringComment } from "./types.js";
|
import type { SteeringComment } from "./types.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -75,6 +76,7 @@ interface CountRow {
|
|||||||
}
|
}
|
||||||
|
|
||||||
interface CategoryRow {
|
interface CategoryRow {
|
||||||
|
toolName: string | null;
|
||||||
category: string | null;
|
category: string | null;
|
||||||
count: number;
|
count: number;
|
||||||
}
|
}
|
||||||
@@ -175,17 +177,27 @@ export function aggregateToolAnalytics(
|
|||||||
.get(...eventParams) as CountRow
|
.get(...eventParams) as CountRow
|
||||||
).count;
|
).count;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* FNXC:CommandCenter 2026-06-17-21:43:
|
||||||
|
* Historical usage rows were logged with `category = "other"` before Fusion tool families were mapped, so aggregation must re-derive those buckets from `toolName`.
|
||||||
|
* Preserve explicit non-`other` categories because external callers may already provide a deliberate custom bucket.
|
||||||
|
*/
|
||||||
const categoryRows = db
|
const categoryRows = db
|
||||||
.prepare(
|
.prepare(
|
||||||
`SELECT category AS category, COUNT(*) AS count
|
`SELECT toolName AS toolName, category AS category, COUNT(*) AS count
|
||||||
FROM usage_events
|
FROM usage_events
|
||||||
WHERE kind = 'tool_call' ${rangeWhere}
|
WHERE kind = 'tool_call' ${rangeWhere}
|
||||||
GROUP BY category`,
|
GROUP BY toolName, category`,
|
||||||
)
|
)
|
||||||
.all(...eventParams) as CategoryRow[];
|
.all(...eventParams) as CategoryRow[];
|
||||||
const byCategory: ToolCategoryCount[] = categoryRows
|
const categoryCounts = new Map<string, number>();
|
||||||
.map((r) => ({ category: r.category ?? "other", count: r.count }))
|
for (const row of categoryRows) {
|
||||||
.sort((a, b) => b.count - a.count);
|
const category = row.category && row.category !== "other" ? row.category : categorizeToolName(row.toolName);
|
||||||
|
categoryCounts.set(category, (categoryCounts.get(category) ?? 0) + row.count);
|
||||||
|
}
|
||||||
|
const byCategory: ToolCategoryCount[] = [...categoryCounts.entries()]
|
||||||
|
.map(([category, count]) => ({ category, count }))
|
||||||
|
.sort((a, b) => b.count - a.count || a.category.localeCompare(b.category));
|
||||||
|
|
||||||
const sessions = (
|
const sessions = (
|
||||||
db
|
db
|
||||||
|
|||||||
@@ -99,14 +99,74 @@ interface UsageEventRow {
|
|||||||
/**
|
/**
|
||||||
* Coarse tool category derived from a tool name, for the Tools analytics area.
|
* Coarse tool category derived from a tool name, for the Tools analytics area.
|
||||||
* Pure and side-effect free; callers may also pass an explicit `category`.
|
* Pure and side-effect free; callers may also pass an explicit `category`.
|
||||||
|
*
|
||||||
|
* FNXC:CommandCenter 2026-06-17-21:35:
|
||||||
|
* Fusion agents mostly call namespaced `fn_*` tools, so Command Center analytics must bucket those families meaningfully instead of letting the Tools chart collapse into `other`.
|
||||||
|
* Keep this mapping pure and lowercase-normalized because it is used both at log-write time and when re-bucketing historical rows.
|
||||||
*/
|
*/
|
||||||
export function categorizeToolName(toolName: string | null | undefined): string {
|
export function categorizeToolName(toolName: string | null | undefined): string {
|
||||||
if (!toolName) return "other";
|
if (!toolName) return "other";
|
||||||
const name = toolName.toLowerCase();
|
const name = toolName.trim().toLowerCase();
|
||||||
if (name === "read" || name === "grep" || name === "glob" || name === "ls" || name.includes("search")) {
|
if (!name) return "other";
|
||||||
|
|
||||||
|
if (name.startsWith("fn_task_import_github") || name.startsWith("fn_task_browse_github")) {
|
||||||
|
return "github";
|
||||||
|
}
|
||||||
|
if (name === "fn_web_fetch") return "network";
|
||||||
|
if (name.startsWith("fn_secret_")) return "secrets";
|
||||||
|
if (name.startsWith("fn_skills_")) return "skills";
|
||||||
|
if (name.startsWith("fn_memory_")) return "memory";
|
||||||
|
if (name === "fn_list_agents" || name === "fn_agent_org_chart") return "read";
|
||||||
|
if (name.startsWith("fn_agent_") || name === "fn_delegate_task") return "agents";
|
||||||
|
if (
|
||||||
|
name.startsWith("fn_mission_") ||
|
||||||
|
name.startsWith("fn_milestone_") ||
|
||||||
|
name.startsWith("fn_slice_") ||
|
||||||
|
name.startsWith("fn_feature_") ||
|
||||||
|
name.startsWith("fn_goal_") ||
|
||||||
|
name === "fn_task_plan"
|
||||||
|
) {
|
||||||
|
return "planning";
|
||||||
|
}
|
||||||
|
if (name.startsWith("fn_research_") || name.startsWith("fn_insight_") || name.startsWith("fn_experiment_")) {
|
||||||
|
return "research";
|
||||||
|
}
|
||||||
|
if (name.startsWith("fn_workflow_") || name === "fn_review_spec") return "workflow";
|
||||||
|
|
||||||
|
if (
|
||||||
|
name === "read" ||
|
||||||
|
name === "grep" ||
|
||||||
|
name === "glob" ||
|
||||||
|
name === "ls" ||
|
||||||
|
name.includes("search") ||
|
||||||
|
name === "fn_list_agents" ||
|
||||||
|
name === "fn_agent_org_chart" ||
|
||||||
|
name === "fn_task_document_read" ||
|
||||||
|
name.endsWith("_list") ||
|
||||||
|
name.endsWith("_show") ||
|
||||||
|
name.endsWith("_get") ||
|
||||||
|
name.endsWith("_search")
|
||||||
|
) {
|
||||||
return "read";
|
return "read";
|
||||||
}
|
}
|
||||||
if (name === "edit" || name === "write" || name === "multiedit" || name.includes("notebook")) {
|
if (
|
||||||
|
name === "edit" ||
|
||||||
|
name === "write" ||
|
||||||
|
name === "multiedit" ||
|
||||||
|
name.includes("notebook") ||
|
||||||
|
name === "fn_task_create" ||
|
||||||
|
name === "fn_task_update" ||
|
||||||
|
name === "fn_task_attach" ||
|
||||||
|
name === "fn_task_pause" ||
|
||||||
|
name === "fn_task_unpause" ||
|
||||||
|
name === "fn_task_retry" ||
|
||||||
|
name === "fn_task_duplicate" ||
|
||||||
|
name === "fn_task_refine" ||
|
||||||
|
name === "fn_task_archive" ||
|
||||||
|
name === "fn_task_unarchive" ||
|
||||||
|
name === "fn_task_delete" ||
|
||||||
|
name === "fn_task_document_write"
|
||||||
|
) {
|
||||||
return "edit";
|
return "edit";
|
||||||
}
|
}
|
||||||
if (name === "bash" || name.includes("exec") || name.includes("command") || name.includes("terminal")) {
|
if (name === "bash" || name.includes("exec") || name.includes("command") || name.includes("terminal")) {
|
||||||
|
|||||||
Reference in New Issue
Block a user