From 53bb1d8f37561f86a7d6dbf615b53db90fcd350e Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Mon, 15 Jun 2026 19:27:45 -0700 Subject: [PATCH] =?UTF-8?q?feat(analytics):=20U2=20=E2=80=94=20core=20date?= =?UTF-8?q?-range=20aggregators=20(tokens/tools/activity/productivity)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reusable aggregate() over tasks + usage_events with model/provider/node/agent grouping. Autonomy ratio sources interventions from approval audit events + user-authored steers (agent steers excluded); fully-autonomous sessions report tool-calls-per-session, never divide-by-zero. LOC/MTTR seams flagged unavailable. --- .../src/__tests__/activity-analytics.test.ts | 91 ++++++++ .../__tests__/productivity-analytics.test.ts | 103 ++++++++ .../src/__tests__/token-analytics.test.ts | 157 +++++++++++++ .../core/src/__tests__/tool-analytics.test.ts | 146 ++++++++++++ packages/core/src/activity-analytics.ts | 193 +++++++++++++++ packages/core/src/index.ts | 31 +++ packages/core/src/productivity-analytics.ts | 175 ++++++++++++++ packages/core/src/token-analytics.ts | 175 ++++++++++++++ packages/core/src/tool-analytics.ts | 221 ++++++++++++++++++ 9 files changed, 1292 insertions(+) create mode 100644 packages/core/src/__tests__/activity-analytics.test.ts create mode 100644 packages/core/src/__tests__/productivity-analytics.test.ts create mode 100644 packages/core/src/__tests__/token-analytics.test.ts create mode 100644 packages/core/src/__tests__/tool-analytics.test.ts create mode 100644 packages/core/src/activity-analytics.ts create mode 100644 packages/core/src/productivity-analytics.ts create mode 100644 packages/core/src/token-analytics.ts create mode 100644 packages/core/src/tool-analytics.ts diff --git a/packages/core/src/__tests__/activity-analytics.test.ts b/packages/core/src/__tests__/activity-analytics.test.ts new file mode 100644 index 0000000000..76f26085e6 --- /dev/null +++ b/packages/core/src/__tests__/activity-analytics.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +import { Database } from "../db.js"; +import { emitUsageEvent } from "../usage-events.js"; +import { aggregateActivityAnalytics } from "../activity-analytics.js"; + +function insertCliSession(db: Database, id: string, createdAt: string): void { + db.prepare( + `INSERT INTO cli_sessions + (id, purpose, projectId, adapterId, agentState, createdAt, updatedAt) + VALUES (?, 'task', 'proj-1', 'claude-local', 'running', ?, ?)`, + ).run(id, createdAt, createdAt); +} + +describe("activity-analytics", () => { + let tmpDir: string; + let db: Database; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "kb-activity-analytics-")); + db = new Database(join(tmpDir, ".fusion")); + db.init(); + }); + + afterEach(async () => { + db.close(); + await rm(tmpDir, { recursive: true, force: true }); + }); + + it("counts sessions, messages, and distinct active nodes/agents over a range", () => { + insertCliSession(db, "s1", "2026-03-01T00:00:00.000Z"); + insertCliSession(db, "s2", "2026-03-02T00:00:00.000Z"); + // session outside range + insertCliSession(db, "s-old", "2025-01-01T00:00:00.000Z"); + + emitUsageEvent(db, { kind: "user_message", agentId: "agent-1", nodeId: "node-1", ts: "2026-03-01T00:00:00.000Z" }); + emitUsageEvent(db, { kind: "user_message", agentId: "agent-2", nodeId: "node-1", ts: "2026-03-01T01:00:00.000Z" }); + emitUsageEvent(db, { kind: "tool_call", agentId: "agent-2", nodeId: "node-2", ts: "2026-03-02T00:00:00.000Z" }); + + const result = aggregateActivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(result.sessions).toBe(2); + expect(result.messages).toBe(2); + expect(result.activeNodes).toBe(2); // node-1, node-2 + expect(result.activeAgents).toBe(2); // agent-1, agent-2 + }); + + it("produces a per-day breakdown ascending by day", () => { + emitUsageEvent(db, { kind: "user_message", agentId: "agent-1", nodeId: "node-1", ts: "2026-03-01T08:00:00.000Z" }); + emitUsageEvent(db, { kind: "tool_call", agentId: "agent-1", nodeId: "node-1", ts: "2026-03-01T09:00:00.000Z" }); + emitUsageEvent(db, { kind: "user_message", agentId: "agent-2", nodeId: "node-2", ts: "2026-03-02T08:00:00.000Z" }); + + const result = aggregateActivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(result.daily.map((d) => d.day)).toEqual(["2026-03-01", "2026-03-02"]); + expect(result.daily[0]).toMatchObject({ day: "2026-03-01", activeNodes: 1, activeAgents: 1, messages: 1 }); + expect(result.daily[1]).toMatchObject({ day: "2026-03-02", activeNodes: 1, activeAgents: 1, messages: 1 }); + }); + + it("computes stickiness = DAU/MAU", () => { + // Day 1: agents a,b active. Day 2: agent a active. MAU = {a,b} = 2. + // DAU = mean(2, 1) = 1.5. stickiness = 1.5 / 2 = 0.75. + emitUsageEvent(db, { kind: "tool_call", agentId: "a", nodeId: "n1", ts: "2026-03-01T00:00:00.000Z" }); + emitUsageEvent(db, { kind: "tool_call", agentId: "b", nodeId: "n1", ts: "2026-03-01T01:00:00.000Z" }); + emitUsageEvent(db, { kind: "tool_call", agentId: "a", nodeId: "n1", ts: "2026-03-02T00:00:00.000Z" }); + + const result = aggregateActivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(result.activeAgents).toBe(2); + expect(result.stickiness).toBeCloseTo(0.75, 5); + }); + + it("empty range returns zeroed structures, not nulls", () => { + insertCliSession(db, "s1", "2026-03-01T00:00:00.000Z"); + emitUsageEvent(db, { kind: "user_message", agentId: "a", nodeId: "n1", ts: "2026-03-01T00:00:00.000Z" }); + + const result = aggregateActivityAnalytics(db, { from: "2027-01-01T00:00:00.000Z", to: "2027-12-31T00:00:00.000Z" }); + expect(result.sessions).toBe(0); + expect(result.messages).toBe(0); + expect(result.activeNodes).toBe(0); + expect(result.activeAgents).toBe(0); + expect(result.daily).toEqual([]); + expect(result.stickiness).toBe(0); + }); + + it("leaves a clean MTTR seam for U13 (unavailable, not 0)", () => { + const result = aggregateActivityAnalytics(db, {}); + expect(result.mttr).toEqual({ value: null, unavailable: true }); + }); +}); diff --git a/packages/core/src/__tests__/productivity-analytics.test.ts b/packages/core/src/__tests__/productivity-analytics.test.ts new file mode 100644 index 0000000000..f61622902f --- /dev/null +++ b/packages/core/src/__tests__/productivity-analytics.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +import { Database } from "../db.js"; +import { aggregateProductivityAnalytics } from "../productivity-analytics.js"; + +function insertTaskWithFiles(db: Database, id: string, files: string[], updatedAt: string): void { + db.prepare( + `INSERT INTO tasks (id, description, "column", createdAt, updatedAt, modifiedFiles) + VALUES (?, 'desc', 'todo', ?, ?, ?)`, + ).run(id, updatedAt, updatedAt, JSON.stringify(files)); +} + +function insertCommit(db: Database, id: string, sha: string, authoredAt: string): void { + db.prepare( + `INSERT INTO task_commit_associations + (id, taskLineageId, taskIdSnapshot, commitSha, commitSubject, authoredAt, + matchedBy, confidence, createdAt, updatedAt) + VALUES (?, 'lin-1', 't-1', ?, 'subj', ?, 'canonical-lineage-trailer', 'canonical', ?, ?)`, + ).run(id, sha, authoredAt, authoredAt, authoredAt); +} + +function insertPr(db: Database, id: string, createdAtMs: number): void { + db.prepare( + `INSERT INTO pull_requests + (id, sourceType, sourceId, repo, headBranch, state, createdAt, updatedAt) + VALUES (?, 'task', ?, 'org/repo', ?, 'open', ?, ?)`, + ).run(id, `src-${id}`, `branch-${id}`, createdAtMs, createdAtMs); +} + +describe("productivity-analytics", () => { + let tmpDir: string; + let db: Database; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "kb-productivity-analytics-")); + db = new Database(join(tmpDir, ".fusion")); + db.init(); + }); + + afterEach(async () => { + db.close(); + await rm(tmpDir, { recursive: true, force: true }); + }); + + it("counts modified files and language distribution", () => { + insertTaskWithFiles(db, "t1", ["src/a.ts", "src/b.ts", "README.md"], "2026-03-01T00:00:00.000Z"); + insertTaskWithFiles(db, "t2", ["src/c.ts", "style.css"], "2026-03-02T00:00:00.000Z"); + + const result = aggregateProductivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(result.modifiedFiles).toBe(5); + const byLang = new Map(result.byLanguage.map((l) => [l.language, l.count])); + expect(byLang.get("ts")).toBe(3); + expect(byLang.get("md")).toBe(1); + expect(byLang.get("css")).toBe(1); + // sorted descending by count + expect(result.byLanguage[0]).toEqual({ language: "ts", count: 3 }); + }); + + it("counts commit associations and pull requests in range", () => { + insertCommit(db, "c1", "sha1", "2026-03-01T00:00:00.000Z"); + insertCommit(db, "c2", "sha2", "2026-03-02T00:00:00.000Z"); + insertCommit(db, "c-old", "sha-old", "2025-01-01T00:00:00.000Z"); + + insertPr(db, "pr1", Date.parse("2026-03-01T00:00:00.000Z")); + insertPr(db, "pr2", Date.parse("2026-03-10T00:00:00.000Z")); + insertPr(db, "pr-old", Date.parse("2025-01-01T00:00:00.000Z")); + + const result = aggregateProductivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(result.commits).toBe(2); + expect(result.pullRequests).toBe(2); + }); + + it("reports LOC as unavailable (null + unavailable:true), never 0", () => { + insertTaskWithFiles(db, "t1", ["src/a.ts"], "2026-03-01T00:00:00.000Z"); + const result = aggregateProductivityAnalytics(db, {}); + expect(result.loc).toEqual({ value: null, unavailable: true }); + expect(result.loc.value).not.toBe(0); + }); + + it("empty range returns zeroed structures, not nulls", () => { + insertTaskWithFiles(db, "t1", ["src/a.ts"], "2026-03-01T00:00:00.000Z"); + insertCommit(db, "c1", "sha1", "2026-03-01T00:00:00.000Z"); + insertPr(db, "pr1", Date.parse("2026-03-01T00:00:00.000Z")); + + const result = aggregateProductivityAnalytics(db, { from: "2027-01-01T00:00:00.000Z", to: "2027-12-31T00:00:00.000Z" }); + expect(result.modifiedFiles).toBe(0); + expect(result.byLanguage).toEqual([]); + expect(result.commits).toBe(0); + expect(result.pullRequests).toBe(0); + // LOC unavailable regardless of range + expect(result.loc).toEqual({ value: null, unavailable: true }); + }); + + it("includes a boundary task exactly at `from`", () => { + insertTaskWithFiles(db, "boundary", ["x.ts"], "2026-03-01T00:00:00.000Z"); + const result = aggregateProductivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(result.modifiedFiles).toBe(1); + }); +}); diff --git a/packages/core/src/__tests__/token-analytics.test.ts b/packages/core/src/__tests__/token-analytics.test.ts new file mode 100644 index 0000000000..81e3d6e499 --- /dev/null +++ b/packages/core/src/__tests__/token-analytics.test.ts @@ -0,0 +1,157 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +import { Database } from "../db.js"; +import { aggregateTokenAnalytics } from "../token-analytics.js"; + +interface TaskSeed { + id: string; + inputTokens?: number; + outputTokens?: number; + cachedTokens?: number; + cacheWriteTokens?: number; + totalTokens?: number | null; + lastUsedAt: string | null; + modelProvider?: string | null; + modelId?: string | null; + nodeId?: string | null; + agentId?: string | null; +} + +function insertTask(db: Database, t: TaskSeed): void { + db.prepare( + `INSERT INTO tasks + (id, description, "column", createdAt, updatedAt, + tokenUsageInputTokens, tokenUsageOutputTokens, tokenUsageCachedTokens, + tokenUsageCacheWriteTokens, tokenUsageTotalTokens, tokenUsageLastUsedAt, + modelProvider, modelId, checkoutNodeId, assignedAgentId) + VALUES (?, 'desc', 'todo', '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z', + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + t.id, + t.inputTokens ?? null, + t.outputTokens ?? null, + t.cachedTokens ?? null, + t.cacheWriteTokens ?? null, + t.totalTokens === undefined ? null : t.totalTokens, + t.lastUsedAt, + t.modelProvider ?? null, + t.modelId ?? null, + t.nodeId ?? null, + t.agentId ?? null, + ); +} + +describe("token-analytics", () => { + let tmpDir: string; + let db: Database; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "kb-token-analytics-")); + db = new Database(join(tmpDir, ".fusion")); + db.init(); + }); + + afterEach(async () => { + db.close(); + await rm(tmpDir, { recursive: true, force: true }); + }); + + it("returns correct per-model token totals for 5 tasks across 2 models", () => { + // 3 tasks on model-A, 2 on model-B, all within range. + insertTask(db, { id: "t1", inputTokens: 100, outputTokens: 50, totalTokens: 150, lastUsedAt: "2026-03-01T00:00:00.000Z", modelId: "model-A", modelProvider: "anthropic" }); + insertTask(db, { id: "t2", inputTokens: 200, outputTokens: 80, totalTokens: 280, lastUsedAt: "2026-03-02T00:00:00.000Z", modelId: "model-A", modelProvider: "anthropic" }); + insertTask(db, { id: "t3", inputTokens: 300, outputTokens: 20, totalTokens: 320, lastUsedAt: "2026-03-03T00:00:00.000Z", modelId: "model-A", modelProvider: "anthropic" }); + insertTask(db, { id: "t4", inputTokens: 10, outputTokens: 5, totalTokens: 15, lastUsedAt: "2026-03-04T00:00:00.000Z", modelId: "model-B", modelProvider: "openai" }); + insertTask(db, { id: "t5", inputTokens: 40, outputTokens: 60, totalTokens: 100, lastUsedAt: "2026-03-05T00:00:00.000Z", modelId: "model-B", modelProvider: "openai" }); + + const result = aggregateTokenAnalytics(db, { + from: "2026-03-01T00:00:00.000Z", + to: "2026-03-31T00:00:00.000Z", + groupBy: "model", + }); + + expect(result.totals.inputTokens).toBe(650); + expect(result.totals.outputTokens).toBe(215); + expect(result.totals.totalTokens).toBe(865); + expect(result.totals.nTasks).toBe(5); + + const groups = new Map(result.groups.map((g) => [g.key, g])); + expect(groups.get("model-A")!.inputTokens).toBe(600); + expect(groups.get("model-A")!.totalTokens).toBe(750); + expect(groups.get("model-A")!.nTasks).toBe(3); + expect(groups.get("model-B")!.inputTokens).toBe(50); + expect(groups.get("model-B")!.totalTokens).toBe(115); + expect(groups.get("model-B")!.nTasks).toBe(2); + // groups sorted descending by totalTokens + expect(result.groups[0].key).toBe("model-A"); + }); + + it("groups by provider, node, and agent", () => { + insertTask(db, { id: "t1", inputTokens: 100, totalTokens: 100, lastUsedAt: "2026-03-01T00:00:00.000Z", modelProvider: "anthropic", nodeId: "node-1", agentId: "agent-x" }); + insertTask(db, { id: "t2", inputTokens: 200, totalTokens: 200, lastUsedAt: "2026-03-02T00:00:00.000Z", modelProvider: "openai", nodeId: "node-1", agentId: "agent-y" }); + + const byProvider = aggregateTokenAnalytics(db, { groupBy: "provider" }); + expect(new Map(byProvider.groups.map((g) => [g.key, g.totalTokens]))).toEqual( + new Map([["anthropic", 100], ["openai", 200]]), + ); + + const byNode = aggregateTokenAnalytics(db, { groupBy: "node" }); + expect(byNode.groups).toHaveLength(1); + expect(byNode.groups[0].key).toBe("node-1"); + expect(byNode.groups[0].totalTokens).toBe(300); + + const byAgent = aggregateTokenAnalytics(db, { groupBy: "agent" }); + expect(new Map(byAgent.groups.map((g) => [g.key, g.totalTokens]))).toEqual( + new Map([["agent-x", 100], ["agent-y", 200]]), + ); + }); + + it("empty range returns zeroed structures, not nulls", () => { + insertTask(db, { id: "t1", inputTokens: 100, totalTokens: 100, lastUsedAt: "2026-03-01T00:00:00.000Z", modelId: "model-A" }); + + const result = aggregateTokenAnalytics(db, { + from: "2027-01-01T00:00:00.000Z", + to: "2027-12-31T00:00:00.000Z", + groupBy: "model", + }); + expect(result.totals).toEqual({ + inputTokens: 0, + outputTokens: 0, + cachedTokens: 0, + cacheWriteTokens: 0, + totalTokens: 0, + nTasks: 0, + }); + expect(result.groups).toEqual([]); + }); + + it("includes a boundary task exactly at `from` (inclusive lower bound)", () => { + insertTask(db, { id: "boundary", inputTokens: 42, totalTokens: 42, lastUsedAt: "2026-03-01T00:00:00.000Z", modelId: "model-A" }); + + const result = aggregateTokenAnalytics(db, { + from: "2026-03-01T00:00:00.000Z", + to: "2026-03-31T00:00:00.000Z", + }); + expect(result.totals.nTasks).toBe(1); + expect(result.totals.inputTokens).toBe(42); + }); + + it("excludes tasks with no token usage (lastUsedAt null)", () => { + insertTask(db, { id: "no-usage", lastUsedAt: null, modelId: "model-A" }); + insertTask(db, { id: "has-usage", inputTokens: 5, totalTokens: 5, lastUsedAt: "2026-03-01T00:00:00.000Z", modelId: "model-A" }); + + const result = aggregateTokenAnalytics(db, {}); + expect(result.totals.nTasks).toBe(1); + expect(result.totals.inputTokens).toBe(5); + }); + + it("derives totalTokens from parts when the persisted total is null", () => { + insertTask(db, { id: "t1", inputTokens: 10, outputTokens: 20, cachedTokens: 5, cacheWriteTokens: 1, totalTokens: null, lastUsedAt: "2026-03-01T00:00:00.000Z", modelId: "model-A" }); + const result = aggregateTokenAnalytics(db, {}); + expect(result.totals.totalTokens).toBe(36); + }); +}); diff --git a/packages/core/src/__tests__/tool-analytics.test.ts b/packages/core/src/__tests__/tool-analytics.test.ts new file mode 100644 index 0000000000..ac8dbbd378 --- /dev/null +++ b/packages/core/src/__tests__/tool-analytics.test.ts @@ -0,0 +1,146 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +import { Database } from "../db.js"; +import { emitUsageEvent } from "../usage-events.js"; +import { aggregateToolAnalytics, countInterventions } from "../tool-analytics.js"; +import type { SteeringComment } from "../types.js"; + +function insertTaskWithSteers(db: Database, id: string, steers: SteeringComment[]): void { + db.prepare( + `INSERT INTO tasks (id, description, "column", createdAt, updatedAt, steeringComments) + VALUES (?, 'desc', 'todo', '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z', ?)`, + ).run(id, JSON.stringify(steers)); +} + +function insertApprovalRequest(db: Database, id: string): void { + db.prepare( + `INSERT INTO approval_requests + (id, status, requesterActorId, requesterActorType, requesterActorName, + targetActionCategory, targetActionOperation, targetActionSummary, + targetResourceType, targetResourceId, requestedAt, createdAt, updatedAt) + VALUES (?, 'pending', 'a', 'agent', 'A', 'cat', 'op', 'sum', 'res', 'r1', + '2026-03-01T00:00:00.000Z', '2026-03-01T00:00:00.000Z', '2026-03-01T00:00:00.000Z')`, + ).run(id); +} + +function insertApprovalEvent(db: Database, id: string, requestId: string, eventType: string, createdAt: string): void { + db.prepare( + `INSERT INTO approval_request_audit_events + (id, requestId, eventType, actorId, actorType, actorName, createdAt) + VALUES (?, ?, ?, 'u1', 'user', 'User', ?)`, + ).run(id, requestId, eventType, createdAt); +} + +describe("tool-analytics", () => { + let tmpDir: string; + let db: Database; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "kb-tool-analytics-")); + db = new Database(join(tmpDir, ".fusion")); + db.init(); + }); + + afterEach(async () => { + db.close(); + await rm(tmpDir, { recursive: true, force: true }); + }); + + it("counts tool calls by category, sorted descending", () => { + emitUsageEvent(db, { kind: "tool_call", category: "read", ts: "2026-03-01T00:00:00.000Z" }); + emitUsageEvent(db, { kind: "tool_call", category: "read", ts: "2026-03-01T01:00:00.000Z" }); + emitUsageEvent(db, { kind: "tool_call", category: "edit", ts: "2026-03-01T02:00:00.000Z" }); + // a non-tool_call event is not counted + emitUsageEvent(db, { kind: "user_message", ts: "2026-03-01T03:00:00.000Z" }); + + const result = aggregateToolAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(result.toolCalls).toBe(3); + expect(result.byCategory).toEqual([ + { category: "read", count: 2 }, + { category: "edit", count: 1 }, + ]); + }); + + it("autonomy denominator counts a USER steer + an approval but NOT an agent steer", () => { + insertTaskWithSteers(db, "task-1", [ + { id: "s1", text: "do X", createdAt: "2026-03-02T00:00:00.000Z", author: "user" }, + { id: "s2", text: "agent note", createdAt: "2026-03-02T01:00:00.000Z", author: "agent" }, + ]); + insertApprovalRequest(db, "req-1"); + insertApprovalEvent(db, "ev-created", "req-1", "created", "2026-03-02T00:30:00.000Z"); + insertApprovalEvent(db, "ev-approved", "req-1", "approved", "2026-03-02T00:31:00.000Z"); + // a non-human eventType must NOT count + insertApprovalEvent(db, "ev-completed", "req-1", "completed", "2026-03-02T00:32:00.000Z"); + + const breakdown = countInterventions(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(breakdown.userSteers).toBe(1); // agent steer excluded + expect(breakdown.approvals).toBe(2); // created + approved, completed excluded + expect(breakdown.total).toBe(3); + }); + + it("autonomy ratio = toolCalls / interventions for an interactive session", () => { + // 12 tool calls, 3 interventions (1 user steer + 2 approvals) -> ratio 4 + for (let i = 0; i < 12; i++) { + emitUsageEvent(db, { kind: "tool_call", category: "read", ts: `2026-03-02T00:0${i % 6}:0${i % 6}.000Z` }); + } + emitUsageEvent(db, { kind: "session_start", ts: "2026-03-02T00:00:00.000Z" }); + insertTaskWithSteers(db, "task-1", [{ id: "s1", text: "x", createdAt: "2026-03-02T00:10:00.000Z", author: "user" }]); + insertApprovalRequest(db, "req-1"); + insertApprovalEvent(db, "ev-c", "req-1", "created", "2026-03-02T00:11:00.000Z"); + insertApprovalEvent(db, "ev-a", "req-1", "approved", "2026-03-02T00:12:00.000Z"); + + const result = aggregateToolAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(result.interventions.total).toBe(3); + expect(result.toolCalls).toBe(12); + expect(result.autonomyRatio).toBe(4); + expect(result.fullyAutonomous).toBe(false); + }); + + it("fully-autonomous session (zero interventions) reports tool-calls-per-session, not infinity", () => { + // 10 tool calls across 2 sessions, zero interventions -> 5 per session + for (let i = 0; i < 10; i++) { + emitUsageEvent(db, { kind: "tool_call", category: "execute", ts: "2026-03-02T00:00:00.000Z" }); + } + emitUsageEvent(db, { kind: "session_start", ts: "2026-03-02T00:00:00.000Z" }); + emitUsageEvent(db, { kind: "session_start", ts: "2026-03-02T01:00:00.000Z" }); + + const result = aggregateToolAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(result.interventions.total).toBe(0); + expect(result.fullyAutonomous).toBe(true); + expect(result.autonomyRatio).toBe(5); + expect(Number.isFinite(result.autonomyRatio)).toBe(true); + }); + + it("zero interventions and zero sessions does not divide by zero", () => { + for (let i = 0; i < 4; i++) { + emitUsageEvent(db, { kind: "tool_call", category: "read", ts: "2026-03-02T00:00:00.000Z" }); + } + const result = aggregateToolAnalytics(db, {}); + expect(result.sessions).toBe(0); + expect(result.fullyAutonomous).toBe(true); + // toolCalls / max(sessions, 1) = 4 / 1 + expect(result.autonomyRatio).toBe(4); + }); + + it("empty range returns zeroed structures, not nulls", () => { + const result = aggregateToolAnalytics(db, { from: "2027-01-01T00:00:00.000Z", to: "2027-12-31T00:00:00.000Z" }); + expect(result.toolCalls).toBe(0); + expect(result.byCategory).toEqual([]); + expect(result.sessions).toBe(0); + expect(result.interventions).toEqual({ approvals: 0, userSteers: 0, total: 0 }); + expect(result.autonomyRatio).toBe(0); + }); + + it("user steers outside the range are not counted", () => { + insertTaskWithSteers(db, "task-1", [ + { id: "s1", text: "old", createdAt: "2025-01-01T00:00:00.000Z", author: "user" }, + { id: "s2", text: "in range", createdAt: "2026-03-15T00:00:00.000Z", author: "user" }, + ]); + const breakdown = countInterventions(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" }); + expect(breakdown.userSteers).toBe(1); + }); +}); diff --git a/packages/core/src/activity-analytics.ts b/packages/core/src/activity-analytics.ts new file mode 100644 index 0000000000..dbee0b1a20 --- /dev/null +++ b/packages/core/src/activity-analytics.ts @@ -0,0 +1,193 @@ +import type { Database } from "./db.js"; + +/** + * Activity analytics: distinct active nodes/agents per day, sessions, messages, + * and stickiness (DAU/MAU) over an arbitrary date range. + * + * Sessions come from `cli_sessions` (by `createdAt`); messages and node/agent + * activity come from `usage_events`. Inclusivity: `from`/`to` are inclusive, + * matching `usage-events.ts`. + * + * **MTTR seam (U13).** Mean-time-to-resolve aggregation is deliberately NOT + * implemented here yet — it depends on the deployments/incidents tables U13 + * introduces. {@link aggregateActivityAnalytics} returns an `mttr` field set to + * the documented "unavailable" sentinel so the shape is stable now and U13 can + * fill it in without changing callers. See {@link MttrSummary}. + */ + +export interface ActivityAnalyticsQuery { + /** ISO-8601 lower bound (inclusive). */ + from?: string; + /** ISO-8601 upper bound (inclusive). */ + to?: string; +} + +/** Distinct active nodes/agents and message count for a single UTC day. */ +export interface DailyActivity { + /** UTC date, `YYYY-MM-DD`. */ + day: string; + activeNodes: number; + activeAgents: number; + messages: number; +} + +/** + * MTTR summary placeholder. U13 will populate `value` (mean minutes to resolve) + * once deployments/incidents land; until then it is the documented unavailable + * sentinel — `null` value with `unavailable: true`, never `0`. + */ +export interface MttrSummary { + /** Mean minutes to resolve; null until U13 provides incident data. */ + value: number | null; + /** True when MTTR cannot be computed (no incident data source yet). */ + unavailable: boolean; +} + +export interface ActivityAnalytics { + from: string | null; + to: string | null; + /** Total `session_start` events from `cli_sessions` in range. */ + sessions: number; + /** Total `user_message` events in range. */ + messages: number; + /** Distinct nodes with any usage_event in range. */ + activeNodes: number; + /** Distinct agents with any usage_event in range. */ + activeAgents: number; + /** Per-day breakdown, ascending by day. */ + daily: DailyActivity[]; + /** + * Stickiness = DAU/MAU. DAU = mean distinct-active-agents-per-day over the + * range; MAU = distinct active agents over the whole range. 0 when MAU is 0. + */ + stickiness: number; + /** MTTR placeholder (U13 seam). */ + mttr: MttrSummary; +} + +interface CountRow { + count: number; +} + +interface DistinctRow { + count: number; +} + +interface DayAggRow { + day: string; + activeNodes: number; + activeAgents: number; + messages: number; +} + +function rangeClauses( + column: string, + query: ActivityAnalyticsQuery, +): { where: string; params: string[] } { + const clauses: string[] = []; + const params: string[] = []; + if (query.from !== undefined) { + clauses.push(`${column} >= ?`); + params.push(query.from); + } + if (query.to !== undefined) { + clauses.push(`${column} <= ?`); + params.push(query.to); + } + return { + where: clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "", + params, + }; +} + +/** + * Aggregate activity (sessions, messages, active nodes/agents, daily breakdown, + * stickiness) over a date range. Empty range yields zeroed structures and an + * empty `daily` array — never nulls. `mttr` is the U13 unavailable seam. + */ +export function aggregateActivityAnalytics( + db: Database, + query: ActivityAnalyticsQuery = {}, +): ActivityAnalytics { + // Sessions from cli_sessions (by createdAt). + const sessionRange = rangeClauses("createdAt", query); + const sessions = ( + db + .prepare(`SELECT COUNT(*) AS count FROM cli_sessions ${sessionRange.where}`) + .get(...sessionRange.params) as CountRow + ).count; + + // Messages from usage_events (kind = user_message). + const eventRange = rangeClauses("ts", query); + const eventWhereWith = (extra: string): string => + eventRange.where + ? `${eventRange.where} AND ${extra}` + : `WHERE ${extra}`; + + const messages = ( + db + .prepare( + `SELECT COUNT(*) AS count FROM usage_events ${eventWhereWith("kind = 'user_message'")}`, + ) + .get(...eventRange.params) as CountRow + ).count; + + // Distinct active nodes/agents over the whole range. + const activeNodes = ( + db + .prepare( + `SELECT COUNT(DISTINCT nodeId) AS count FROM usage_events ${eventWhereWith("nodeId IS NOT NULL")}`, + ) + .get(...eventRange.params) as DistinctRow + ).count; + const activeAgents = ( + db + .prepare( + `SELECT COUNT(DISTINCT agentId) AS count FROM usage_events ${eventWhereWith("agentId IS NOT NULL")}`, + ) + .get(...eventRange.params) as DistinctRow + ).count; + + // Per-day distinct nodes/agents + message count. substr(ts,1,10) is the UTC + // day key (ISO-8601 timestamps). + const dailyRows = db + .prepare( + `SELECT + substr(ts, 1, 10) AS day, + COUNT(DISTINCT nodeId) AS activeNodes, + COUNT(DISTINCT agentId) AS activeAgents, + SUM(CASE WHEN kind = 'user_message' THEN 1 ELSE 0 END) AS messages + FROM usage_events ${eventRange.where} + GROUP BY day + ORDER BY day ASC`, + ) + .all(...eventRange.params) as DayAggRow[]; + const daily: DailyActivity[] = dailyRows.map((r) => ({ + day: r.day, + activeNodes: r.activeNodes, + activeAgents: r.activeAgents, + messages: r.messages ?? 0, + })); + + // Stickiness = DAU/MAU. DAU = mean distinct-active-agents-per-day; MAU = + // distinct active agents over the range. + const dau = + daily.length > 0 + ? daily.reduce((sum, d) => sum + d.activeAgents, 0) / daily.length + : 0; + const mau = activeAgents; + const stickiness = mau > 0 ? dau / mau : 0; + + return { + from: query.from ?? null, + to: query.to ?? null, + sessions, + messages, + activeNodes, + activeAgents, + daily, + stickiness, + // U13 seam: no incident data source yet — unavailable, not 0. + mttr: { value: null, unavailable: true }, + }; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 362127af74..e9698c2c33 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -530,6 +530,35 @@ export type { UsageEventKind, UsageEventRangeQuery, } from "./usage-events.js"; +export { aggregateTokenAnalytics } from "./token-analytics.js"; +export type { + TokenAnalytics, + TokenAnalyticsQuery, + TokenGroupBy, + TokenGroupSummary, + TokenTotals, +} from "./token-analytics.js"; +export { aggregateToolAnalytics, countInterventions } from "./tool-analytics.js"; +export type { + ToolAnalytics, + ToolAnalyticsQuery, + ToolCategoryCount, + InterventionBreakdown, +} from "./tool-analytics.js"; +export { aggregateActivityAnalytics } from "./activity-analytics.js"; +export type { + ActivityAnalytics, + ActivityAnalyticsQuery, + DailyActivity, + MttrSummary, +} from "./activity-analytics.js"; +export { aggregateProductivityAnalytics } from "./productivity-analytics.js"; +export type { + ProductivityAnalytics, + ProductivityAnalyticsQuery, + LanguageCount, + LocSummary, +} from "./productivity-analytics.js"; export { STALLED_REVIEW_REENQUEUE_THRESHOLD, STALLED_REVIEW_INVALID_TRANSITION_THRESHOLD, @@ -674,6 +703,8 @@ export { isPrEntityActionable, isPrEntityAutoMergeReady, autoMergeGateReason, + summarizePrThreadActivity, + type PrThreadActivity, } from "./pr-entity.js"; export { findVitestProcessIds, diff --git a/packages/core/src/productivity-analytics.ts b/packages/core/src/productivity-analytics.ts new file mode 100644 index 0000000000..dcb5edba39 --- /dev/null +++ b/packages/core/src/productivity-analytics.ts @@ -0,0 +1,175 @@ +import type { Database } from "./db.js"; + +/** + * Productivity analytics: files modified (count + language distribution) from + * `tasks.modifiedFiles`, commit associations from `task_commit_associations`, + * pull requests from `pull_requests`, and LOC from commit diff stats. + * + * **LOC availability.** Fusion does not currently persist commit diff line + * stats (the `task_commit_associations` schema has no additions/deletions + * columns). LOC is therefore reported as the documented unavailable sentinel — + * `{ value: null, unavailable: true }` — **never `0`**, so a missing data source + * is never mistaken for "zero lines changed". When a diff-stats source is added, + * fill {@link LocSummary.value} and clear `unavailable`. + * + * Inclusivity: `from`/`to` bounds are inclusive. Tasks are filtered by + * `updatedAt` (the last time the task — and therefore its modifiedFiles — was + * touched); commit associations by `authoredAt`; PRs by `createdAt`. + */ + +export interface ProductivityAnalyticsQuery { + /** ISO-8601 lower bound (inclusive). */ + from?: string; + /** ISO-8601 upper bound (inclusive). */ + to?: string; +} + +/** A single language's modified-file count. */ +export interface LanguageCount { + /** Lowercased file extension (no dot), or `other` when none. */ + language: string; + count: number; +} + +/** + * LOC summary. `value` is null and `unavailable` true until a commit diff-stats + * source exists — never `0`. + */ +export interface LocSummary { + value: number | null; + unavailable: boolean; +} + +export interface ProductivityAnalytics { + from: string | null; + to: string | null; + /** Total modified-file paths across matched tasks. */ + modifiedFiles: number; + /** Modified files grouped by language (extension), descending by count. */ + byLanguage: LanguageCount[]; + /** Rows in `task_commit_associations` in range. */ + commits: number; + /** Rows in `pull_requests` in range. */ + pullRequests: number; + /** LOC from commit diff stats — unavailable until a source exists. */ + loc: LocSummary; +} + +interface CountRow { + count: number; +} + +interface ModifiedFilesRow { + modifiedFiles: string | null; +} + +/** Extract a coarse language key from a file path (its lowercased extension). */ +function languageOf(path: string): string { + const base = path.split("/").pop() ?? path; + const dot = base.lastIndexOf("."); + if (dot <= 0 || dot === base.length - 1) return "other"; + return base.slice(dot + 1).toLowerCase(); +} + +/** + * Aggregate productivity metrics over a date range. Empty range yields zeroed + * structures (not nulls); LOC is always the unavailable sentinel until a + * diff-stats source is wired. + */ +export function aggregateProductivityAnalytics( + db: Database, + query: ProductivityAnalyticsQuery = {}, +): ProductivityAnalytics { + // Modified files: read the JSON array off tasks updated in range. + const taskClauses: string[] = [ + "modifiedFiles IS NOT NULL", + "modifiedFiles NOT IN ('', '[]')", + ]; + const taskParams: string[] = []; + if (query.from !== undefined) { + taskClauses.push("updatedAt >= ?"); + taskParams.push(query.from); + } + if (query.to !== undefined) { + taskClauses.push("updatedAt <= ?"); + taskParams.push(query.to); + } + const taskRows = db + .prepare( + `SELECT modifiedFiles FROM tasks WHERE ${taskClauses.join(" AND ")}`, + ) + .all(...taskParams) as ModifiedFilesRow[]; + + let modifiedFiles = 0; + const langMap = new Map(); + for (const row of taskRows) { + if (!row.modifiedFiles) continue; + let files: unknown; + try { + files = JSON.parse(row.modifiedFiles); + } catch { + continue; + } + if (!Array.isArray(files)) continue; + for (const f of files) { + if (typeof f !== "string" || f.length === 0) continue; + modifiedFiles += 1; + const lang = languageOf(f); + langMap.set(lang, (langMap.get(lang) ?? 0) + 1); + } + } + const byLanguage: LanguageCount[] = [...langMap.entries()] + .map(([language, count]) => ({ language, count })) + .sort((a, b) => b.count - a.count); + + // Commits from task_commit_associations (by authoredAt). + const commitClauses: string[] = []; + const commitParams: string[] = []; + if (query.from !== undefined) { + commitClauses.push("authoredAt >= ?"); + commitParams.push(query.from); + } + if (query.to !== undefined) { + commitClauses.push("authoredAt <= ?"); + commitParams.push(query.to); + } + const commitWhere = + commitClauses.length > 0 ? `WHERE ${commitClauses.join(" AND ")}` : ""; + const commits = ( + db + .prepare( + `SELECT COUNT(*) AS count FROM task_commit_associations ${commitWhere}`, + ) + .get(...commitParams) as CountRow + ).count; + + // Pull requests. `pull_requests.createdAt` is an INTEGER epoch-ms column, so + // convert the ISO bounds to epoch ms for comparison. + const prClauses: string[] = []; + const prParams: number[] = []; + if (query.from !== undefined) { + prClauses.push("createdAt >= ?"); + prParams.push(Date.parse(query.from)); + } + if (query.to !== undefined) { + prClauses.push("createdAt <= ?"); + prParams.push(Date.parse(query.to)); + } + const prWhere = prClauses.length > 0 ? `WHERE ${prClauses.join(" AND ")}` : ""; + const pullRequests = ( + db + .prepare(`SELECT COUNT(*) AS count FROM pull_requests ${prWhere}`) + .get(...prParams) as CountRow + ).count; + + return { + from: query.from ?? null, + to: query.to ?? null, + modifiedFiles, + byLanguage, + commits, + pullRequests, + // No commit diff-stats source yet — unavailable, never 0. + loc: { value: null, unavailable: true }, + }; +} diff --git a/packages/core/src/token-analytics.ts b/packages/core/src/token-analytics.ts new file mode 100644 index 0000000000..695cf5eb17 --- /dev/null +++ b/packages/core/src/token-analytics.ts @@ -0,0 +1,175 @@ +import type { Database } from "./db.js"; + +/** + * Token-consumption analytics over the `tasks` table, generalizing the fixed + * 24h/7d/all-time windows of `agent-token-usage.ts` to an arbitrary `(from, to)` + * range. Sums the `tokenUsage*` columns filtered by `tokenUsageLastUsedAt` and + * groups by model / provider / node / agent. + * + * Inclusivity: `from`/`to` bounds are **inclusive** (`>= from AND <= to`), + * matching `usage-events.ts` and the range-scan house style. A task whose + * `tokenUsageLastUsedAt` is exactly equal to `from` is therefore included. + * + * Pure read-only aggregation: takes a `Database` handle and returns plain data. + */ + +/** Dimension to group token totals by. */ +export type TokenGroupBy = "model" | "provider" | "node" | "agent"; + +/** Summed token counts for a group (or the grand total). */ +export interface TokenTotals { + inputTokens: number; + outputTokens: number; + cachedTokens: number; + cacheWriteTokens: number; + totalTokens: number; + /** Number of tasks that contributed to these totals. */ + nTasks: number; +} + +/** One group's token totals, keyed by the grouped dimension value. */ +export interface TokenGroupSummary extends TokenTotals { + /** The group key (model id, provider, nodeId, or agentId); null when unset. */ + key: string | null; +} + +/** Result of {@link aggregateTokenAnalytics}. */ +export interface TokenAnalytics { + from: string | null; + to: string | null; + groupBy: TokenGroupBy | null; + /** Grand total across all matched tasks. */ + totals: TokenTotals; + /** Per-group totals; empty array when no `groupBy` requested. */ + groups: TokenGroupSummary[]; +} + +export interface TokenAnalyticsQuery { + /** ISO-8601 lower bound (inclusive) on `tokenUsageLastUsedAt`. */ + from?: string; + /** ISO-8601 upper bound (inclusive) on `tokenUsageLastUsedAt`. */ + to?: string; + groupBy?: TokenGroupBy; +} + +function emptyTotals(): TokenTotals { + return { + inputTokens: 0, + outputTokens: 0, + cachedTokens: 0, + cacheWriteTokens: 0, + totalTokens: 0, + nTasks: 0, + }; +} + +interface TaskTokenRow { + inputTokens: number | null; + outputTokens: number | null; + cachedTokens: number | null; + cacheWriteTokens: number | null; + totalTokens: number | null; + modelProvider: string | null; + modelId: string | null; + checkoutNodeId: string | null; + assignedAgentId: string | null; +} + +function groupKeyFor(row: TaskTokenRow, groupBy: TokenGroupBy): string | null { + switch (groupBy) { + case "model": + return row.modelId; + case "provider": + return row.modelProvider; + case "node": + return row.checkoutNodeId; + case "agent": + return row.assignedAgentId; + } +} + +function addRow(totals: TokenTotals, row: TaskTokenRow): void { + totals.inputTokens += row.inputTokens ?? 0; + totals.outputTokens += row.outputTokens ?? 0; + totals.cachedTokens += row.cachedTokens ?? 0; + totals.cacheWriteTokens += row.cacheWriteTokens ?? 0; + // Prefer the persisted total when present; otherwise derive it from the parts + // so callers always get a coherent `totalTokens` even on older rows. + const persistedTotal = row.totalTokens; + totals.totalTokens += + persistedTotal ?? + (row.inputTokens ?? 0) + + (row.outputTokens ?? 0) + + (row.cachedTokens ?? 0) + + (row.cacheWriteTokens ?? 0); + totals.nTasks += 1; +} + +/** + * Aggregate per-task token usage over a date range, optionally grouped. + * + * Tasks are matched by `tokenUsageLastUsedAt` within `[from, to]` (inclusive). + * Tasks with no token usage (`tokenUsageLastUsedAt IS NULL`) are excluded. An + * empty range yields zeroed `totals` and an empty `groups` array — never nulls. + */ +export function aggregateTokenAnalytics( + db: Database, + query: TokenAnalyticsQuery = {}, +): TokenAnalytics { + const clauses: string[] = ["tokenUsageLastUsedAt IS NOT NULL"]; + const params: string[] = []; + if (query.from !== undefined) { + clauses.push("tokenUsageLastUsedAt >= ?"); + params.push(query.from); + } + if (query.to !== undefined) { + clauses.push("tokenUsageLastUsedAt <= ?"); + params.push(query.to); + } + const where = `WHERE ${clauses.join(" AND ")}`; + + const rows = db + .prepare( + `SELECT + tokenUsageInputTokens AS inputTokens, + tokenUsageOutputTokens AS outputTokens, + tokenUsageCachedTokens AS cachedTokens, + tokenUsageCacheWriteTokens AS cacheWriteTokens, + tokenUsageTotalTokens AS totalTokens, + modelProvider, + modelId, + checkoutNodeId, + assignedAgentId + FROM tasks ${where}`, + ) + .all(...params) as TaskTokenRow[]; + + const totals = emptyTotals(); + const groupMap = new Map(); + const groupBy = query.groupBy; + + for (const row of rows) { + addRow(totals, row); + if (groupBy) { + const key = groupKeyFor(row, groupBy); + let group = groupMap.get(key); + if (!group) { + group = { key, ...emptyTotals() }; + groupMap.set(key, group); + } + addRow(group, row); + } + } + + const groups = [...groupMap.values()].sort( + (a, b) => b.totalTokens - a.totalTokens, + ); + + return { + from: query.from ?? null, + to: query.to ?? null, + groupBy: groupBy ?? null, + totals, + groups, + }; +} diff --git a/packages/core/src/tool-analytics.ts b/packages/core/src/tool-analytics.ts new file mode 100644 index 0000000000..e1135648bd --- /dev/null +++ b/packages/core/src/tool-analytics.ts @@ -0,0 +1,221 @@ +import type { Database } from "./db.js"; +import type { SteeringComment } from "./types.js"; + +/** + * Tool-usage analytics over `usage_events`, plus the **autonomy ratio**. + * + * Autonomy ratio = tool_call count / human-intervention count. The denominator + * is NOT raw user messages (which trend to zero for autonomous execution); it is + * the count of human interventions, which has **three distinct sources** — they + * are not one queryable table: + * + * 1. **Approvals** — rows in `approval_request_audit_events` whose `eventType` + * is `created` or `approved` (a human was asked to / did approve an action), + * timestamped by `createdAt`. + * 2. **User-authored steers** — entries in the `steeringComments` JSON column + * on the `tasks` row, filtered to `author === "user"` (agent-authored steers + * are excluded), timestamped by each comment's `createdAt`. + * 3. **Waiting-on-input** — a task *status*, not a counted event; intentionally + * DROPPED here (no concrete answer event is defined). + * + * A fully-autonomous session (zero interventions) must not divide by zero or + * report ∞: when `interventions === 0` the ratio falls back to + * tool-calls-per-session (`toolCalls / max(sessions, 1)`), and the result flags + * `interventions: 0` so callers can render it as "fully autonomous". + * + * Inclusivity: `from`/`to` bounds are inclusive, matching `usage-events.ts`. + */ + +export interface ToolAnalyticsQuery { + /** ISO-8601 lower bound (inclusive). */ + from?: string; + /** ISO-8601 upper bound (inclusive). */ + to?: string; +} + +/** Tool-call count for a single coarse category. */ +export interface ToolCategoryCount { + category: string; + count: number; +} + +/** Breakdown of the autonomy-ratio denominator by source. */ +export interface InterventionBreakdown { + /** `created`/`approved` rows in `approval_request_audit_events`. */ + approvals: number; + /** `steeringComments` entries with `author === "user"`. */ + userSteers: number; + /** Total human interventions (sum of the components above). */ + total: number; +} + +export interface ToolAnalytics { + from: string | null; + to: string | null; + /** Total `tool_call` events in range. */ + toolCalls: number; + /** Tool calls grouped by `category`, descending by count. */ + byCategory: ToolCategoryCount[]; + /** Distinct sessions (`session_start` events) in range. */ + sessions: number; + interventions: InterventionBreakdown; + /** + * Autonomy ratio. When `interventions.total > 0` this is + * `toolCalls / interventions.total`. When there are zero interventions it is + * tool-calls-per-session (`toolCalls / max(sessions, 1)`) and + * `fullyAutonomous` is true — never ∞ or NaN. + */ + autonomyRatio: number; + /** True when zero human interventions were recorded in range. */ + fullyAutonomous: boolean; +} + +interface CountRow { + count: number; +} + +interface CategoryRow { + category: string | null; + count: number; +} + +interface SteeringRow { + steeringComments: string | null; +} + +function inRange(ts: string, from?: string, to?: string): boolean { + if (from !== undefined && ts < from) return false; + if (to !== undefined && ts > to) return false; + return true; +} + +/** + * Count human interventions from the three named sources (waiting-on-input is a + * status, not counted). Returns the per-source breakdown plus the total. + */ +export function countInterventions( + db: Database, + query: ToolAnalyticsQuery = {}, +): InterventionBreakdown { + // Source 1: approvals. `approval_request_audit_events.createdAt` is the ts; + // count only the human-touch event types. + const approvalClauses: string[] = ["eventType IN ('created', 'approved')"]; + const approvalParams: string[] = []; + if (query.from !== undefined) { + approvalClauses.push("createdAt >= ?"); + approvalParams.push(query.from); + } + if (query.to !== undefined) { + approvalClauses.push("createdAt <= ?"); + approvalParams.push(query.to); + } + const approvals = ( + db + .prepare( + `SELECT COUNT(*) AS count FROM approval_request_audit_events WHERE ${approvalClauses.join(" AND ")}`, + ) + .get(...approvalParams) as CountRow + ).count; + + // Source 2: user-authored steers from the `steeringComments` JSON on tasks. + // This re-introduces a per-task JSON read (documented in U2). Only rows with a + // non-empty JSON array are scanned. + const steeringRows = db + .prepare( + `SELECT steeringComments FROM tasks + WHERE steeringComments IS NOT NULL AND steeringComments NOT IN ('', '[]')`, + ) + .all() as SteeringRow[]; + let userSteers = 0; + for (const row of steeringRows) { + if (!row.steeringComments) continue; + let parsed: SteeringComment[]; + try { + parsed = JSON.parse(row.steeringComments) as SteeringComment[]; + } catch { + continue; + } + if (!Array.isArray(parsed)) continue; + for (const comment of parsed) { + if (comment?.author !== "user") continue; + if (!inRange(comment.createdAt ?? "", query.from, query.to)) continue; + userSteers += 1; + } + } + + return { approvals, userSteers, total: approvals + userSteers }; +} + +/** + * Aggregate tool usage and the autonomy ratio over a date range. + * + * Empty range yields zeroed structures (not nulls) and `autonomyRatio: 0`. + */ +export function aggregateToolAnalytics( + db: Database, + query: ToolAnalyticsQuery = {}, +): ToolAnalytics { + const eventClauses: string[] = []; + const eventParams: string[] = []; + if (query.from !== undefined) { + eventClauses.push("ts >= ?"); + eventParams.push(query.from); + } + if (query.to !== undefined) { + eventClauses.push("ts <= ?"); + eventParams.push(query.to); + } + const rangeWhere = eventClauses.length > 0 ? `AND ${eventClauses.join(" AND ")}` : ""; + + const toolCalls = ( + db + .prepare( + `SELECT COUNT(*) AS count FROM usage_events WHERE kind = 'tool_call' ${rangeWhere}`, + ) + .get(...eventParams) as CountRow + ).count; + + const categoryRows = db + .prepare( + `SELECT category AS category, COUNT(*) AS count + FROM usage_events + WHERE kind = 'tool_call' ${rangeWhere} + GROUP BY category`, + ) + .all(...eventParams) as CategoryRow[]; + const byCategory: ToolCategoryCount[] = categoryRows + .map((r) => ({ category: r.category ?? "other", count: r.count })) + .sort((a, b) => b.count - a.count); + + const sessions = ( + db + .prepare( + `SELECT COUNT(*) AS count FROM usage_events WHERE kind = 'session_start' ${rangeWhere}`, + ) + .get(...eventParams) as CountRow + ).count; + + const interventions = countInterventions(db, query); + + let autonomyRatio: number; + let fullyAutonomous: boolean; + if (interventions.total > 0) { + autonomyRatio = toolCalls / interventions.total; + fullyAutonomous = false; + } else { + // Zero interventions: report tool-calls-per-session, never ∞ / divide-by-zero. + autonomyRatio = toolCalls / Math.max(sessions, 1); + fullyAutonomous = true; + } + + return { + from: query.from ?? null, + to: query.to ?? null, + toolCalls, + byCategory, + sessions, + interventions, + autonomyRatio, + fullyAutonomous, + }; +}