feat(analytics): U2 — core date-range aggregators (tokens/tools/activity/productivity)
Reusable aggregate() over tasks + usage_events with model/provider/node/agent grouping. Autonomy ratio sources interventions from approval audit events + user-authored steers (agent steers excluded); fully-autonomous sessions report tool-calls-per-session, never divide-by-zero. LOC/MTTR seams flagged unavailable.
This commit is contained in:
91
packages/core/src/__tests__/activity-analytics.test.ts
Normal file
91
packages/core/src/__tests__/activity-analytics.test.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
import { Database } from "../db.js";
|
||||
import { emitUsageEvent } from "../usage-events.js";
|
||||
import { aggregateActivityAnalytics } from "../activity-analytics.js";
|
||||
|
||||
function insertCliSession(db: Database, id: string, createdAt: string): void {
|
||||
db.prepare(
|
||||
`INSERT INTO cli_sessions
|
||||
(id, purpose, projectId, adapterId, agentState, createdAt, updatedAt)
|
||||
VALUES (?, 'task', 'proj-1', 'claude-local', 'running', ?, ?)`,
|
||||
).run(id, createdAt, createdAt);
|
||||
}
|
||||
|
||||
describe("activity-analytics", () => {
|
||||
let tmpDir: string;
|
||||
let db: Database;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "kb-activity-analytics-"));
|
||||
db = new Database(join(tmpDir, ".fusion"));
|
||||
db.init();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
db.close();
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("counts sessions, messages, and distinct active nodes/agents over a range", () => {
|
||||
insertCliSession(db, "s1", "2026-03-01T00:00:00.000Z");
|
||||
insertCliSession(db, "s2", "2026-03-02T00:00:00.000Z");
|
||||
// session outside range
|
||||
insertCliSession(db, "s-old", "2025-01-01T00:00:00.000Z");
|
||||
|
||||
emitUsageEvent(db, { kind: "user_message", agentId: "agent-1", nodeId: "node-1", ts: "2026-03-01T00:00:00.000Z" });
|
||||
emitUsageEvent(db, { kind: "user_message", agentId: "agent-2", nodeId: "node-1", ts: "2026-03-01T01:00:00.000Z" });
|
||||
emitUsageEvent(db, { kind: "tool_call", agentId: "agent-2", nodeId: "node-2", ts: "2026-03-02T00:00:00.000Z" });
|
||||
|
||||
const result = aggregateActivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(result.sessions).toBe(2);
|
||||
expect(result.messages).toBe(2);
|
||||
expect(result.activeNodes).toBe(2); // node-1, node-2
|
||||
expect(result.activeAgents).toBe(2); // agent-1, agent-2
|
||||
});
|
||||
|
||||
it("produces a per-day breakdown ascending by day", () => {
|
||||
emitUsageEvent(db, { kind: "user_message", agentId: "agent-1", nodeId: "node-1", ts: "2026-03-01T08:00:00.000Z" });
|
||||
emitUsageEvent(db, { kind: "tool_call", agentId: "agent-1", nodeId: "node-1", ts: "2026-03-01T09:00:00.000Z" });
|
||||
emitUsageEvent(db, { kind: "user_message", agentId: "agent-2", nodeId: "node-2", ts: "2026-03-02T08:00:00.000Z" });
|
||||
|
||||
const result = aggregateActivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(result.daily.map((d) => d.day)).toEqual(["2026-03-01", "2026-03-02"]);
|
||||
expect(result.daily[0]).toMatchObject({ day: "2026-03-01", activeNodes: 1, activeAgents: 1, messages: 1 });
|
||||
expect(result.daily[1]).toMatchObject({ day: "2026-03-02", activeNodes: 1, activeAgents: 1, messages: 1 });
|
||||
});
|
||||
|
||||
it("computes stickiness = DAU/MAU", () => {
|
||||
// Day 1: agents a,b active. Day 2: agent a active. MAU = {a,b} = 2.
|
||||
// DAU = mean(2, 1) = 1.5. stickiness = 1.5 / 2 = 0.75.
|
||||
emitUsageEvent(db, { kind: "tool_call", agentId: "a", nodeId: "n1", ts: "2026-03-01T00:00:00.000Z" });
|
||||
emitUsageEvent(db, { kind: "tool_call", agentId: "b", nodeId: "n1", ts: "2026-03-01T01:00:00.000Z" });
|
||||
emitUsageEvent(db, { kind: "tool_call", agentId: "a", nodeId: "n1", ts: "2026-03-02T00:00:00.000Z" });
|
||||
|
||||
const result = aggregateActivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(result.activeAgents).toBe(2);
|
||||
expect(result.stickiness).toBeCloseTo(0.75, 5);
|
||||
});
|
||||
|
||||
it("empty range returns zeroed structures, not nulls", () => {
|
||||
insertCliSession(db, "s1", "2026-03-01T00:00:00.000Z");
|
||||
emitUsageEvent(db, { kind: "user_message", agentId: "a", nodeId: "n1", ts: "2026-03-01T00:00:00.000Z" });
|
||||
|
||||
const result = aggregateActivityAnalytics(db, { from: "2027-01-01T00:00:00.000Z", to: "2027-12-31T00:00:00.000Z" });
|
||||
expect(result.sessions).toBe(0);
|
||||
expect(result.messages).toBe(0);
|
||||
expect(result.activeNodes).toBe(0);
|
||||
expect(result.activeAgents).toBe(0);
|
||||
expect(result.daily).toEqual([]);
|
||||
expect(result.stickiness).toBe(0);
|
||||
});
|
||||
|
||||
it("leaves a clean MTTR seam for U13 (unavailable, not 0)", () => {
|
||||
const result = aggregateActivityAnalytics(db, {});
|
||||
expect(result.mttr).toEqual({ value: null, unavailable: true });
|
||||
});
|
||||
});
|
||||
103
packages/core/src/__tests__/productivity-analytics.test.ts
Normal file
103
packages/core/src/__tests__/productivity-analytics.test.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
import { Database } from "../db.js";
|
||||
import { aggregateProductivityAnalytics } from "../productivity-analytics.js";
|
||||
|
||||
function insertTaskWithFiles(db: Database, id: string, files: string[], updatedAt: string): void {
|
||||
db.prepare(
|
||||
`INSERT INTO tasks (id, description, "column", createdAt, updatedAt, modifiedFiles)
|
||||
VALUES (?, 'desc', 'todo', ?, ?, ?)`,
|
||||
).run(id, updatedAt, updatedAt, JSON.stringify(files));
|
||||
}
|
||||
|
||||
function insertCommit(db: Database, id: string, sha: string, authoredAt: string): void {
|
||||
db.prepare(
|
||||
`INSERT INTO task_commit_associations
|
||||
(id, taskLineageId, taskIdSnapshot, commitSha, commitSubject, authoredAt,
|
||||
matchedBy, confidence, createdAt, updatedAt)
|
||||
VALUES (?, 'lin-1', 't-1', ?, 'subj', ?, 'canonical-lineage-trailer', 'canonical', ?, ?)`,
|
||||
).run(id, sha, authoredAt, authoredAt, authoredAt);
|
||||
}
|
||||
|
||||
function insertPr(db: Database, id: string, createdAtMs: number): void {
|
||||
db.prepare(
|
||||
`INSERT INTO pull_requests
|
||||
(id, sourceType, sourceId, repo, headBranch, state, createdAt, updatedAt)
|
||||
VALUES (?, 'task', ?, 'org/repo', ?, 'open', ?, ?)`,
|
||||
).run(id, `src-${id}`, `branch-${id}`, createdAtMs, createdAtMs);
|
||||
}
|
||||
|
||||
describe("productivity-analytics", () => {
|
||||
let tmpDir: string;
|
||||
let db: Database;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "kb-productivity-analytics-"));
|
||||
db = new Database(join(tmpDir, ".fusion"));
|
||||
db.init();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
db.close();
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("counts modified files and language distribution", () => {
|
||||
insertTaskWithFiles(db, "t1", ["src/a.ts", "src/b.ts", "README.md"], "2026-03-01T00:00:00.000Z");
|
||||
insertTaskWithFiles(db, "t2", ["src/c.ts", "style.css"], "2026-03-02T00:00:00.000Z");
|
||||
|
||||
const result = aggregateProductivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(result.modifiedFiles).toBe(5);
|
||||
const byLang = new Map(result.byLanguage.map((l) => [l.language, l.count]));
|
||||
expect(byLang.get("ts")).toBe(3);
|
||||
expect(byLang.get("md")).toBe(1);
|
||||
expect(byLang.get("css")).toBe(1);
|
||||
// sorted descending by count
|
||||
expect(result.byLanguage[0]).toEqual({ language: "ts", count: 3 });
|
||||
});
|
||||
|
||||
it("counts commit associations and pull requests in range", () => {
|
||||
insertCommit(db, "c1", "sha1", "2026-03-01T00:00:00.000Z");
|
||||
insertCommit(db, "c2", "sha2", "2026-03-02T00:00:00.000Z");
|
||||
insertCommit(db, "c-old", "sha-old", "2025-01-01T00:00:00.000Z");
|
||||
|
||||
insertPr(db, "pr1", Date.parse("2026-03-01T00:00:00.000Z"));
|
||||
insertPr(db, "pr2", Date.parse("2026-03-10T00:00:00.000Z"));
|
||||
insertPr(db, "pr-old", Date.parse("2025-01-01T00:00:00.000Z"));
|
||||
|
||||
const result = aggregateProductivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(result.commits).toBe(2);
|
||||
expect(result.pullRequests).toBe(2);
|
||||
});
|
||||
|
||||
it("reports LOC as unavailable (null + unavailable:true), never 0", () => {
|
||||
insertTaskWithFiles(db, "t1", ["src/a.ts"], "2026-03-01T00:00:00.000Z");
|
||||
const result = aggregateProductivityAnalytics(db, {});
|
||||
expect(result.loc).toEqual({ value: null, unavailable: true });
|
||||
expect(result.loc.value).not.toBe(0);
|
||||
});
|
||||
|
||||
it("empty range returns zeroed structures, not nulls", () => {
|
||||
insertTaskWithFiles(db, "t1", ["src/a.ts"], "2026-03-01T00:00:00.000Z");
|
||||
insertCommit(db, "c1", "sha1", "2026-03-01T00:00:00.000Z");
|
||||
insertPr(db, "pr1", Date.parse("2026-03-01T00:00:00.000Z"));
|
||||
|
||||
const result = aggregateProductivityAnalytics(db, { from: "2027-01-01T00:00:00.000Z", to: "2027-12-31T00:00:00.000Z" });
|
||||
expect(result.modifiedFiles).toBe(0);
|
||||
expect(result.byLanguage).toEqual([]);
|
||||
expect(result.commits).toBe(0);
|
||||
expect(result.pullRequests).toBe(0);
|
||||
// LOC unavailable regardless of range
|
||||
expect(result.loc).toEqual({ value: null, unavailable: true });
|
||||
});
|
||||
|
||||
it("includes a boundary task exactly at `from`", () => {
|
||||
insertTaskWithFiles(db, "boundary", ["x.ts"], "2026-03-01T00:00:00.000Z");
|
||||
const result = aggregateProductivityAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(result.modifiedFiles).toBe(1);
|
||||
});
|
||||
});
|
||||
157
packages/core/src/__tests__/token-analytics.test.ts
Normal file
157
packages/core/src/__tests__/token-analytics.test.ts
Normal file
@@ -0,0 +1,157 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
import { Database } from "../db.js";
|
||||
import { aggregateTokenAnalytics } from "../token-analytics.js";
|
||||
|
||||
interface TaskSeed {
|
||||
id: string;
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
cachedTokens?: number;
|
||||
cacheWriteTokens?: number;
|
||||
totalTokens?: number | null;
|
||||
lastUsedAt: string | null;
|
||||
modelProvider?: string | null;
|
||||
modelId?: string | null;
|
||||
nodeId?: string | null;
|
||||
agentId?: string | null;
|
||||
}
|
||||
|
||||
function insertTask(db: Database, t: TaskSeed): void {
|
||||
db.prepare(
|
||||
`INSERT INTO tasks
|
||||
(id, description, "column", createdAt, updatedAt,
|
||||
tokenUsageInputTokens, tokenUsageOutputTokens, tokenUsageCachedTokens,
|
||||
tokenUsageCacheWriteTokens, tokenUsageTotalTokens, tokenUsageLastUsedAt,
|
||||
modelProvider, modelId, checkoutNodeId, assignedAgentId)
|
||||
VALUES (?, 'desc', 'todo', '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z',
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
).run(
|
||||
t.id,
|
||||
t.inputTokens ?? null,
|
||||
t.outputTokens ?? null,
|
||||
t.cachedTokens ?? null,
|
||||
t.cacheWriteTokens ?? null,
|
||||
t.totalTokens === undefined ? null : t.totalTokens,
|
||||
t.lastUsedAt,
|
||||
t.modelProvider ?? null,
|
||||
t.modelId ?? null,
|
||||
t.nodeId ?? null,
|
||||
t.agentId ?? null,
|
||||
);
|
||||
}
|
||||
|
||||
describe("token-analytics", () => {
|
||||
let tmpDir: string;
|
||||
let db: Database;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "kb-token-analytics-"));
|
||||
db = new Database(join(tmpDir, ".fusion"));
|
||||
db.init();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
db.close();
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("returns correct per-model token totals for 5 tasks across 2 models", () => {
|
||||
// 3 tasks on model-A, 2 on model-B, all within range.
|
||||
insertTask(db, { id: "t1", inputTokens: 100, outputTokens: 50, totalTokens: 150, lastUsedAt: "2026-03-01T00:00:00.000Z", modelId: "model-A", modelProvider: "anthropic" });
|
||||
insertTask(db, { id: "t2", inputTokens: 200, outputTokens: 80, totalTokens: 280, lastUsedAt: "2026-03-02T00:00:00.000Z", modelId: "model-A", modelProvider: "anthropic" });
|
||||
insertTask(db, { id: "t3", inputTokens: 300, outputTokens: 20, totalTokens: 320, lastUsedAt: "2026-03-03T00:00:00.000Z", modelId: "model-A", modelProvider: "anthropic" });
|
||||
insertTask(db, { id: "t4", inputTokens: 10, outputTokens: 5, totalTokens: 15, lastUsedAt: "2026-03-04T00:00:00.000Z", modelId: "model-B", modelProvider: "openai" });
|
||||
insertTask(db, { id: "t5", inputTokens: 40, outputTokens: 60, totalTokens: 100, lastUsedAt: "2026-03-05T00:00:00.000Z", modelId: "model-B", modelProvider: "openai" });
|
||||
|
||||
const result = aggregateTokenAnalytics(db, {
|
||||
from: "2026-03-01T00:00:00.000Z",
|
||||
to: "2026-03-31T00:00:00.000Z",
|
||||
groupBy: "model",
|
||||
});
|
||||
|
||||
expect(result.totals.inputTokens).toBe(650);
|
||||
expect(result.totals.outputTokens).toBe(215);
|
||||
expect(result.totals.totalTokens).toBe(865);
|
||||
expect(result.totals.nTasks).toBe(5);
|
||||
|
||||
const groups = new Map(result.groups.map((g) => [g.key, g]));
|
||||
expect(groups.get("model-A")!.inputTokens).toBe(600);
|
||||
expect(groups.get("model-A")!.totalTokens).toBe(750);
|
||||
expect(groups.get("model-A")!.nTasks).toBe(3);
|
||||
expect(groups.get("model-B")!.inputTokens).toBe(50);
|
||||
expect(groups.get("model-B")!.totalTokens).toBe(115);
|
||||
expect(groups.get("model-B")!.nTasks).toBe(2);
|
||||
// groups sorted descending by totalTokens
|
||||
expect(result.groups[0].key).toBe("model-A");
|
||||
});
|
||||
|
||||
it("groups by provider, node, and agent", () => {
|
||||
insertTask(db, { id: "t1", inputTokens: 100, totalTokens: 100, lastUsedAt: "2026-03-01T00:00:00.000Z", modelProvider: "anthropic", nodeId: "node-1", agentId: "agent-x" });
|
||||
insertTask(db, { id: "t2", inputTokens: 200, totalTokens: 200, lastUsedAt: "2026-03-02T00:00:00.000Z", modelProvider: "openai", nodeId: "node-1", agentId: "agent-y" });
|
||||
|
||||
const byProvider = aggregateTokenAnalytics(db, { groupBy: "provider" });
|
||||
expect(new Map(byProvider.groups.map((g) => [g.key, g.totalTokens]))).toEqual(
|
||||
new Map([["anthropic", 100], ["openai", 200]]),
|
||||
);
|
||||
|
||||
const byNode = aggregateTokenAnalytics(db, { groupBy: "node" });
|
||||
expect(byNode.groups).toHaveLength(1);
|
||||
expect(byNode.groups[0].key).toBe("node-1");
|
||||
expect(byNode.groups[0].totalTokens).toBe(300);
|
||||
|
||||
const byAgent = aggregateTokenAnalytics(db, { groupBy: "agent" });
|
||||
expect(new Map(byAgent.groups.map((g) => [g.key, g.totalTokens]))).toEqual(
|
||||
new Map([["agent-x", 100], ["agent-y", 200]]),
|
||||
);
|
||||
});
|
||||
|
||||
it("empty range returns zeroed structures, not nulls", () => {
|
||||
insertTask(db, { id: "t1", inputTokens: 100, totalTokens: 100, lastUsedAt: "2026-03-01T00:00:00.000Z", modelId: "model-A" });
|
||||
|
||||
const result = aggregateTokenAnalytics(db, {
|
||||
from: "2027-01-01T00:00:00.000Z",
|
||||
to: "2027-12-31T00:00:00.000Z",
|
||||
groupBy: "model",
|
||||
});
|
||||
expect(result.totals).toEqual({
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
cachedTokens: 0,
|
||||
cacheWriteTokens: 0,
|
||||
totalTokens: 0,
|
||||
nTasks: 0,
|
||||
});
|
||||
expect(result.groups).toEqual([]);
|
||||
});
|
||||
|
||||
it("includes a boundary task exactly at `from` (inclusive lower bound)", () => {
|
||||
insertTask(db, { id: "boundary", inputTokens: 42, totalTokens: 42, lastUsedAt: "2026-03-01T00:00:00.000Z", modelId: "model-A" });
|
||||
|
||||
const result = aggregateTokenAnalytics(db, {
|
||||
from: "2026-03-01T00:00:00.000Z",
|
||||
to: "2026-03-31T00:00:00.000Z",
|
||||
});
|
||||
expect(result.totals.nTasks).toBe(1);
|
||||
expect(result.totals.inputTokens).toBe(42);
|
||||
});
|
||||
|
||||
it("excludes tasks with no token usage (lastUsedAt null)", () => {
|
||||
insertTask(db, { id: "no-usage", lastUsedAt: null, modelId: "model-A" });
|
||||
insertTask(db, { id: "has-usage", inputTokens: 5, totalTokens: 5, lastUsedAt: "2026-03-01T00:00:00.000Z", modelId: "model-A" });
|
||||
|
||||
const result = aggregateTokenAnalytics(db, {});
|
||||
expect(result.totals.nTasks).toBe(1);
|
||||
expect(result.totals.inputTokens).toBe(5);
|
||||
});
|
||||
|
||||
it("derives totalTokens from parts when the persisted total is null", () => {
|
||||
insertTask(db, { id: "t1", inputTokens: 10, outputTokens: 20, cachedTokens: 5, cacheWriteTokens: 1, totalTokens: null, lastUsedAt: "2026-03-01T00:00:00.000Z", modelId: "model-A" });
|
||||
const result = aggregateTokenAnalytics(db, {});
|
||||
expect(result.totals.totalTokens).toBe(36);
|
||||
});
|
||||
});
|
||||
146
packages/core/src/__tests__/tool-analytics.test.ts
Normal file
146
packages/core/src/__tests__/tool-analytics.test.ts
Normal file
@@ -0,0 +1,146 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
import { Database } from "../db.js";
|
||||
import { emitUsageEvent } from "../usage-events.js";
|
||||
import { aggregateToolAnalytics, countInterventions } from "../tool-analytics.js";
|
||||
import type { SteeringComment } from "../types.js";
|
||||
|
||||
function insertTaskWithSteers(db: Database, id: string, steers: SteeringComment[]): void {
|
||||
db.prepare(
|
||||
`INSERT INTO tasks (id, description, "column", createdAt, updatedAt, steeringComments)
|
||||
VALUES (?, 'desc', 'todo', '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z', ?)`,
|
||||
).run(id, JSON.stringify(steers));
|
||||
}
|
||||
|
||||
function insertApprovalRequest(db: Database, id: string): void {
|
||||
db.prepare(
|
||||
`INSERT INTO approval_requests
|
||||
(id, status, requesterActorId, requesterActorType, requesterActorName,
|
||||
targetActionCategory, targetActionOperation, targetActionSummary,
|
||||
targetResourceType, targetResourceId, requestedAt, createdAt, updatedAt)
|
||||
VALUES (?, 'pending', 'a', 'agent', 'A', 'cat', 'op', 'sum', 'res', 'r1',
|
||||
'2026-03-01T00:00:00.000Z', '2026-03-01T00:00:00.000Z', '2026-03-01T00:00:00.000Z')`,
|
||||
).run(id);
|
||||
}
|
||||
|
||||
function insertApprovalEvent(db: Database, id: string, requestId: string, eventType: string, createdAt: string): void {
|
||||
db.prepare(
|
||||
`INSERT INTO approval_request_audit_events
|
||||
(id, requestId, eventType, actorId, actorType, actorName, createdAt)
|
||||
VALUES (?, ?, ?, 'u1', 'user', 'User', ?)`,
|
||||
).run(id, requestId, eventType, createdAt);
|
||||
}
|
||||
|
||||
describe("tool-analytics", () => {
|
||||
let tmpDir: string;
|
||||
let db: Database;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "kb-tool-analytics-"));
|
||||
db = new Database(join(tmpDir, ".fusion"));
|
||||
db.init();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
db.close();
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("counts tool calls by category, sorted descending", () => {
|
||||
emitUsageEvent(db, { kind: "tool_call", category: "read", ts: "2026-03-01T00:00:00.000Z" });
|
||||
emitUsageEvent(db, { kind: "tool_call", category: "read", ts: "2026-03-01T01:00:00.000Z" });
|
||||
emitUsageEvent(db, { kind: "tool_call", category: "edit", ts: "2026-03-01T02:00:00.000Z" });
|
||||
// a non-tool_call event is not counted
|
||||
emitUsageEvent(db, { kind: "user_message", ts: "2026-03-01T03:00:00.000Z" });
|
||||
|
||||
const result = aggregateToolAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(result.toolCalls).toBe(3);
|
||||
expect(result.byCategory).toEqual([
|
||||
{ category: "read", count: 2 },
|
||||
{ category: "edit", count: 1 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("autonomy denominator counts a USER steer + an approval but NOT an agent steer", () => {
|
||||
insertTaskWithSteers(db, "task-1", [
|
||||
{ id: "s1", text: "do X", createdAt: "2026-03-02T00:00:00.000Z", author: "user" },
|
||||
{ id: "s2", text: "agent note", createdAt: "2026-03-02T01:00:00.000Z", author: "agent" },
|
||||
]);
|
||||
insertApprovalRequest(db, "req-1");
|
||||
insertApprovalEvent(db, "ev-created", "req-1", "created", "2026-03-02T00:30:00.000Z");
|
||||
insertApprovalEvent(db, "ev-approved", "req-1", "approved", "2026-03-02T00:31:00.000Z");
|
||||
// a non-human eventType must NOT count
|
||||
insertApprovalEvent(db, "ev-completed", "req-1", "completed", "2026-03-02T00:32:00.000Z");
|
||||
|
||||
const breakdown = countInterventions(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(breakdown.userSteers).toBe(1); // agent steer excluded
|
||||
expect(breakdown.approvals).toBe(2); // created + approved, completed excluded
|
||||
expect(breakdown.total).toBe(3);
|
||||
});
|
||||
|
||||
it("autonomy ratio = toolCalls / interventions for an interactive session", () => {
|
||||
// 12 tool calls, 3 interventions (1 user steer + 2 approvals) -> ratio 4
|
||||
for (let i = 0; i < 12; i++) {
|
||||
emitUsageEvent(db, { kind: "tool_call", category: "read", ts: `2026-03-02T00:0${i % 6}:0${i % 6}.000Z` });
|
||||
}
|
||||
emitUsageEvent(db, { kind: "session_start", ts: "2026-03-02T00:00:00.000Z" });
|
||||
insertTaskWithSteers(db, "task-1", [{ id: "s1", text: "x", createdAt: "2026-03-02T00:10:00.000Z", author: "user" }]);
|
||||
insertApprovalRequest(db, "req-1");
|
||||
insertApprovalEvent(db, "ev-c", "req-1", "created", "2026-03-02T00:11:00.000Z");
|
||||
insertApprovalEvent(db, "ev-a", "req-1", "approved", "2026-03-02T00:12:00.000Z");
|
||||
|
||||
const result = aggregateToolAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(result.interventions.total).toBe(3);
|
||||
expect(result.toolCalls).toBe(12);
|
||||
expect(result.autonomyRatio).toBe(4);
|
||||
expect(result.fullyAutonomous).toBe(false);
|
||||
});
|
||||
|
||||
it("fully-autonomous session (zero interventions) reports tool-calls-per-session, not infinity", () => {
|
||||
// 10 tool calls across 2 sessions, zero interventions -> 5 per session
|
||||
for (let i = 0; i < 10; i++) {
|
||||
emitUsageEvent(db, { kind: "tool_call", category: "execute", ts: "2026-03-02T00:00:00.000Z" });
|
||||
}
|
||||
emitUsageEvent(db, { kind: "session_start", ts: "2026-03-02T00:00:00.000Z" });
|
||||
emitUsageEvent(db, { kind: "session_start", ts: "2026-03-02T01:00:00.000Z" });
|
||||
|
||||
const result = aggregateToolAnalytics(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(result.interventions.total).toBe(0);
|
||||
expect(result.fullyAutonomous).toBe(true);
|
||||
expect(result.autonomyRatio).toBe(5);
|
||||
expect(Number.isFinite(result.autonomyRatio)).toBe(true);
|
||||
});
|
||||
|
||||
it("zero interventions and zero sessions does not divide by zero", () => {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
emitUsageEvent(db, { kind: "tool_call", category: "read", ts: "2026-03-02T00:00:00.000Z" });
|
||||
}
|
||||
const result = aggregateToolAnalytics(db, {});
|
||||
expect(result.sessions).toBe(0);
|
||||
expect(result.fullyAutonomous).toBe(true);
|
||||
// toolCalls / max(sessions, 1) = 4 / 1
|
||||
expect(result.autonomyRatio).toBe(4);
|
||||
});
|
||||
|
||||
it("empty range returns zeroed structures, not nulls", () => {
|
||||
const result = aggregateToolAnalytics(db, { from: "2027-01-01T00:00:00.000Z", to: "2027-12-31T00:00:00.000Z" });
|
||||
expect(result.toolCalls).toBe(0);
|
||||
expect(result.byCategory).toEqual([]);
|
||||
expect(result.sessions).toBe(0);
|
||||
expect(result.interventions).toEqual({ approvals: 0, userSteers: 0, total: 0 });
|
||||
expect(result.autonomyRatio).toBe(0);
|
||||
});
|
||||
|
||||
it("user steers outside the range are not counted", () => {
|
||||
insertTaskWithSteers(db, "task-1", [
|
||||
{ id: "s1", text: "old", createdAt: "2025-01-01T00:00:00.000Z", author: "user" },
|
||||
{ id: "s2", text: "in range", createdAt: "2026-03-15T00:00:00.000Z", author: "user" },
|
||||
]);
|
||||
const breakdown = countInterventions(db, { from: "2026-03-01T00:00:00.000Z", to: "2026-03-31T00:00:00.000Z" });
|
||||
expect(breakdown.userSteers).toBe(1);
|
||||
});
|
||||
});
|
||||
193
packages/core/src/activity-analytics.ts
Normal file
193
packages/core/src/activity-analytics.ts
Normal file
@@ -0,0 +1,193 @@
|
||||
import type { Database } from "./db.js";
|
||||
|
||||
/**
|
||||
* Activity analytics: distinct active nodes/agents per day, sessions, messages,
|
||||
* and stickiness (DAU/MAU) over an arbitrary date range.
|
||||
*
|
||||
* Sessions come from `cli_sessions` (by `createdAt`); messages and node/agent
|
||||
* activity come from `usage_events`. Inclusivity: `from`/`to` are inclusive,
|
||||
* matching `usage-events.ts`.
|
||||
*
|
||||
* **MTTR seam (U13).** Mean-time-to-resolve aggregation is deliberately NOT
|
||||
* implemented here yet — it depends on the deployments/incidents tables U13
|
||||
* introduces. {@link aggregateActivityAnalytics} returns an `mttr` field set to
|
||||
* the documented "unavailable" sentinel so the shape is stable now and U13 can
|
||||
* fill it in without changing callers. See {@link MttrSummary}.
|
||||
*/
|
||||
|
||||
export interface ActivityAnalyticsQuery {
|
||||
/** ISO-8601 lower bound (inclusive). */
|
||||
from?: string;
|
||||
/** ISO-8601 upper bound (inclusive). */
|
||||
to?: string;
|
||||
}
|
||||
|
||||
/** Distinct active nodes/agents and message count for a single UTC day. */
|
||||
export interface DailyActivity {
|
||||
/** UTC date, `YYYY-MM-DD`. */
|
||||
day: string;
|
||||
activeNodes: number;
|
||||
activeAgents: number;
|
||||
messages: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* MTTR summary placeholder. U13 will populate `value` (mean minutes to resolve)
|
||||
* once deployments/incidents land; until then it is the documented unavailable
|
||||
* sentinel — `null` value with `unavailable: true`, never `0`.
|
||||
*/
|
||||
export interface MttrSummary {
|
||||
/** Mean minutes to resolve; null until U13 provides incident data. */
|
||||
value: number | null;
|
||||
/** True when MTTR cannot be computed (no incident data source yet). */
|
||||
unavailable: boolean;
|
||||
}
|
||||
|
||||
export interface ActivityAnalytics {
|
||||
from: string | null;
|
||||
to: string | null;
|
||||
/** Total `session_start` events from `cli_sessions` in range. */
|
||||
sessions: number;
|
||||
/** Total `user_message` events in range. */
|
||||
messages: number;
|
||||
/** Distinct nodes with any usage_event in range. */
|
||||
activeNodes: number;
|
||||
/** Distinct agents with any usage_event in range. */
|
||||
activeAgents: number;
|
||||
/** Per-day breakdown, ascending by day. */
|
||||
daily: DailyActivity[];
|
||||
/**
|
||||
* Stickiness = DAU/MAU. DAU = mean distinct-active-agents-per-day over the
|
||||
* range; MAU = distinct active agents over the whole range. 0 when MAU is 0.
|
||||
*/
|
||||
stickiness: number;
|
||||
/** MTTR placeholder (U13 seam). */
|
||||
mttr: MttrSummary;
|
||||
}
|
||||
|
||||
interface CountRow {
|
||||
count: number;
|
||||
}
|
||||
|
||||
interface DistinctRow {
|
||||
count: number;
|
||||
}
|
||||
|
||||
interface DayAggRow {
|
||||
day: string;
|
||||
activeNodes: number;
|
||||
activeAgents: number;
|
||||
messages: number;
|
||||
}
|
||||
|
||||
function rangeClauses(
|
||||
column: string,
|
||||
query: ActivityAnalyticsQuery,
|
||||
): { where: string; params: string[] } {
|
||||
const clauses: string[] = [];
|
||||
const params: string[] = [];
|
||||
if (query.from !== undefined) {
|
||||
clauses.push(`${column} >= ?`);
|
||||
params.push(query.from);
|
||||
}
|
||||
if (query.to !== undefined) {
|
||||
clauses.push(`${column} <= ?`);
|
||||
params.push(query.to);
|
||||
}
|
||||
return {
|
||||
where: clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "",
|
||||
params,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate activity (sessions, messages, active nodes/agents, daily breakdown,
|
||||
* stickiness) over a date range. Empty range yields zeroed structures and an
|
||||
* empty `daily` array — never nulls. `mttr` is the U13 unavailable seam.
|
||||
*/
|
||||
export function aggregateActivityAnalytics(
|
||||
db: Database,
|
||||
query: ActivityAnalyticsQuery = {},
|
||||
): ActivityAnalytics {
|
||||
// Sessions from cli_sessions (by createdAt).
|
||||
const sessionRange = rangeClauses("createdAt", query);
|
||||
const sessions = (
|
||||
db
|
||||
.prepare(`SELECT COUNT(*) AS count FROM cli_sessions ${sessionRange.where}`)
|
||||
.get(...sessionRange.params) as CountRow
|
||||
).count;
|
||||
|
||||
// Messages from usage_events (kind = user_message).
|
||||
const eventRange = rangeClauses("ts", query);
|
||||
const eventWhereWith = (extra: string): string =>
|
||||
eventRange.where
|
||||
? `${eventRange.where} AND ${extra}`
|
||||
: `WHERE ${extra}`;
|
||||
|
||||
const messages = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count FROM usage_events ${eventWhereWith("kind = 'user_message'")}`,
|
||||
)
|
||||
.get(...eventRange.params) as CountRow
|
||||
).count;
|
||||
|
||||
// Distinct active nodes/agents over the whole range.
|
||||
const activeNodes = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(DISTINCT nodeId) AS count FROM usage_events ${eventWhereWith("nodeId IS NOT NULL")}`,
|
||||
)
|
||||
.get(...eventRange.params) as DistinctRow
|
||||
).count;
|
||||
const activeAgents = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(DISTINCT agentId) AS count FROM usage_events ${eventWhereWith("agentId IS NOT NULL")}`,
|
||||
)
|
||||
.get(...eventRange.params) as DistinctRow
|
||||
).count;
|
||||
|
||||
// Per-day distinct nodes/agents + message count. substr(ts,1,10) is the UTC
|
||||
// day key (ISO-8601 timestamps).
|
||||
const dailyRows = db
|
||||
.prepare(
|
||||
`SELECT
|
||||
substr(ts, 1, 10) AS day,
|
||||
COUNT(DISTINCT nodeId) AS activeNodes,
|
||||
COUNT(DISTINCT agentId) AS activeAgents,
|
||||
SUM(CASE WHEN kind = 'user_message' THEN 1 ELSE 0 END) AS messages
|
||||
FROM usage_events ${eventRange.where}
|
||||
GROUP BY day
|
||||
ORDER BY day ASC`,
|
||||
)
|
||||
.all(...eventRange.params) as DayAggRow[];
|
||||
const daily: DailyActivity[] = dailyRows.map((r) => ({
|
||||
day: r.day,
|
||||
activeNodes: r.activeNodes,
|
||||
activeAgents: r.activeAgents,
|
||||
messages: r.messages ?? 0,
|
||||
}));
|
||||
|
||||
// Stickiness = DAU/MAU. DAU = mean distinct-active-agents-per-day; MAU =
|
||||
// distinct active agents over the range.
|
||||
const dau =
|
||||
daily.length > 0
|
||||
? daily.reduce((sum, d) => sum + d.activeAgents, 0) / daily.length
|
||||
: 0;
|
||||
const mau = activeAgents;
|
||||
const stickiness = mau > 0 ? dau / mau : 0;
|
||||
|
||||
return {
|
||||
from: query.from ?? null,
|
||||
to: query.to ?? null,
|
||||
sessions,
|
||||
messages,
|
||||
activeNodes,
|
||||
activeAgents,
|
||||
daily,
|
||||
stickiness,
|
||||
// U13 seam: no incident data source yet — unavailable, not 0.
|
||||
mttr: { value: null, unavailable: true },
|
||||
};
|
||||
}
|
||||
@@ -530,6 +530,35 @@ export type {
|
||||
UsageEventKind,
|
||||
UsageEventRangeQuery,
|
||||
} from "./usage-events.js";
|
||||
export { aggregateTokenAnalytics } from "./token-analytics.js";
|
||||
export type {
|
||||
TokenAnalytics,
|
||||
TokenAnalyticsQuery,
|
||||
TokenGroupBy,
|
||||
TokenGroupSummary,
|
||||
TokenTotals,
|
||||
} from "./token-analytics.js";
|
||||
export { aggregateToolAnalytics, countInterventions } from "./tool-analytics.js";
|
||||
export type {
|
||||
ToolAnalytics,
|
||||
ToolAnalyticsQuery,
|
||||
ToolCategoryCount,
|
||||
InterventionBreakdown,
|
||||
} from "./tool-analytics.js";
|
||||
export { aggregateActivityAnalytics } from "./activity-analytics.js";
|
||||
export type {
|
||||
ActivityAnalytics,
|
||||
ActivityAnalyticsQuery,
|
||||
DailyActivity,
|
||||
MttrSummary,
|
||||
} from "./activity-analytics.js";
|
||||
export { aggregateProductivityAnalytics } from "./productivity-analytics.js";
|
||||
export type {
|
||||
ProductivityAnalytics,
|
||||
ProductivityAnalyticsQuery,
|
||||
LanguageCount,
|
||||
LocSummary,
|
||||
} from "./productivity-analytics.js";
|
||||
export {
|
||||
STALLED_REVIEW_REENQUEUE_THRESHOLD,
|
||||
STALLED_REVIEW_INVALID_TRANSITION_THRESHOLD,
|
||||
@@ -674,6 +703,8 @@ export {
|
||||
isPrEntityActionable,
|
||||
isPrEntityAutoMergeReady,
|
||||
autoMergeGateReason,
|
||||
summarizePrThreadActivity,
|
||||
type PrThreadActivity,
|
||||
} from "./pr-entity.js";
|
||||
export {
|
||||
findVitestProcessIds,
|
||||
|
||||
175
packages/core/src/productivity-analytics.ts
Normal file
175
packages/core/src/productivity-analytics.ts
Normal file
@@ -0,0 +1,175 @@
|
||||
import type { Database } from "./db.js";
|
||||
|
||||
/**
|
||||
* Productivity analytics: files modified (count + language distribution) from
|
||||
* `tasks.modifiedFiles`, commit associations from `task_commit_associations`,
|
||||
* pull requests from `pull_requests`, and LOC from commit diff stats.
|
||||
*
|
||||
* **LOC availability.** Fusion does not currently persist commit diff line
|
||||
* stats (the `task_commit_associations` schema has no additions/deletions
|
||||
* columns). LOC is therefore reported as the documented unavailable sentinel —
|
||||
* `{ value: null, unavailable: true }` — **never `0`**, so a missing data source
|
||||
* is never mistaken for "zero lines changed". When a diff-stats source is added,
|
||||
* fill {@link LocSummary.value} and clear `unavailable`.
|
||||
*
|
||||
* Inclusivity: `from`/`to` bounds are inclusive. Tasks are filtered by
|
||||
* `updatedAt` (the last time the task — and therefore its modifiedFiles — was
|
||||
* touched); commit associations by `authoredAt`; PRs by `createdAt`.
|
||||
*/
|
||||
|
||||
export interface ProductivityAnalyticsQuery {
|
||||
/** ISO-8601 lower bound (inclusive). */
|
||||
from?: string;
|
||||
/** ISO-8601 upper bound (inclusive). */
|
||||
to?: string;
|
||||
}
|
||||
|
||||
/** A single language's modified-file count. */
|
||||
export interface LanguageCount {
|
||||
/** Lowercased file extension (no dot), or `other` when none. */
|
||||
language: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* LOC summary. `value` is null and `unavailable` true until a commit diff-stats
|
||||
* source exists — never `0`.
|
||||
*/
|
||||
export interface LocSummary {
|
||||
value: number | null;
|
||||
unavailable: boolean;
|
||||
}
|
||||
|
||||
export interface ProductivityAnalytics {
|
||||
from: string | null;
|
||||
to: string | null;
|
||||
/** Total modified-file paths across matched tasks. */
|
||||
modifiedFiles: number;
|
||||
/** Modified files grouped by language (extension), descending by count. */
|
||||
byLanguage: LanguageCount[];
|
||||
/** Rows in `task_commit_associations` in range. */
|
||||
commits: number;
|
||||
/** Rows in `pull_requests` in range. */
|
||||
pullRequests: number;
|
||||
/** LOC from commit diff stats — unavailable until a source exists. */
|
||||
loc: LocSummary;
|
||||
}
|
||||
|
||||
interface CountRow {
|
||||
count: number;
|
||||
}
|
||||
|
||||
interface ModifiedFilesRow {
|
||||
modifiedFiles: string | null;
|
||||
}
|
||||
|
||||
/** Extract a coarse language key from a file path (its lowercased extension). */
|
||||
function languageOf(path: string): string {
|
||||
const base = path.split("/").pop() ?? path;
|
||||
const dot = base.lastIndexOf(".");
|
||||
if (dot <= 0 || dot === base.length - 1) return "other";
|
||||
return base.slice(dot + 1).toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate productivity metrics over a date range. Empty range yields zeroed
|
||||
* structures (not nulls); LOC is always the unavailable sentinel until a
|
||||
* diff-stats source is wired.
|
||||
*/
|
||||
export function aggregateProductivityAnalytics(
|
||||
db: Database,
|
||||
query: ProductivityAnalyticsQuery = {},
|
||||
): ProductivityAnalytics {
|
||||
// Modified files: read the JSON array off tasks updated in range.
|
||||
const taskClauses: string[] = [
|
||||
"modifiedFiles IS NOT NULL",
|
||||
"modifiedFiles NOT IN ('', '[]')",
|
||||
];
|
||||
const taskParams: string[] = [];
|
||||
if (query.from !== undefined) {
|
||||
taskClauses.push("updatedAt >= ?");
|
||||
taskParams.push(query.from);
|
||||
}
|
||||
if (query.to !== undefined) {
|
||||
taskClauses.push("updatedAt <= ?");
|
||||
taskParams.push(query.to);
|
||||
}
|
||||
const taskRows = db
|
||||
.prepare(
|
||||
`SELECT modifiedFiles FROM tasks WHERE ${taskClauses.join(" AND ")}`,
|
||||
)
|
||||
.all(...taskParams) as ModifiedFilesRow[];
|
||||
|
||||
let modifiedFiles = 0;
|
||||
const langMap = new Map<string, number>();
|
||||
for (const row of taskRows) {
|
||||
if (!row.modifiedFiles) continue;
|
||||
let files: unknown;
|
||||
try {
|
||||
files = JSON.parse(row.modifiedFiles);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(files)) continue;
|
||||
for (const f of files) {
|
||||
if (typeof f !== "string" || f.length === 0) continue;
|
||||
modifiedFiles += 1;
|
||||
const lang = languageOf(f);
|
||||
langMap.set(lang, (langMap.get(lang) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
const byLanguage: LanguageCount[] = [...langMap.entries()]
|
||||
.map(([language, count]) => ({ language, count }))
|
||||
.sort((a, b) => b.count - a.count);
|
||||
|
||||
// Commits from task_commit_associations (by authoredAt).
|
||||
const commitClauses: string[] = [];
|
||||
const commitParams: string[] = [];
|
||||
if (query.from !== undefined) {
|
||||
commitClauses.push("authoredAt >= ?");
|
||||
commitParams.push(query.from);
|
||||
}
|
||||
if (query.to !== undefined) {
|
||||
commitClauses.push("authoredAt <= ?");
|
||||
commitParams.push(query.to);
|
||||
}
|
||||
const commitWhere =
|
||||
commitClauses.length > 0 ? `WHERE ${commitClauses.join(" AND ")}` : "";
|
||||
const commits = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count FROM task_commit_associations ${commitWhere}`,
|
||||
)
|
||||
.get(...commitParams) as CountRow
|
||||
).count;
|
||||
|
||||
// Pull requests. `pull_requests.createdAt` is an INTEGER epoch-ms column, so
|
||||
// convert the ISO bounds to epoch ms for comparison.
|
||||
const prClauses: string[] = [];
|
||||
const prParams: number[] = [];
|
||||
if (query.from !== undefined) {
|
||||
prClauses.push("createdAt >= ?");
|
||||
prParams.push(Date.parse(query.from));
|
||||
}
|
||||
if (query.to !== undefined) {
|
||||
prClauses.push("createdAt <= ?");
|
||||
prParams.push(Date.parse(query.to));
|
||||
}
|
||||
const prWhere = prClauses.length > 0 ? `WHERE ${prClauses.join(" AND ")}` : "";
|
||||
const pullRequests = (
|
||||
db
|
||||
.prepare(`SELECT COUNT(*) AS count FROM pull_requests ${prWhere}`)
|
||||
.get(...prParams) as CountRow
|
||||
).count;
|
||||
|
||||
return {
|
||||
from: query.from ?? null,
|
||||
to: query.to ?? null,
|
||||
modifiedFiles,
|
||||
byLanguage,
|
||||
commits,
|
||||
pullRequests,
|
||||
// No commit diff-stats source yet — unavailable, never 0.
|
||||
loc: { value: null, unavailable: true },
|
||||
};
|
||||
}
|
||||
175
packages/core/src/token-analytics.ts
Normal file
175
packages/core/src/token-analytics.ts
Normal file
@@ -0,0 +1,175 @@
|
||||
import type { Database } from "./db.js";
|
||||
|
||||
/**
|
||||
* Token-consumption analytics over the `tasks` table, generalizing the fixed
|
||||
* 24h/7d/all-time windows of `agent-token-usage.ts` to an arbitrary `(from, to)`
|
||||
* range. Sums the `tokenUsage*` columns filtered by `tokenUsageLastUsedAt` and
|
||||
* groups by model / provider / node / agent.
|
||||
*
|
||||
* Inclusivity: `from`/`to` bounds are **inclusive** (`>= from AND <= to`),
|
||||
* matching `usage-events.ts` and the range-scan house style. A task whose
|
||||
* `tokenUsageLastUsedAt` is exactly equal to `from` is therefore included.
|
||||
*
|
||||
* Pure read-only aggregation: takes a `Database` handle and returns plain data.
|
||||
*/
|
||||
|
||||
/** Dimension to group token totals by. */
|
||||
export type TokenGroupBy = "model" | "provider" | "node" | "agent";
|
||||
|
||||
/** Summed token counts for a group (or the grand total). */
|
||||
export interface TokenTotals {
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheWriteTokens: number;
|
||||
totalTokens: number;
|
||||
/** Number of tasks that contributed to these totals. */
|
||||
nTasks: number;
|
||||
}
|
||||
|
||||
/** One group's token totals, keyed by the grouped dimension value. */
|
||||
export interface TokenGroupSummary extends TokenTotals {
|
||||
/** The group key (model id, provider, nodeId, or agentId); null when unset. */
|
||||
key: string | null;
|
||||
}
|
||||
|
||||
/** Result of {@link aggregateTokenAnalytics}. */
|
||||
export interface TokenAnalytics {
|
||||
from: string | null;
|
||||
to: string | null;
|
||||
groupBy: TokenGroupBy | null;
|
||||
/** Grand total across all matched tasks. */
|
||||
totals: TokenTotals;
|
||||
/** Per-group totals; empty array when no `groupBy` requested. */
|
||||
groups: TokenGroupSummary[];
|
||||
}
|
||||
|
||||
export interface TokenAnalyticsQuery {
|
||||
/** ISO-8601 lower bound (inclusive) on `tokenUsageLastUsedAt`. */
|
||||
from?: string;
|
||||
/** ISO-8601 upper bound (inclusive) on `tokenUsageLastUsedAt`. */
|
||||
to?: string;
|
||||
groupBy?: TokenGroupBy;
|
||||
}
|
||||
|
||||
function emptyTotals(): TokenTotals {
|
||||
return {
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
cachedTokens: 0,
|
||||
cacheWriteTokens: 0,
|
||||
totalTokens: 0,
|
||||
nTasks: 0,
|
||||
};
|
||||
}
|
||||
|
||||
interface TaskTokenRow {
|
||||
inputTokens: number | null;
|
||||
outputTokens: number | null;
|
||||
cachedTokens: number | null;
|
||||
cacheWriteTokens: number | null;
|
||||
totalTokens: number | null;
|
||||
modelProvider: string | null;
|
||||
modelId: string | null;
|
||||
checkoutNodeId: string | null;
|
||||
assignedAgentId: string | null;
|
||||
}
|
||||
|
||||
function groupKeyFor(row: TaskTokenRow, groupBy: TokenGroupBy): string | null {
|
||||
switch (groupBy) {
|
||||
case "model":
|
||||
return row.modelId;
|
||||
case "provider":
|
||||
return row.modelProvider;
|
||||
case "node":
|
||||
return row.checkoutNodeId;
|
||||
case "agent":
|
||||
return row.assignedAgentId;
|
||||
}
|
||||
}
|
||||
|
||||
function addRow(totals: TokenTotals, row: TaskTokenRow): void {
|
||||
totals.inputTokens += row.inputTokens ?? 0;
|
||||
totals.outputTokens += row.outputTokens ?? 0;
|
||||
totals.cachedTokens += row.cachedTokens ?? 0;
|
||||
totals.cacheWriteTokens += row.cacheWriteTokens ?? 0;
|
||||
// Prefer the persisted total when present; otherwise derive it from the parts
|
||||
// so callers always get a coherent `totalTokens` even on older rows.
|
||||
const persistedTotal = row.totalTokens;
|
||||
totals.totalTokens +=
|
||||
persistedTotal ??
|
||||
(row.inputTokens ?? 0) +
|
||||
(row.outputTokens ?? 0) +
|
||||
(row.cachedTokens ?? 0) +
|
||||
(row.cacheWriteTokens ?? 0);
|
||||
totals.nTasks += 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate per-task token usage over a date range, optionally grouped.
|
||||
*
|
||||
* Tasks are matched by `tokenUsageLastUsedAt` within `[from, to]` (inclusive).
|
||||
* Tasks with no token usage (`tokenUsageLastUsedAt IS NULL`) are excluded. An
|
||||
* empty range yields zeroed `totals` and an empty `groups` array — never nulls.
|
||||
*/
|
||||
export function aggregateTokenAnalytics(
|
||||
db: Database,
|
||||
query: TokenAnalyticsQuery = {},
|
||||
): TokenAnalytics {
|
||||
const clauses: string[] = ["tokenUsageLastUsedAt IS NOT NULL"];
|
||||
const params: string[] = [];
|
||||
if (query.from !== undefined) {
|
||||
clauses.push("tokenUsageLastUsedAt >= ?");
|
||||
params.push(query.from);
|
||||
}
|
||||
if (query.to !== undefined) {
|
||||
clauses.push("tokenUsageLastUsedAt <= ?");
|
||||
params.push(query.to);
|
||||
}
|
||||
const where = `WHERE ${clauses.join(" AND ")}`;
|
||||
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT
|
||||
tokenUsageInputTokens AS inputTokens,
|
||||
tokenUsageOutputTokens AS outputTokens,
|
||||
tokenUsageCachedTokens AS cachedTokens,
|
||||
tokenUsageCacheWriteTokens AS cacheWriteTokens,
|
||||
tokenUsageTotalTokens AS totalTokens,
|
||||
modelProvider,
|
||||
modelId,
|
||||
checkoutNodeId,
|
||||
assignedAgentId
|
||||
FROM tasks ${where}`,
|
||||
)
|
||||
.all(...params) as TaskTokenRow[];
|
||||
|
||||
const totals = emptyTotals();
|
||||
const groupMap = new Map<string | null, TokenGroupSummary>();
|
||||
const groupBy = query.groupBy;
|
||||
|
||||
for (const row of rows) {
|
||||
addRow(totals, row);
|
||||
if (groupBy) {
|
||||
const key = groupKeyFor(row, groupBy);
|
||||
let group = groupMap.get(key);
|
||||
if (!group) {
|
||||
group = { key, ...emptyTotals() };
|
||||
groupMap.set(key, group);
|
||||
}
|
||||
addRow(group, row);
|
||||
}
|
||||
}
|
||||
|
||||
const groups = [...groupMap.values()].sort(
|
||||
(a, b) => b.totalTokens - a.totalTokens,
|
||||
);
|
||||
|
||||
return {
|
||||
from: query.from ?? null,
|
||||
to: query.to ?? null,
|
||||
groupBy: groupBy ?? null,
|
||||
totals,
|
||||
groups,
|
||||
};
|
||||
}
|
||||
221
packages/core/src/tool-analytics.ts
Normal file
221
packages/core/src/tool-analytics.ts
Normal file
@@ -0,0 +1,221 @@
|
||||
import type { Database } from "./db.js";
|
||||
import type { SteeringComment } from "./types.js";
|
||||
|
||||
/**
|
||||
* Tool-usage analytics over `usage_events`, plus the **autonomy ratio**.
|
||||
*
|
||||
* Autonomy ratio = tool_call count / human-intervention count. The denominator
|
||||
* is NOT raw user messages (which trend to zero for autonomous execution); it is
|
||||
* the count of human interventions, which has **three distinct sources** — they
|
||||
* are not one queryable table:
|
||||
*
|
||||
* 1. **Approvals** — rows in `approval_request_audit_events` whose `eventType`
|
||||
* is `created` or `approved` (a human was asked to / did approve an action),
|
||||
* timestamped by `createdAt`.
|
||||
* 2. **User-authored steers** — entries in the `steeringComments` JSON column
|
||||
* on the `tasks` row, filtered to `author === "user"` (agent-authored steers
|
||||
* are excluded), timestamped by each comment's `createdAt`.
|
||||
* 3. **Waiting-on-input** — a task *status*, not a counted event; intentionally
|
||||
* DROPPED here (no concrete answer event is defined).
|
||||
*
|
||||
* A fully-autonomous session (zero interventions) must not divide by zero or
|
||||
* report ∞: when `interventions === 0` the ratio falls back to
|
||||
* tool-calls-per-session (`toolCalls / max(sessions, 1)`), and the result flags
|
||||
* `interventions: 0` so callers can render it as "fully autonomous".
|
||||
*
|
||||
* Inclusivity: `from`/`to` bounds are inclusive, matching `usage-events.ts`.
|
||||
*/
|
||||
|
||||
export interface ToolAnalyticsQuery {
|
||||
/** ISO-8601 lower bound (inclusive). */
|
||||
from?: string;
|
||||
/** ISO-8601 upper bound (inclusive). */
|
||||
to?: string;
|
||||
}
|
||||
|
||||
/** Tool-call count for a single coarse category. */
|
||||
export interface ToolCategoryCount {
|
||||
category: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
/** Breakdown of the autonomy-ratio denominator by source. */
|
||||
export interface InterventionBreakdown {
|
||||
/** `created`/`approved` rows in `approval_request_audit_events`. */
|
||||
approvals: number;
|
||||
/** `steeringComments` entries with `author === "user"`. */
|
||||
userSteers: number;
|
||||
/** Total human interventions (sum of the components above). */
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface ToolAnalytics {
|
||||
from: string | null;
|
||||
to: string | null;
|
||||
/** Total `tool_call` events in range. */
|
||||
toolCalls: number;
|
||||
/** Tool calls grouped by `category`, descending by count. */
|
||||
byCategory: ToolCategoryCount[];
|
||||
/** Distinct sessions (`session_start` events) in range. */
|
||||
sessions: number;
|
||||
interventions: InterventionBreakdown;
|
||||
/**
|
||||
* Autonomy ratio. When `interventions.total > 0` this is
|
||||
* `toolCalls / interventions.total`. When there are zero interventions it is
|
||||
* tool-calls-per-session (`toolCalls / max(sessions, 1)`) and
|
||||
* `fullyAutonomous` is true — never ∞ or NaN.
|
||||
*/
|
||||
autonomyRatio: number;
|
||||
/** True when zero human interventions were recorded in range. */
|
||||
fullyAutonomous: boolean;
|
||||
}
|
||||
|
||||
interface CountRow {
|
||||
count: number;
|
||||
}
|
||||
|
||||
interface CategoryRow {
|
||||
category: string | null;
|
||||
count: number;
|
||||
}
|
||||
|
||||
interface SteeringRow {
|
||||
steeringComments: string | null;
|
||||
}
|
||||
|
||||
function inRange(ts: string, from?: string, to?: string): boolean {
|
||||
if (from !== undefined && ts < from) return false;
|
||||
if (to !== undefined && ts > to) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count human interventions from the three named sources (waiting-on-input is a
|
||||
* status, not counted). Returns the per-source breakdown plus the total.
|
||||
*/
|
||||
export function countInterventions(
|
||||
db: Database,
|
||||
query: ToolAnalyticsQuery = {},
|
||||
): InterventionBreakdown {
|
||||
// Source 1: approvals. `approval_request_audit_events.createdAt` is the ts;
|
||||
// count only the human-touch event types.
|
||||
const approvalClauses: string[] = ["eventType IN ('created', 'approved')"];
|
||||
const approvalParams: string[] = [];
|
||||
if (query.from !== undefined) {
|
||||
approvalClauses.push("createdAt >= ?");
|
||||
approvalParams.push(query.from);
|
||||
}
|
||||
if (query.to !== undefined) {
|
||||
approvalClauses.push("createdAt <= ?");
|
||||
approvalParams.push(query.to);
|
||||
}
|
||||
const approvals = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count FROM approval_request_audit_events WHERE ${approvalClauses.join(" AND ")}`,
|
||||
)
|
||||
.get(...approvalParams) as CountRow
|
||||
).count;
|
||||
|
||||
// Source 2: user-authored steers from the `steeringComments` JSON on tasks.
|
||||
// This re-introduces a per-task JSON read (documented in U2). Only rows with a
|
||||
// non-empty JSON array are scanned.
|
||||
const steeringRows = db
|
||||
.prepare(
|
||||
`SELECT steeringComments FROM tasks
|
||||
WHERE steeringComments IS NOT NULL AND steeringComments NOT IN ('', '[]')`,
|
||||
)
|
||||
.all() as SteeringRow[];
|
||||
let userSteers = 0;
|
||||
for (const row of steeringRows) {
|
||||
if (!row.steeringComments) continue;
|
||||
let parsed: SteeringComment[];
|
||||
try {
|
||||
parsed = JSON.parse(row.steeringComments) as SteeringComment[];
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(parsed)) continue;
|
||||
for (const comment of parsed) {
|
||||
if (comment?.author !== "user") continue;
|
||||
if (!inRange(comment.createdAt ?? "", query.from, query.to)) continue;
|
||||
userSteers += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return { approvals, userSteers, total: approvals + userSteers };
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate tool usage and the autonomy ratio over a date range.
|
||||
*
|
||||
* Empty range yields zeroed structures (not nulls) and `autonomyRatio: 0`.
|
||||
*/
|
||||
export function aggregateToolAnalytics(
|
||||
db: Database,
|
||||
query: ToolAnalyticsQuery = {},
|
||||
): ToolAnalytics {
|
||||
const eventClauses: string[] = [];
|
||||
const eventParams: string[] = [];
|
||||
if (query.from !== undefined) {
|
||||
eventClauses.push("ts >= ?");
|
||||
eventParams.push(query.from);
|
||||
}
|
||||
if (query.to !== undefined) {
|
||||
eventClauses.push("ts <= ?");
|
||||
eventParams.push(query.to);
|
||||
}
|
||||
const rangeWhere = eventClauses.length > 0 ? `AND ${eventClauses.join(" AND ")}` : "";
|
||||
|
||||
const toolCalls = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count FROM usage_events WHERE kind = 'tool_call' ${rangeWhere}`,
|
||||
)
|
||||
.get(...eventParams) as CountRow
|
||||
).count;
|
||||
|
||||
const categoryRows = db
|
||||
.prepare(
|
||||
`SELECT category AS category, COUNT(*) AS count
|
||||
FROM usage_events
|
||||
WHERE kind = 'tool_call' ${rangeWhere}
|
||||
GROUP BY category`,
|
||||
)
|
||||
.all(...eventParams) as CategoryRow[];
|
||||
const byCategory: ToolCategoryCount[] = categoryRows
|
||||
.map((r) => ({ category: r.category ?? "other", count: r.count }))
|
||||
.sort((a, b) => b.count - a.count);
|
||||
|
||||
const sessions = (
|
||||
db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS count FROM usage_events WHERE kind = 'session_start' ${rangeWhere}`,
|
||||
)
|
||||
.get(...eventParams) as CountRow
|
||||
).count;
|
||||
|
||||
const interventions = countInterventions(db, query);
|
||||
|
||||
let autonomyRatio: number;
|
||||
let fullyAutonomous: boolean;
|
||||
if (interventions.total > 0) {
|
||||
autonomyRatio = toolCalls / interventions.total;
|
||||
fullyAutonomous = false;
|
||||
} else {
|
||||
// Zero interventions: report tool-calls-per-session, never ∞ / divide-by-zero.
|
||||
autonomyRatio = toolCalls / Math.max(sessions, 1);
|
||||
fullyAutonomous = true;
|
||||
}
|
||||
|
||||
return {
|
||||
from: query.from ?? null,
|
||||
to: query.to ?? null,
|
||||
toolCalls,
|
||||
byCategory,
|
||||
sessions,
|
||||
interventions,
|
||||
autonomyRatio,
|
||||
fullyAutonomous,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user