diff --git a/.changeset/fn-7393-planner-metrics.md b/.changeset/fn-7393-planner-metrics.md new file mode 100644 index 0000000000..89142221a0 --- /dev/null +++ b/.changeset/fn-7393-planner-metrics.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": minor +--- + +summary: Let task-detail planner Chat answer current-task token, cost, and timing questions. +category: feature +dev: Adds read-only task-scoped planner chat tool `fn_task_planner_get_task_metrics` with derived pricing semantics. diff --git a/docs/dashboard-guide.md b/docs/dashboard-guide.md index 4eb428359e..df7a530a2f 100644 --- a/docs/dashboard-guide.md +++ b/docs/dashboard-guide.md @@ -406,7 +406,7 @@ Chat view provides project-scoped conversations with agents. - On mobile direct-chat threads, tapping the active title/identity in the thread header opens a lightweight conversation dropdown so you can switch to another direct session or start a New Chat without backing out to the sidebar list first; long conversation titles now stay readable in the dropdown via wrapped option text and taller touch-friendly rows. - Direct chat sessions can be renamed from the desktop conversation context menu and from the mobile session switcher; blank rename submissions clear the custom title so the default session label is shown again. -- Task-detail planner Chat conversations stay available from each task's **Chat** tab. They are hidden from the common Direct/common Chat feed by default; enable **Settings → Project General → Show task chats in common Chat feed** to include populated task chats again. Empty task chat sessions stay hidden either way. +- Task-detail planner Chat conversations stay available from each task's **Chat** tab. They are hidden from the common Direct/common Chat feed by default; enable **Settings → Project General → Show task chats in common Chat feed** to include populated task chats again. Empty task chat sessions stay hidden either way. Planner Chat can answer token-count, estimated-cost, runtime, timing-event, workflow-step duration, and per-model usage questions for the current task through a read-only task-scoped metrics tool; unknown/stale pricing is reported as uncertain instead of `$0`. - On desktop/tablet Direct chat, the thread header shows an estimated token count against the active model's known context window (for example `~12.3k / 200k`). It is hidden on mobile, narrow floating chat, rooms, and unknown-context-window models. diff --git a/packages/dashboard/src/__tests__/chat-manager.test.ts b/packages/dashboard/src/__tests__/chat-manager.test.ts index 039ebb2e63..97f59b9f6b 100644 --- a/packages/dashboard/src/__tests__/chat-manager.test.ts +++ b/packages/dashboard/src/__tests__/chat-manager.test.ts @@ -1359,10 +1359,15 @@ describe("ChatManager.sendMessage", () => { expect(createOptions.systemPrompt).toContain("Activity transcript loaded"); expect(createOptions.systemPrompt).toContain("fn_ask_question"); expect(createOptions.systemPrompt).toContain("Do not create steering for ordinary questions"); + expect(createOptions.systemPrompt).toContain("fn_task_planner_get_task_metrics"); + expect(createOptions.systemPrompt).toContain("token counts, input/output/cache usage, model cost"); + expect(createOptions.systemPrompt).toContain("state that uncertainty instead of inventing a number"); + expect(createOptions.systemPrompt).toContain("ordinary status/progress/metrics questions"); expect(createOptions.systemPrompt).toContain("Ask a clarifying question"); expect(createOptions.systemPrompt).toContain("credential/secrets"); expect(createOptions.systemPrompt).toContain("destructive removals"); expect(createOptions.customTools.map((tool: { name: string }) => tool.name)).toContain("fn_task_planner_add_steering"); + expect(createOptions.customTools.map((tool: { name: string }) => tool.name)).toContain("fn_task_planner_get_task_metrics"); expect(mockChatStore.addMessage).toHaveBeenCalledWith("chat-001", expect.objectContaining({ role: "user", content: "How should I plan this?", @@ -1375,6 +1380,122 @@ describe("ChatManager.sendMessage", () => { expect(taskStore.getTask).toHaveBeenCalledWith("FN-7309"); }); + it("exposes read-only metrics through the task-scoped planner tool", async () => { + mockChatStore.getSession.mockReturnValue({ + id: "chat-001", + agentId: "task-planner:FN-7310", + status: "active", + }); + + const createResolvedSession = vi.fn(async () => ({ + session: { prompt: vi.fn().mockResolvedValue(undefined), dispose: vi.fn(), state: { messages: [] } }, + })); + __setCreateResolvedAgentSession(createResolvedSession as any); + + const taskStore = { + getTask: vi.fn().mockResolvedValue({ + id: "FN-7310", + title: "Metric task", + column: "done", + status: "complete", + tokenUsage: { + inputTokens: 1000, + outputTokens: 200, + cachedTokens: 50, + cacheWriteTokens: 10, + totalTokens: 1260, + firstUsedAt: "2026-07-01T10:00:00.000Z", + lastUsedAt: "2026-07-01T10:10:00.000Z", + modelProvider: "test-provider", + modelId: "model-a", + }, + executionStartedAt: "2026-07-01T10:00:00.000Z", + executionCompletedAt: "2026-07-01T10:02:00.000Z", + log: [{ timestamp: "2026-07-01T10:01:00.000Z", action: "[timing] setup completed in 500ms" }], + workflowStepResults: [], + }), + addSteeringComment: vi.fn(), + getSettings: vi.fn().mockResolvedValue({}), + }; + const getSettings = vi.fn(async () => ({ + modelPricingOverrides: { + "test-provider:model-a": { inputPer1M: 1, outputPer1M: 2, cacheReadPer1M: 0.5, cacheWritePer1M: 1.5, source: "test" }, + }, + })); + const chatManager = new ChatManager(mockChatStore as any, "/tmp/test", mockAgentStore as any, undefined, getSettings as any, undefined, taskStore as any); + + await chatManager.sendMessage("chat-001", "How much did this task cost?"); + + const createOptions = createResolvedSession.mock.calls[0]?.[0]; + const metricsTool = createOptions.customTools.find((tool: { name: string }) => tool.name === "fn_task_planner_get_task_metrics"); + expect(metricsTool.parameters).toEqual({ type: "object", properties: {}, additionalProperties: false }); + + taskStore.getTask.mockClear(); + const result = await metricsTool.execute("call-1", { task_id: "FN-OTHER" }); + + expect(taskStore.getTask).toHaveBeenCalledTimes(1); + expect(taskStore.getTask).toHaveBeenCalledWith("FN-7310", { activityLogLimit: 100 }); + expect(taskStore.addSteeringComment).not.toHaveBeenCalled(); + expect(result.isError).toBeUndefined(); + expect(result.content[0].text).toContain("Task FN-7310 metrics"); + expect(result.details).toMatchObject({ + taskId: "FN-7310", + tokens: { + totalTokens: 1260, + cost: { costUnavailable: false, pricingStale: false }, + perModel: [expect.objectContaining({ key: "test-provider:model-a", totalTokens: 1260 })], + }, + timing: { + endToEndExecutionMs: 120_000, + logTimingDurationMs: 500, + timingEventCount: 1, + }, + }); + expect(result.details.tokens.cost.usd).toBeCloseTo(0.00144); + expect(result.details.tokens.perModel[0].cost.usd).toBeCloseTo(0.00144); + }); + + it("returns a safe scoped error when planner metrics cannot load the current task", async () => { + mockChatStore.getSession.mockReturnValue({ id: "chat-001", agentId: "task-planner:FN-MISSING", status: "active" }); + const createResolvedSession = vi.fn(async () => ({ + session: { prompt: vi.fn().mockResolvedValue(undefined), dispose: vi.fn(), state: { messages: [] } }, + })); + __setCreateResolvedAgentSession(createResolvedSession as any); + const taskStore = { + getTask: vi.fn().mockRejectedValue(new Error("not found")), + addSteeringComment: vi.fn(), + getSettings: vi.fn().mockResolvedValue({}), + }; + const chatManager = new ChatManager(mockChatStore as any, "/tmp/test", mockAgentStore as any, undefined, undefined, undefined, taskStore as any); + + await chatManager.sendMessage("chat-001", "How many tokens?"); + + const createOptions = createResolvedSession.mock.calls[0]?.[0]; + const metricsTool = createOptions.customTools.find((tool: { name: string }) => tool.name === "fn_task_planner_get_task_metrics"); + const result = await metricsTool.execute("call-1", {}); + + expect(result.isError).toBe(true); + expect(result.details).toEqual({ taskId: "FN-MISSING", error: "not found" }); + expect(result.content[0].text).toContain("Could not load metrics for the current task FN-MISSING"); + expect(taskStore.addSteeringComment).not.toHaveBeenCalled(); + }); + + it("does not expose the current-task metrics tool outside synthetic task planner chat", async () => { + mockChatStore.getSession.mockReturnValue({ id: "chat-001", agentId: "agent-001", status: "active" }); + const createResolvedSession = vi.fn(async () => ({ + session: { prompt: vi.fn().mockResolvedValue(undefined), dispose: vi.fn(), state: { messages: [] } }, + })); + __setCreateResolvedAgentSession(createResolvedSession as any); + const taskStore = { getTask: vi.fn(), getSettings: vi.fn().mockResolvedValue({}) }; + const chatManager = new ChatManager(mockChatStore as any, "/tmp/test", mockAgentStore as any, undefined, undefined, undefined, taskStore as any); + + await chatManager.sendMessage("chat-001", "How many tokens did FN-7310 use?"); + + const createOptions = createResolvedSession.mock.calls[0]?.[0]; + const toolNames = (createOptions.customTools ?? []).map((tool: { name: string }) => tool.name); + expect(toolNames).not.toContain("fn_task_planner_get_task_metrics"); + }); + it("adds steering through the task-scoped planner tool without accepting a caller task id", async () => { mockChatStore.getSession.mockReturnValue({ id: "chat-001", @@ -2847,6 +2968,38 @@ describe("ChatManager generation isolation", () => { } }); + it("sendRoomMessage does not expose the task-planner metrics tool to room responders", async () => { + (mockChatStore as any).getRoom = vi.fn().mockReturnValue({ id: "room-1", name: "team", projectId: "project-1" }); + (mockChatStore as any).listRoomMembers = vi.fn().mockReturnValue([ + { roomId: "room-1", agentId: "agent-001", role: "member", addedAt: "2026-01-01" }, + ]); + (mockChatStore as any).addRoomMessage = vi.fn().mockImplementation((_roomId: string, input: any) => ({ + id: input.role === "user" ? "user-room-msg" : "assistant-room-msg", + roomId: "room-1", + ...input, + })); + mockAgentStore.listAgents.mockResolvedValue([{ id: "agent-001", name: "Avery", role: "executor", state: "idle" }]); + mockAgentStore.getAgent.mockResolvedValue({ id: "agent-001", name: "Avery", role: "executor", state: "idle" }); + + let capturedTools: Array<{ name: string }> = []; + __setCreateResolvedAgentSession(async (options: any) => { + capturedTools = options.customTools ?? []; + return { + session: { + prompt: vi.fn().mockResolvedValue(undefined), + dispose: vi.fn(), + state: { messages: [{ role: "assistant", content: "Room answer" }] }, + }, + }; + }); + + const taskStore = { getTask: vi.fn(), getSettings: vi.fn().mockResolvedValue({}) }; + const chatManager = new ChatManager(mockChatStore as any, "/tmp/test", mockAgentStore as any, undefined, undefined, undefined, taskStore as any); + await chatManager.sendRoomMessage("room-1", "How many tokens did FN-7310 use?"); + + expect(capturedTools.map((tool) => tool.name)).not.toContain("fn_task_planner_get_task_metrics"); + }); + it("sendRoomMessage persists assistant room replies", async () => { (mockChatStore as any).getRoom = vi.fn().mockReturnValue({ id: "room-1", name: "team" }); (mockChatStore as any).listRoomMembers = vi.fn().mockReturnValue([ diff --git a/packages/dashboard/src/__tests__/task-planner-chat-metrics.test.ts b/packages/dashboard/src/__tests__/task-planner-chat-metrics.test.ts new file mode 100644 index 0000000000..d5790fd2e6 --- /dev/null +++ b/packages/dashboard/src/__tests__/task-planner-chat-metrics.test.ts @@ -0,0 +1,243 @@ +import { describe, expect, it } from "vitest"; +import { formatTaskPlannerChatMetrics } from "../task-planner-chat-metrics.js"; +import type { Task } from "@fusion/core"; + +function makeTask(overrides: Partial = {}): Task { + return { + id: "FN-METRICS", + description: "Metrics task", + column: "done", + dependencies: [], + steps: [], + currentStep: 0, + log: [], + createdAt: "2026-07-01T00:00:00.000Z", + updatedAt: "2026-07-01T00:00:00.000Z", + ...overrides, + } as Task; +} + +describe("formatTaskPlannerChatMetrics", () => { + it("derives token totals, merged per-model costs, and task timing from durable fields", () => { + const result = formatTaskPlannerChatMetrics(makeTask({ + id: "FN-100", + title: "Costed task", + column: "done", + status: "complete", + tokenUsage: { + inputTokens: 3000, + outputTokens: 750, + cachedTokens: 120, + cacheWriteTokens: 30, + totalTokens: 3900, + firstUsedAt: "2026-07-01T10:00:00.000Z", + lastUsedAt: "2026-07-01T10:30:00.000Z", + perModel: [ + { + modelProvider: "test-provider", + modelId: "model-a", + inputTokens: 1000, + outputTokens: 200, + cachedTokens: 50, + cacheWriteTokens: 10, + totalTokens: 1260, + firstUsedAt: "2026-07-01T10:00:00.000Z", + lastUsedAt: "2026-07-01T10:10:00.000Z", + }, + { + modelProvider: "test-provider", + modelId: "model-a", + inputTokens: 500, + outputTokens: 100, + cachedTokens: 20, + cacheWriteTokens: 5, + totalTokens: 625, + firstUsedAt: "2026-07-01T10:05:00.000Z", + lastUsedAt: "2026-07-01T10:20:00.000Z", + }, + { + modelProvider: "test-provider", + modelId: "model-b", + inputTokens: 1500, + outputTokens: 450, + cachedTokens: 50, + cacheWriteTokens: 15, + totalTokens: 2015, + firstUsedAt: "2026-07-01T10:12:00.000Z", + lastUsedAt: "2026-07-01T10:30:00.000Z", + }, + ], + }, + executionStartedAt: "2026-07-01T10:00:00.000Z", + executionCompletedAt: "2026-07-01T10:05:00.000Z", + firstExecutionAt: "2026-07-01T09:50:00.000Z", + cumulativeActiveMs: 240_000, + timedExecutionMs: 120_000, + log: [ + { timestamp: "2026-07-01T10:01:00.000Z", action: "[timing] setup completed in 500ms" }, + { timestamp: "2026-07-01T10:02:00.000Z", action: "non timing event" }, + { timestamp: "2026-07-01T10:03:00.000Z", action: "tool call", outcome: "[timing] verify completed after 1500ms" }, + ], + workflowStepResults: [ + { + workflowStepId: "plan-review", + workflowStepName: "Plan Review", + status: "passed", + startedAt: "2026-07-01T10:03:00.000Z", + completedAt: "2026-07-01T10:04:10.000Z", + }, + { + workflowStepId: "code-review", + workflowStepName: "Code Review", + status: "passed", + startedAt: "2026-07-01T10:04:00.000Z", + completedAt: "2026-07-01T10:04:30.000Z", + }, + ], + }), { + nowMs: Date.parse("2026-07-01T10:06:00.000Z"), + pricingOverrides: { + "test-provider:model-a": { inputPer1M: 1, outputPer1M: 2, cacheReadPer1M: 0.5, cacheWritePer1M: 1.5, source: "test" }, + "test-provider:model-b": { inputPer1M: 3, outputPer1M: 4, cacheReadPer1M: 1, cacheWritePer1M: 2, source: "test" }, + }, + }); + + expect(result.metrics.tokens).toMatchObject({ + available: true, + inputTokens: 3000, + outputTokens: 750, + cachedTokens: 120, + cacheWriteTokens: 30, + totalTokens: 3900, + firstUsedAt: "2026-07-01T10:00:00.000Z", + lastUsedAt: "2026-07-01T10:30:00.000Z", + cost: { costUnavailable: false, pricingStale: false }, + }); + expect(result.metrics.tokens.perModel).toHaveLength(2); + expect(result.metrics.tokens.perModel[0]).toMatchObject({ + key: "test-provider:model-a", + inputTokens: 1500, + outputTokens: 300, + cachedTokens: 70, + cacheWriteTokens: 15, + totalTokens: 1885, + firstUsedAt: "2026-07-01T10:00:00.000Z", + lastUsedAt: "2026-07-01T10:20:00.000Z", + }); + expect(result.metrics.tokens.perModel[0].cost.usd).toBeCloseTo(0.0021575); + expect(result.metrics.tokens.perModel[1].cost.usd).toBeCloseTo(0.00638); + expect(result.metrics.tokens.cost.usd).toBeCloseTo(0.0085375); + + expect(result.metrics.timing).toMatchObject({ + executionStartedAt: "2026-07-01T10:00:00.000Z", + executionCompletedAt: "2026-07-01T10:05:00.000Z", + firstExecutionAt: "2026-07-01T09:50:00.000Z", + endToEndExecutionMs: 300_000, + wallClockSinceFirstExecutionMs: 900_000, + activeRuntimeMs: 240_000, + cumulativeActiveMs: 240_000, + timedExecutionMs: 120_000, + logTimingDurationMs: 2_000, + timingEventCount: 2, + timedTimingEventCount: 2, + workflowRuntimeMs: 100_000, + timedWorkflowStepCount: 2, + totalExecutionMs: 240_000, + }); + expect(result.metrics.timing.longestTimingEvent).toMatchObject({ summary: "verify completed", durationMs: 1500 }); + expect(result.metrics.timing.longestWorkflowStep).toMatchObject({ workflowStepName: "Plan Review", durationMs: 70_000 }); + expect(result.summaryText).toContain("3,900 total tokens"); + expect(result.summaryText).toContain("estimated cost $0.0085"); + }); + + it("marks unpriced and stale model costs unavailable instead of reporting zero", () => { + const result = formatTaskPlannerChatMetrics(makeTask({ + tokenUsage: { + inputTokens: 100, + outputTokens: 200, + cachedTokens: 0, + cacheWriteTokens: 0, + totalTokens: 300, + firstUsedAt: "not-a-date", + lastUsedAt: "2026-07-01T10:00:00.000Z", + modelProvider: "unknown", + modelId: "unpriced-model", + }, + }), { nowMs: Date.parse("2027-07-01T00:00:00.000Z") }); + + expect(result.metrics.tokens.cost).toEqual({ usd: null, costUnavailable: true, pricingStale: true }); + expect(result.metrics.tokens.perModel[0].cost).toEqual({ usd: null, costUnavailable: true, pricingStale: true }); + expect(result.metrics.tokens.malformedTimestamps).toEqual(["not-a-date"]); + expect(result.summaryText).toContain("cost unavailable"); + expect(result.summaryText).toContain("pricing is stale"); + }); + + it("returns deterministic empty metrics when token and timing data are absent", () => { + const result = formatTaskPlannerChatMetrics(makeTask({ id: "FN-EMPTY", column: "archived", status: "done" }), { + nowMs: Date.parse("2026-07-01T12:00:00.000Z"), + }); + + expect(result.metrics.tokens).toMatchObject({ + available: false, + inputTokens: 0, + outputTokens: 0, + cachedTokens: 0, + cacheWriteTokens: 0, + totalTokens: 0, + firstUsedAt: null, + lastUsedAt: null, + perModel: [], + cost: { usd: null, costUnavailable: false, pricingStale: false }, + }); + expect(result.metrics.timing).toMatchObject({ + endToEndExecutionMs: null, + wallClockSinceFirstExecutionMs: null, + activeRuntimeMs: null, + timedExecutionMs: null, + logTimingDurationMs: null, + timingEventCount: 0, + timedWorkflowStepCount: 0, + workflowRuntimeMs: null, + totalExecutionMs: null, + longestTimingEvent: null, + longestWorkflowStep: null, + }); + }); + + it("uses now for running tasks and malformed workflow timestamps stay bounded", () => { + const result = formatTaskPlannerChatMetrics(makeTask({ + column: "in-progress", + executionStartedAt: "2026-07-01T10:00:00.000Z", + firstExecutionAt: "bad-first", + cumulativeActiveMs: 60_000, + log: [{ timestamp: "2026-07-01T10:01:00.000Z", action: "[timing] pending marker without duration" }], + workflowStepResults: [ + { + workflowStepId: "running-step", + workflowStepName: "Running Step", + status: "pending", + startedAt: "2026-07-01T10:02:00.000Z", + }, + { + workflowStepId: "bad-step", + workflowStepName: "Bad Step", + status: "failed", + startedAt: "not-a-date", + completedAt: "2026-07-01T10:03:00.000Z", + }, + ], + }), { nowMs: Date.parse("2026-07-01T10:05:00.000Z") }); + + expect(result.metrics.timing.endToEndExecutionMs).toBe(300_000); + expect(result.metrics.timing.activeRuntimeMs).toBe(360_000); + expect(result.metrics.timing.wallClockSinceFirstExecutionMs).toBeNull(); + expect(result.metrics.timing.timingEventCount).toBe(1); + expect(result.metrics.timing.timedTimingEventCount).toBe(0); + expect(result.metrics.timing.logTimingDurationMs).toBeNull(); + expect(result.metrics.timing.workflowRuntimeMs).toBe(180_000); + expect(result.metrics.timing.workflowSteps[0]).toMatchObject({ running: true, durationMs: 180_000 }); + expect(result.metrics.timing.workflowSteps[1]).toMatchObject({ running: false, durationMs: null }); + expect(result.metrics.timing.malformedTimestamps).toContain("bad-first"); + expect(result.metrics.timing.malformedTimestamps).toContain("not-a-date"); + }); +}); diff --git a/packages/dashboard/src/chat.ts b/packages/dashboard/src/chat.ts index 9d26a572ac..57c92a9bb5 100644 --- a/packages/dashboard/src/chat.ts +++ b/packages/dashboard/src/chat.ts @@ -35,6 +35,7 @@ import { SessionManager } from "@earendil-works/pi-coding-agent"; import { SessionEventBuffer } from "./sse-buffer.js"; import { formatChatAttachmentContents, readChatAttachmentContents } from "./chat-attachment-content.js"; import { buildTaskPlannerChatContext, TASK_PLANNER_CHAT_CONTEXT_PROMPT_GUIDANCE } from "./task-planner-chat-context.js"; +import { formatTaskPlannerChatMetrics } from "./task-planner-chat-metrics.js"; import { emitWorkflowSseEvent, type WorkflowSseEventType } from "./sse.js"; import { @@ -279,6 +280,39 @@ function createChatWorkflowAuthoringTools(taskStore: TaskStore | undefined, proj .map((tool) => wrapWorkflowMutationTool(tool, projectId)); } +function createTaskPlannerMetricsTool(taskStore: TaskStore, taskId: string, getPricingOverrides: () => Promise) { + return { + name: "fn_task_planner_get_task_metrics", + label: "Get Current Task Metrics", + description: "Read token usage, derived model cost, and execution timing metrics for the current task. The task id is fixed by server context; this tool never accepts or reveals metrics for another task.", + parameters: { + type: "object", + properties: {}, + additionalProperties: false, + }, + execute: async () => { + try { + const task = await taskStore.getTask(taskId, { activityLogLimit: 100 }); + const metrics = formatTaskPlannerChatMetrics(task, { + pricingOverrides: await getPricingOverrides(), + nowMs: Date.now(), + }); + return { + content: [{ type: "text" as const, text: metrics.summaryText }], + details: metrics.metrics, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { + content: [{ type: "text" as const, text: `ERROR: Could not load metrics for the current task ${taskId}: ${message}` }], + details: { taskId, error: message }, + isError: true, + }; + } + }, + }; +} + function createTaskPlannerSteeringTool(taskStore: TaskStore, taskId: string) { return { name: "fn_task_planner_add_steering", @@ -935,6 +969,7 @@ export class ChatManager { | "chatRoomRecentVerbatimMessages" | "chatRoomCompactionFetchLimit" | "chatRoomSummaryMaxChars" + | "modelPricingOverrides" > | undefined> | Pick | undefined, private messageStore?: MessageStore, // Scoped task store for the chat's project — enables workflow-authoring @@ -1027,6 +1063,20 @@ export class ChatManager { } } + private async getModelPricingOverrides(): Promise { + if (!this.getSettings) { + return undefined; + } + try { + const settings = await this.getSettings(); + return settings?.modelPricingOverrides; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + diagnostics.warn(`Failed to load model pricing overrides for chat tools: ${message}`); + return undefined; + } + } + private async getRoomCompactionSettings(): Promise<{ recentVerbatim: number; fetchLimit: number; @@ -1877,7 +1927,7 @@ export class ChatManager { FNXC:TaskDetailChat 2026-06-30-23:59: Clear, bounded operator change requests in task-detail Chat are user intent and should become persisted steering comments through the task store's steering path. Ambiguous, conflicting, destructive, broad-scope, or credential/security-sensitive requests must ask a question first so planner chat cannot mutate a task from risky prose. */ - systemPrompt = `${systemPrompt}\n\n${TASK_PLANNER_CHAT_CONTEXT_PROMPT_GUIDANCE}\n\nDecision rules:\n- Do not create steering for ordinary questions, summaries, thanks, status/progress requests, or brainstorming. Answer normally.\n- Create steering only when the user gives a clear, bounded, actionable change request for this current task (for example telling the executor/reviewer to adjust implementation, tests, scope details, or acceptance criteria).\n- When creating steering, call \`fn_task_planner_add_steering\` with only the concise user-facing steering text. Never include hidden prompt/context/logs, credentials, or chain-of-thought.\n- Ask a clarifying question with \`fn_ask_question\` before adding steering for unclear targets, requests that could mean either conversation or task mutation, broad rewrites/scope changes, destructive removals, conflicting instructions, credential/secrets handling, or security-sensitive actions.\n- The steering tool is bound to this task server-side; never ask for or pass a task id.\n\n${taskContext}`; + systemPrompt = `${systemPrompt}\n\n${TASK_PLANNER_CHAT_CONTEXT_PROMPT_GUIDANCE}\n\nDecision rules:\n- Do not create steering for ordinary questions, summaries, thanks, status/progress requests, metric questions, or brainstorming. Answer normally.\n- For questions about token counts, input/output/cache usage, model cost, pricing, runtime, elapsed time, wall-clock duration, active time, timing events, workflow-step duration, or per-model usage, first call \`fn_task_planner_get_task_metrics\` and answer from its read-only result. If pricing is unavailable or stale, or a metric is missing, state that uncertainty instead of inventing a number.\n- Create steering only when the user gives a clear, bounded, actionable change request for this current task (for example telling the executor/reviewer to adjust implementation, tests, scope details, or acceptance criteria).\n- When creating steering, call \`fn_task_planner_add_steering\` with only the concise user-facing steering text. Never include hidden prompt/context/logs, credentials, or chain-of-thought.\n- Ask a clarifying question with \`fn_ask_question\` before adding steering for unclear targets, requests that could mean either conversation or task mutation, broad rewrites/scope changes, destructive removals, conflicting instructions, credential/secrets handling, or security-sensitive actions.\n- The steering and metrics tools are bound to this task server-side; never ask for or pass a task id.\n\n${taskContext}`; } if (agent) { @@ -1988,8 +2038,15 @@ export class ChatManager { const taskPlannerSteeringTools = this.taskStore && taskPlannerChatTaskId ? [createTaskPlannerSteeringTool(this.taskStore, taskPlannerChatTaskId)] : []; + /* + FNXC:TaskPlannerChatMetrics 2026-07-01-20:55: + Task-detail planner Chat needs a read-only, task-scoped metrics tool so token, cost, and timing answers come from persisted task fields. Register it only for synthetic task-planner: sessions and bind the task id server-side so normal Chat, room Chat, and arbitrary task lookup stay out of scope. + */ + const taskPlannerMetricsTools = this.taskStore && taskPlannerChatTaskId + ? [createTaskPlannerMetricsTool(this.taskStore, taskPlannerChatTaskId, () => this.getModelPricingOverrides())] + : []; - const customTools = [createAskQuestionTool(), ...taskPlannerSteeringTools, ...messagingTools, ...workflowTools, ...documentTools, ...artifactTools]; + const customTools = [createAskQuestionTool(), ...taskPlannerSteeringTools, ...taskPlannerMetricsTools, ...messagingTools, ...workflowTools, ...documentTools, ...artifactTools]; const sessionOptions = { cwd: this.rootDir, diff --git a/packages/dashboard/src/task-planner-chat-context.ts b/packages/dashboard/src/task-planner-chat-context.ts index 77d5646bc5..6ad6780fd1 100644 --- a/packages/dashboard/src/task-planner-chat-context.ts +++ b/packages/dashboard/src/task-planner-chat-context.ts @@ -321,6 +321,9 @@ export async function buildTaskPlannerChatContext(taskStore: TaskStore, taskId: /* FNXC:TaskDetailPlannerChat 2026-06-30-23:58: Task-detail planner Chat receives this server-built, bounded task snapshot so the planner can answer current status, progress, dependency, recent-activity, and prompt/plan questions without trusting client-supplied context. Activity remains the operational steering/execution transcript; this formatter exposes comments/log excerpts as read-only context and records unavailable sections explicitly so the planner states uncertainty instead of inventing fresh execution evidence. + +FNXC:TaskPlannerChatMetrics 2026-07-01-20:58: +Metric questions in task-detail planner Chat must use the read-only task-scoped metrics tool, not Activity steering or prose inference. Prompt guidance names token, cost, runtime, timing-event, workflow-step, and per-model surfaces so ordinary metrics questions remain answers rather than task mutations. */ export const TASK_PLANNER_CHAT_CONTEXT_PROMPT_GUIDANCE = `## Task Planner Chat Context -You are answering in the task detail Chat tab for a single Fusion task. Use the bounded server-supplied context below to answer questions about current status, progress, dependencies, recent activity, source/review state, and the task prompt or plan. State uncertainty when a section is absent, stale, truncated, or marked unavailable. Do not claim you ran code, tests, builds, or inspected files unless the supplied context or explicit tool output says so. Keep Activity separate from Chat: Activity is the execution/steering transcript, while this Chat reply is a planner conversation. Do not mutate steering comments for ordinary status/progress questions.`; +You are answering in the task detail Chat tab for a single Fusion task. Use the bounded server-supplied context below to answer questions about current status, progress, dependencies, recent activity, source/review state, and the task prompt or plan. For token counts, cost, runtime, elapsed/wall-clock duration, timing events, workflow-step duration, or per-model usage, call \`fn_task_planner_get_task_metrics\` and answer from the tool result; state when pricing is unavailable/stale or metrics are missing instead of inventing values. State uncertainty when a section is absent, stale, truncated, or marked unavailable. Do not claim you ran code, tests, builds, or inspected files unless the supplied context or explicit tool output says so. Keep Activity separate from Chat: Activity is the execution/steering transcript, while this Chat reply is a planner conversation. Do not mutate steering comments for ordinary status/progress/metrics questions.`; diff --git a/packages/dashboard/src/task-planner-chat-metrics.ts b/packages/dashboard/src/task-planner-chat-metrics.ts new file mode 100644 index 0000000000..42c56300ea --- /dev/null +++ b/packages/dashboard/src/task-planner-chat-metrics.ts @@ -0,0 +1,445 @@ +import type { ModelPricingOverrides, Task, TaskLogEntry, TaskTokenUsagePerModel, WorkflowStepResult } from "@fusion/core"; +import { costFor } from "@fusion/core"; + +type MetricsTask = Pick< + Task, + | "id" + | "title" + | "column" + | "status" + | "tokenUsage" + | "log" + | "timedExecutionMs" + | "workflowStepResults" + | "executionStartedAt" + | "executionCompletedAt" + | "firstExecutionAt" + | "cumulativeActiveMs" +>; + +type TokenBucketInput = Pick< + TaskTokenUsagePerModel, + "modelProvider" | "modelId" | "inputTokens" | "outputTokens" | "cachedTokens" | "cacheWriteTokens" | "totalTokens" +> & Partial>; + +export interface TaskPlannerTokenCostMetrics { + usd: number | null; + costUnavailable: boolean; + pricingStale: boolean; +} + +export interface TaskPlannerTokenBucketMetrics extends TokenBucketInput { + key: string; + cost: TaskPlannerTokenCostMetrics; +} + +export interface TaskPlannerTimingEventMetrics { + timestamp?: string; + summary: string; + durationMs: number | null; +} + +export interface TaskPlannerWorkflowStepTimingMetrics { + workflowStepId: string; + workflowStepName: string; + status: string; + startedAt?: string; + completedAt?: string; + durationMs: number | null; + running: boolean; +} + +export interface TaskPlannerChatMetricsPayload { + taskId: string; + title?: string; + column?: string; + status?: string; + tokens: { + available: boolean; + inputTokens: number; + outputTokens: number; + cachedTokens: number; + cacheWriteTokens: number; + totalTokens: number; + firstUsedAt: string | null; + lastUsedAt: string | null; + malformedTimestamps: string[]; + perModel: TaskPlannerTokenBucketMetrics[]; + cost: TaskPlannerTokenCostMetrics; + }; + timing: { + executionStartedAt: string | null; + executionCompletedAt: string | null; + firstExecutionAt: string | null; + endToEndExecutionMs: number | null; + wallClockSinceFirstExecutionMs: number | null; + activeRuntimeMs: number | null; + cumulativeActiveMs: number | null; + timedExecutionMs: number | null; + logTimingDurationMs: number | null; + timingEventCount: number; + timedTimingEventCount: number; + workflowRuntimeMs: number | null; + timedWorkflowStepCount: number; + totalExecutionMs: number | null; + longestTimingEvent: TaskPlannerTimingEventMetrics | null; + longestWorkflowStep: TaskPlannerWorkflowStepTimingMetrics | null; + timingEvents: TaskPlannerTimingEventMetrics[]; + workflowSteps: TaskPlannerWorkflowStepTimingMetrics[]; + malformedTimestamps: string[]; + }; +} + +export interface TaskPlannerChatMetricsResult { + metrics: TaskPlannerChatMetricsPayload; + summaryText: string; +} + +function finiteNumber(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : 0; +} + +function optionalFiniteNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null; +} + +function validTimestamp(value: unknown, malformed: string[]): string | null { + if (typeof value !== "string" || !value.trim()) return null; + const trimmed = value.trim(); + if (!Number.isFinite(Date.parse(trimmed))) { + malformed.push(trimmed); + return null; + } + return trimmed; +} + +function parseTimestampToMs(value: unknown, malformed: string[]): number | null { + const timestamp = validTimestamp(value, malformed); + if (!timestamp) return null; + return Date.parse(timestamp); +} + +function formatDuration(ms: number | null): string { + if (ms == null) return "not available"; + if (ms < 1000) return `${Math.round(ms)} ms`; + const seconds = ms / 1000; + if (seconds < 60) return `${seconds.toFixed(1)} s`; + const minutes = Math.floor(seconds / 60); + const remainingSeconds = Math.round(seconds % 60); + if (minutes < 60) return `${minutes}m ${remainingSeconds}s`; + const hours = Math.floor(minutes / 60); + const remainingMinutes = minutes % 60; + return `${hours}h ${remainingMinutes}m ${remainingSeconds}s`; +} + +function formatUsd(usd: number | null): string { + if (usd == null || !Number.isFinite(usd)) return "unavailable"; + return `$${usd.toLocaleString(undefined, { minimumFractionDigits: 4, maximumFractionDigits: 4 })}`; +} + +function summarizeTimingLabel(entry: TaskLogEntry): string { + const actionText = typeof entry.action === "string" ? entry.action : ""; + const outcomeText = typeof entry.outcome === "string" ? entry.outcome : ""; + const timingText = actionText.includes("[timing]") ? actionText : outcomeText.includes("[timing]") ? outcomeText : `${actionText}\n${outcomeText}`; + const stripped = timingText + .replace(/^\[timing\]\s*/i, "") + .replace(/^\[[^\]]+\]\s*/i, "") + .replace(/\s+in\s+\d+(?:\.\d+)?ms\b/i, "") + .replace(/\s+after\s+\d+(?:\.\d+)?ms\b/i, "") + .trim(); + return stripped || "Timing event"; +} + +function extractTimingEvents(logEntries: TaskLogEntry[] | undefined): TaskPlannerTimingEventMetrics[] { + return (logEntries ?? []) + .filter((entry) => { + const actionText = typeof entry.action === "string" ? entry.action : ""; + const outcomeText = typeof entry.outcome === "string" ? entry.outcome : ""; + return actionText.includes("[timing]") || outcomeText.includes("[timing]"); + }) + .map((entry) => { + const haystack = `${entry.action ?? ""}\n${entry.outcome ?? ""}`; + const durationMatch = haystack.match(/(\d+(?:\.\d+)?)ms\b/i); + const durationMs = durationMatch ? Number(durationMatch[1]) : NaN; + return { + timestamp: entry.timestamp, + summary: summarizeTimingLabel(entry), + durationMs: Number.isFinite(durationMs) ? durationMs : null, + }; + }); +} + +function bucketKey(bucket: Pick): string { + return `${bucket.modelProvider ?? ""}:${bucket.modelId ?? ""}`; +} + +function normalizeBucket(bucket: TokenBucketInput): TokenBucketInput { + return { + modelProvider: bucket.modelProvider?.trim() || undefined, + modelId: bucket.modelId?.trim() || undefined, + inputTokens: finiteNumber(bucket.inputTokens), + outputTokens: finiteNumber(bucket.outputTokens), + cachedTokens: finiteNumber(bucket.cachedTokens), + cacheWriteTokens: finiteNumber(bucket.cacheWriteTokens), + totalTokens: finiteNumber(bucket.totalTokens), + firstUsedAt: bucket.firstUsedAt, + lastUsedAt: bucket.lastUsedAt, + }; +} + +function mergeBuckets(buckets: TokenBucketInput[]): TokenBucketInput[] { + const merged = new Map(); + for (const rawBucket of buckets) { + const bucket = normalizeBucket(rawBucket); + const key = bucketKey(bucket); + const current = merged.get(key); + if (!current) { + merged.set(key, { ...bucket }); + continue; + } + current.inputTokens += bucket.inputTokens; + current.outputTokens += bucket.outputTokens; + current.cachedTokens += bucket.cachedTokens; + current.cacheWriteTokens += bucket.cacheWriteTokens; + current.totalTokens += bucket.totalTokens; + current.firstUsedAt = minTimestampString(current.firstUsedAt, bucket.firstUsedAt); + current.lastUsedAt = maxTimestampString(current.lastUsedAt, bucket.lastUsedAt); + } + return Array.from(merged.values()); +} + +function minTimestampString(left?: string, right?: string): string | undefined { + if (!left) return right; + if (!right) return left; + const leftMs = Date.parse(left); + const rightMs = Date.parse(right); + if (!Number.isFinite(leftMs)) return right; + if (!Number.isFinite(rightMs)) return left; + return rightMs < leftMs ? right : left; +} + +function maxTimestampString(left?: string, right?: string): string | undefined { + if (!left) return right; + if (!right) return left; + const leftMs = Date.parse(left); + const rightMs = Date.parse(right); + if (!Number.isFinite(leftMs)) return right; + if (!Number.isFinite(rightMs)) return left; + return rightMs > leftMs ? right : left; +} + +function buildTokenMetrics(task: MetricsTask, pricingOverrides: ModelPricingOverrides | undefined, nowMs: number): TaskPlannerChatMetricsPayload["tokens"] { + const tokenUsage = task.tokenUsage; + const malformedTimestamps: string[] = []; + if (!tokenUsage) { + return { + available: false, + inputTokens: 0, + outputTokens: 0, + cachedTokens: 0, + cacheWriteTokens: 0, + totalTokens: 0, + firstUsedAt: null, + lastUsedAt: null, + malformedTimestamps, + perModel: [], + cost: { usd: null, costUnavailable: false, pricingStale: false }, + }; + } + + const buckets = tokenUsage.perModel?.length + ? mergeBuckets(tokenUsage.perModel) + : mergeBuckets([{ + modelProvider: tokenUsage.modelProvider, + modelId: tokenUsage.modelId, + inputTokens: tokenUsage.inputTokens, + outputTokens: tokenUsage.outputTokens, + cachedTokens: tokenUsage.cachedTokens, + cacheWriteTokens: tokenUsage.cacheWriteTokens, + totalTokens: tokenUsage.totalTokens, + firstUsedAt: tokenUsage.firstUsedAt, + lastUsedAt: tokenUsage.lastUsedAt, + }]); + + let totalUsd = 0; + let costUnavailable = false; + let pricingStale = false; + const perModel = buckets.map((bucket) => { + const cost = costFor( + { + inputTokens: bucket.inputTokens, + outputTokens: bucket.outputTokens, + cachedTokens: bucket.cachedTokens, + cacheWriteTokens: bucket.cacheWriteTokens, + }, + { provider: bucket.modelProvider, model: bucket.modelId }, + nowMs, + pricingOverrides, + ); + if (bucket.totalTokens > 0 && (cost.unavailable || cost.usd === null || !Number.isFinite(cost.usd))) { + costUnavailable = true; + } else if (cost.usd != null && Number.isFinite(cost.usd)) { + totalUsd += cost.usd; + } + pricingStale ||= cost.stale; + return { + ...bucket, + key: bucketKey(bucket), + cost: { + usd: cost.usd, + costUnavailable: cost.unavailable, + pricingStale: cost.stale, + }, + }; + }); + + return { + available: true, + inputTokens: finiteNumber(tokenUsage.inputTokens), + outputTokens: finiteNumber(tokenUsage.outputTokens), + cachedTokens: finiteNumber(tokenUsage.cachedTokens), + cacheWriteTokens: finiteNumber(tokenUsage.cacheWriteTokens), + totalTokens: finiteNumber(tokenUsage.totalTokens), + firstUsedAt: validTimestamp(tokenUsage.firstUsedAt, malformedTimestamps), + lastUsedAt: validTimestamp(tokenUsage.lastUsedAt, malformedTimestamps), + malformedTimestamps, + perModel, + cost: { + usd: costUnavailable ? null : totalUsd, + costUnavailable, + pricingStale, + }, + }; +} + +function buildWorkflowStepTimings(results: WorkflowStepResult[] | undefined, nowMs: number, malformed: string[]): TaskPlannerWorkflowStepTimingMetrics[] { + return (results ?? []).map((step) => { + const startedMs = parseTimestampToMs(step.startedAt, malformed); + if (startedMs == null) { + return { + workflowStepId: step.workflowStepId, + workflowStepName: step.workflowStepName || step.workflowStepId, + status: step.status, + startedAt: typeof step.startedAt === "string" ? step.startedAt : undefined, + completedAt: typeof step.completedAt === "string" ? step.completedAt : undefined, + durationMs: null, + running: false, + }; + } + const completedMs = parseTimestampToMs(step.completedAt, malformed); + const running = completedMs == null; + const endMs = completedMs != null && completedMs >= startedMs ? completedMs : Math.max(startedMs, nowMs); + return { + workflowStepId: step.workflowStepId, + workflowStepName: step.workflowStepName || step.workflowStepId, + status: step.status, + startedAt: step.startedAt, + completedAt: step.completedAt, + durationMs: endMs - startedMs, + running, + }; + }); +} + +function buildTimingMetrics(task: MetricsTask, nowMs: number): TaskPlannerChatMetricsPayload["timing"] { + const malformedTimestamps: string[] = []; + const executionStartedMs = parseTimestampToMs(task.executionStartedAt, malformedTimestamps); + const executionCompletedMs = parseTimestampToMs(task.executionCompletedAt, malformedTimestamps); + const firstExecutionMs = parseTimestampToMs(task.firstExecutionAt, malformedTimestamps); + const executionStartedAt = executionStartedMs == null ? null : task.executionStartedAt ?? null; + const executionCompletedAt = executionCompletedMs == null ? null : task.executionCompletedAt ?? null; + const firstExecutionAt = firstExecutionMs == null ? null : task.firstExecutionAt ?? null; + + const endToEndExecutionMs = executionStartedMs == null + ? null + : Math.max(0, (executionCompletedMs != null && executionCompletedMs >= executionStartedMs ? executionCompletedMs : nowMs) - executionStartedMs); + const wallClockSinceFirstExecutionMs = firstExecutionMs == null + ? null + : Math.max(0, (executionCompletedMs ?? nowMs) - firstExecutionMs); + const cumulativeActiveMs = optionalFiniteNumber(task.cumulativeActiveMs); + const activeRuntimeMs = task.column === "in-progress" && executionStartedMs != null + ? (cumulativeActiveMs ?? 0) + Math.max(0, nowMs - executionStartedMs) + : cumulativeActiveMs; + + const timingEvents = extractTimingEvents(task.log); + const timedEvents = timingEvents.filter((event) => event.durationMs != null); + const logTimingDurationMs = timedEvents.length > 0 + ? timedEvents.reduce((sum, event) => sum + (event.durationMs ?? 0), 0) + : null; + const timedExecutionMs = optionalFiniteNumber(task.timedExecutionMs); + const workflowSteps = buildWorkflowStepTimings(task.workflowStepResults, nowMs, malformedTimestamps); + const timedWorkflowSteps = workflowSteps.filter((step) => step.durationMs != null); + const workflowRuntimeMs = timedWorkflowSteps.length > 0 + ? timedWorkflowSteps.reduce((sum, step) => sum + (step.durationMs ?? 0), 0) + : null; + const longestTimingEvent = timedEvents.reduce((longest, event) => { + if (!longest || (event.durationMs ?? 0) > (longest.durationMs ?? 0)) return event; + return longest; + }, null); + const longestWorkflowStep = timedWorkflowSteps.reduce((longest, step) => { + if (!longest || (step.durationMs ?? 0) > (longest.durationMs ?? 0)) return step; + return longest; + }, null); + const totalExecutionMs = activeRuntimeMs + ?? endToEndExecutionMs + ?? timedExecutionMs + ?? (logTimingDurationMs != null || workflowRuntimeMs != null ? (logTimingDurationMs ?? 0) + (workflowRuntimeMs ?? 0) : null); + + return { + executionStartedAt, + executionCompletedAt, + firstExecutionAt, + endToEndExecutionMs, + wallClockSinceFirstExecutionMs, + activeRuntimeMs, + cumulativeActiveMs, + timedExecutionMs, + logTimingDurationMs, + timingEventCount: timingEvents.length, + timedTimingEventCount: timedEvents.length, + workflowRuntimeMs, + timedWorkflowStepCount: timedWorkflowSteps.length, + totalExecutionMs, + longestTimingEvent, + longestWorkflowStep, + timingEvents, + workflowSteps, + malformedTimestamps, + }; +} + +/** + * FNXC:TaskPlannerChatMetrics 2026-07-01-20:48: + * Task-detail planner Chat must answer token, cost, and timing questions from durable task fields instead of asking the model to infer numbers from prose. Keep this helper pure and read-only so the scoped chat tool can expose exact persisted metrics without mutating Activity, steering, documents, or task state. + * + * FNXC:TaskPlannerChatMetrics 2026-07-01-20:48: + * Pricing estimates are derived at read time with costFor and optional settings overrides; never persist them here. Unknown or stale pricing stays explicit as unavailable/stale so planner Chat cannot understate cost by reporting missing model prices as $0. + */ +export function formatTaskPlannerChatMetrics( + task: MetricsTask, + options: { pricingOverrides?: ModelPricingOverrides; nowMs?: number } = {}, +): TaskPlannerChatMetricsResult { + const nowMs = options.nowMs ?? Date.now(); + const metrics: TaskPlannerChatMetricsPayload = { + taskId: task.id, + title: task.title, + column: task.column, + status: task.status, + tokens: buildTokenMetrics(task, options.pricingOverrides, nowMs), + timing: buildTimingMetrics(task, nowMs), + }; + + const tokenSummary = metrics.tokens.available + ? `${metrics.tokens.totalTokens.toLocaleString()} total tokens (${metrics.tokens.inputTokens.toLocaleString()} input, ${metrics.tokens.outputTokens.toLocaleString()} output, ${metrics.tokens.cachedTokens.toLocaleString()} cache read, ${metrics.tokens.cacheWriteTokens.toLocaleString()} cache write)` + : "no token usage recorded"; + const costSummary = metrics.tokens.cost.costUnavailable + ? "cost unavailable because at least one model has no pricing" + : `estimated cost ${formatUsd(metrics.tokens.cost.usd)}`; + const staleSuffix = metrics.tokens.cost.pricingStale ? "; pricing is stale" : ""; + const timingSummary = `total execution ${formatDuration(metrics.timing.totalExecutionMs)}, active runtime ${formatDuration(metrics.timing.activeRuntimeMs)}, ${metrics.timing.timingEventCount.toLocaleString()} timing events, ${metrics.timing.timedWorkflowStepCount.toLocaleString()} workflow steps with timing`; + + return { + metrics, + summaryText: `Task ${metrics.taskId} metrics: ${tokenSummary}; ${costSummary}${staleSuffix}; ${timingSummary}.`, + }; +}