From 605e4d7734ac75245b1868cead7dcdb24128d290 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 27 Jun 2026 20:33:22 -0700 Subject: [PATCH] FN-7158: emit reflection run-audit telemetry Record agent reflection outcomes in run audit without persisting reflection prose.\n\n- Emit generated, skipped, and failed reflection telemetry from AgentReflectionService.\n- Add run-audit metadata contracts, diagnostics docs, and operator-facing run-audit guidance.\n- Cover telemetry payloads and best-effort audit failures in agent reflection tests.\n- Add a minor changeset for the published Fusion CLI package.\n\nFiles changed:\n .changeset/fn-7158-reflection-telemetry.md | 7 ++\n AGENTS.md | 1 +\n docs/diagnostics.md | 12 +++\n .../engine/src/__tests__/agent-reflection.test.ts | 99 ++++++++++++++++++++++\n packages/engine/src/agent-reflection.ts | 60 ++++++++++++-\n packages/engine/src/run-audit.ts | 41 +++++++++\n 6 files changed, 219 insertions(+), 1 deletion(-) Fusion-Task-Id: FN-7158 Fusion-Task-Lineage: e8a83fe9-2112-4ce4-9367-a6fa38b6fbb4 Co-authored-by: Fusion (runfusion.ai) --- .changeset/fn-7158-reflection-telemetry.md | 7 ++ AGENTS.md | 1 + docs/diagnostics.md | 12 +++ .../src/__tests__/agent-reflection.test.ts | 99 +++++++++++++++++++ packages/engine/src/agent-reflection.ts | 60 ++++++++++- packages/engine/src/run-audit.ts | 41 ++++++++ 6 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 .changeset/fn-7158-reflection-telemetry.md diff --git a/.changeset/fn-7158-reflection-telemetry.md b/.changeset/fn-7158-reflection-telemetry.md new file mode 100644 index 0000000000..637d629667 --- /dev/null +++ b/.changeset/fn-7158-reflection-telemetry.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": minor +--- + +summary: Emit run-audit telemetry for agent performance reflections. +category: feature +dev: Adds reflection:generated/skipped/failed DatabaseMutationType events emitted from AgentReflectionService.generateReflection; metadata carries ids/counts/outcomes only. diff --git a/AGENTS.md b/AGENTS.md index e63c248105..1db71d73d1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -219,6 +219,7 @@ Scoped exception (FN-5819): shared-branch-group members (`branchContext.assignme ### Run Audit +- FN-7158: agent performance reflections emit `reflection:generated`, `reflection:skipped`, and `reflection:failed` with ids/counts/outcomes-only metadata; never persist reflection prose or prompt text in run-audit. - FN-7011: self-healing emits `task:reconcile-engine-downtime-active-timing` when startup recovery shifts active task segment anchors to exclude proven engine-process downtime, and `task:reconcile-engine-downtime-active-timing-no-action` when no active task qualifies. - FN-5419: git run-audit now includes `pull:fast-forward` and `stash:pop-conflict`; dashboard git surfaces now include the extended `POST /api/git/pull` integration-worktree path plus companion `POST /api/git/stash-resolve`, `POST /api/git/stash-drop`, and `POST /api/git/stash-apply` routes. - FN-6292: self-healing emits `task:reconcile-dependency-blocking-lease` when it rebounds an in-progress holder whose stale file-scope lease blocks an unmet dependency, and `task:reconcile-dependency-blocking-lease-no-action` when triple-proof blocks that backward move. diff --git a/docs/diagnostics.md b/docs/diagnostics.md index 1e67cbaff2..8f32af2551 100644 --- a/docs/diagnostics.md +++ b/docs/diagnostics.md @@ -13,6 +13,18 @@ Executor, heartbeat, and planning runs emit one goal-injection diagnostic with o - `goalIds` / `goalCount` describe the active goals injected into the prompt; `provenanceGoalIds` additively records mission-derived task provenance and does not affect prompt selection. - Guardrail: diagnostics persist goal IDs/counts only; never prompt text, goal titles, or goal descriptions. +## Agent performance reflection telemetry + +Agent reflection generation emits one run-audit event for every `AgentReflectionService.generateReflection` attempt, covering manual dashboard requests, executor/post-task tools, heartbeat tools, and self-improve callers from the shared service seam. + +- Run-audit events (`database` domain, target `agentId`): + - `reflection:generated` metadata: `{ agentId, trigger, taskId?, reflectionId, tasksCompleted?, tasksFailed?, avgDurationMs?, commonErrorCount, insightCount, suggestedImprovementCount }`. + - `reflection:skipped` metadata: `{ agentId, trigger, taskId?, reason: "no-history" }`. + - `reflection:failed` metadata: `{ agentId, trigger, taskId?, errorClass }`. +- Trigger taxonomy is preserved from `ReflectionTrigger`: `manual`, `periodic`, `post-task`, and `user-requested`. +- The events use synthetic run context with phase `reflection` and source equal to the trigger so they correlate in the run-audit stream without requiring caller-specific wiring. +- Guardrail: reflection diagnostics persist IDs, counts, reasons, and error classes only; never prompt text, reflection summaries, insight strings, suggested-improvement text, or free-form trigger details. + ## Insight run sweeper (`[insight-sweeper]`) The dashboard insight router runs stale-run recovery sweeps for `project_insight_runs` rows stuck in `pending`/`running` without a live controller owner. diff --git a/packages/engine/src/__tests__/agent-reflection.test.ts b/packages/engine/src/__tests__/agent-reflection.test.ts index 613c7201bd..3221832ed1 100644 --- a/packages/engine/src/__tests__/agent-reflection.test.ts +++ b/packages/engine/src/__tests__/agent-reflection.test.ts @@ -107,6 +107,7 @@ function createMockDeps() { const taskStore = { listTasks: vi.fn().mockResolvedValue([makeTask()]), + recordRunAuditEvent: vi.fn(), } as any; const reflectionStore = { @@ -152,6 +153,17 @@ function createMockSession() { } as any; } +function expectNoReflectionProse(metadata: Record) { + expect(metadata).not.toHaveProperty("summary"); + expect(metadata).not.toHaveProperty("insights"); + expect(metadata).not.toHaveProperty("suggestedImprovements"); + expect(metadata).not.toHaveProperty("triggerDetail"); + expect(Object.values(metadata)).not.toContain("Execution quality is strong with room to tighten feedback loops."); + expect(Object.values(metadata)).not.toContain("Strong execution on scoped changes"); + expect(Object.values(metadata)).not.toContain("Run tests earlier in the cycle"); + expect(Object.values(metadata)).not.toContain("manual check"); +} + describe("AgentReflectionService", () => { let tempRoot: string; @@ -382,6 +394,29 @@ describe("AgentReflectionService", () => { expect(mockedCreateFnAgent).toHaveBeenCalledWith(expect.objectContaining({ tools: "readonly", })); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledTimes(1); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + agentId: "agent-1", + taskId: "FN-001", + domain: "database", + mutationType: "reflection:generated", + target: "agent-1", + metadata: expect.objectContaining({ + phase: "reflection", + source: "manual", + agentId: "agent-1", + trigger: "manual", + taskId: "FN-001", + reflectionId: "reflection-1", + tasksCompleted: 1, + tasksFailed: 0, + avgDurationMs: 3_600_000, + commonErrorCount: 1, + insightCount: 1, + suggestedImprovementCount: 1, + }), + })); + expectNoReflectionProse(taskStore.recordRunAuditEvent.mock.calls[0][0].metadata); }); it("returns null when no meaningful data exists", async () => { @@ -395,6 +430,21 @@ describe("AgentReflectionService", () => { expect(reflection).toBeNull(); expect(mockedCreateFnAgent).not.toHaveBeenCalled(); expect(reflectionStore.createReflection).not.toHaveBeenCalled(); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledTimes(1); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + agentId: "agent-1", + domain: "database", + mutationType: "reflection:skipped", + target: "agent-1", + metadata: expect.objectContaining({ + phase: "reflection", + source: "manual", + agentId: "agent-1", + trigger: "manual", + reason: "no-history", + }), + })); + expectNoReflectionProse(taskStore.recordRunAuditEvent.mock.calls[0][0].metadata); }); it("returns null on AI session failure", async () => { @@ -406,6 +456,21 @@ describe("AgentReflectionService", () => { expect(reflection).toBeNull(); expect(reflectionStore.createReflection).not.toHaveBeenCalled(); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledTimes(1); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + agentId: "agent-1", + domain: "database", + mutationType: "reflection:failed", + target: "agent-1", + metadata: expect.objectContaining({ + phase: "reflection", + source: "manual", + agentId: "agent-1", + trigger: "manual", + errorClass: "Error", + }), + })); + expectNoReflectionProse(taskStore.recordRunAuditEvent.mock.calls[0][0].metadata); }); it("persists reflection via reflectionStore.createReflection", async () => { @@ -433,6 +498,40 @@ describe("AgentReflectionService", () => { taskId: "FN-777", triggerDetail: "post-task reflection", })); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledWith(expect.objectContaining({ + mutationType: "reflection:generated", + metadata: expect.objectContaining({ + source: "post-task", + trigger: "post-task", + taskId: "FN-777", + }), + })); + }); + + it("keeps successful reflection generation best-effort when audit persistence fails", async () => { + const { agentStore, taskStore, reflectionStore } = createMockDeps(); + const session = createMockSession(); + taskStore.recordRunAuditEvent.mockRejectedValue(new Error("audit unavailable")); + + mockedCreateFnAgent.mockImplementation(async (options: any) => { + options.onText?.(JSON.stringify({ + insights: ["Insight A"], + suggestedImprovements: ["Improve B"], + summary: "Summary C", + })); + return { session }; + }); + mockedPromptWithFallback.mockResolvedValue(undefined); + + const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot }); + const reflection = await service.generateReflection("agent-1", "user-requested", { + taskId: "FN-999", + }); + + expect(reflection).not.toBeNull(); + expect(reflection?.trigger).toBe("user-requested"); + expect(reflectionStore.createReflection).toHaveBeenCalledTimes(1); + expect(taskStore.recordRunAuditEvent).toHaveBeenCalledTimes(1); }); }); diff --git a/packages/engine/src/agent-reflection.ts b/packages/engine/src/agent-reflection.ts index 523367b25a..72c9317c46 100644 --- a/packages/engine/src/agent-reflection.ts +++ b/packages/engine/src/agent-reflection.ts @@ -15,6 +15,7 @@ import type { import { createLogger } from "./logger.js"; import { createFnAgent, promptWithFallback } from "./pi.js"; import { resolveMcpServersForStore } from "./mcp-resolution.js"; +import { createRunAuditor, generateSyntheticRunId, type EngineRunContext, type RunAuditor } from "./run-audit.js"; const reflectionLog = createLogger("reflection"); @@ -93,12 +94,24 @@ export class AgentReflectionService { trigger: ReflectionTrigger, options: { taskId?: string; triggerDetail?: string } = {}, ): Promise { + const runContext: EngineRunContext = { + runId: generateSyntheticRunId("reflection", agentId), + agentId, + ...(options.taskId ? { taskId: options.taskId } : {}), + phase: "reflection", + source: trigger, + }; + const auditor = createRunAuditor(this.taskStore, runContext); + try { const context = await this.buildReflectionContext(agentId); const recentRuns = await this.agentStore.getRecentRuns(agentId, DEFAULT_OUTCOME_LIMIT); if (context.recentOutcomes.length === 0 && recentRuns.length === 0) { reflectionLog.log(`Skipping reflection for ${agentId}: no recent tasks or heartbeat runs`); + await this.emitReflectionAudit(auditor, "reflection:skipped", agentId, trigger, options, { + reason: "no-history", + }); return null; } @@ -135,7 +148,7 @@ export class AgentReflectionService { const parsed = this.parseReflectionResponse(responseText); const metrics = this.buildReflectionMetrics(context.recentOutcomes, context.performanceSummary, recentRuns); - return await this.reflectionStore.createReflection({ + const reflection = await this.reflectionStore.createReflection({ agentId, trigger, triggerDetail: options.triggerDetail, @@ -145,12 +158,57 @@ export class AgentReflectionService { suggestedImprovements: parsed.suggestedImprovements, summary: parsed.summary, }); + + await this.emitReflectionAudit(auditor, "reflection:generated", agentId, trigger, options, { + reflectionId: reflection.id, + ...(metrics.tasksCompleted !== undefined ? { tasksCompleted: metrics.tasksCompleted } : {}), + ...(metrics.tasksFailed !== undefined ? { tasksFailed: metrics.tasksFailed } : {}), + ...(metrics.avgDurationMs !== undefined ? { avgDurationMs: metrics.avgDurationMs } : {}), + commonErrorCount: metrics.commonErrors?.length ?? 0, + insightCount: parsed.insights.length, + suggestedImprovementCount: parsed.suggestedImprovements.length, + }); + + return reflection; } catch (error) { + await this.emitReflectionAudit(auditor, "reflection:failed", agentId, trigger, options, { + errorClass: error instanceof Error ? error.name : typeof error, + }); reflectionLog.error(`Failed to generate reflection for ${agentId}: ${(error as Error).message}`); return null; } } + private async emitReflectionAudit( + auditor: RunAuditor, + type: "reflection:generated" | "reflection:skipped" | "reflection:failed", + agentId: string, + trigger: ReflectionTrigger, + options: { taskId?: string; triggerDetail?: string }, + metadata: Record, + ): Promise { + try { + /* + FNXC:AgentReflectionTelemetry 2026-06-27-00:00: + Emitting from AgentReflectionService.generateReflection covers manual dashboard, executor post-task/in-session tool, heartbeat tool, and self-improve callers through one seam. Keep the payload ids/counts/outcomes-only so run-audit can diagnose reflection activity without storing reflection prose, triggerDetail, or prompt text. + */ + await auditor.database({ + type, + target: agentId, + metadata: { + agentId, + trigger, + ...(options.taskId ? { taskId: options.taskId } : {}), + ...metadata, + }, + }); + } catch (auditError) { + reflectionLog.warn( + `Failed to record reflection telemetry for ${agentId}: ${auditError instanceof Error ? auditError.message : String(auditError)}`, + ); + } + } + async buildReflectionContext(agentId: string): Promise { const [agentRecord, recentOutcomes, performanceSummaryRaw, latestReflection] = await Promise.all([ this.agentStore.getAgent(agentId), diff --git a/packages/engine/src/run-audit.ts b/packages/engine/src/run-audit.ts index 6da9a05fe1..57f90139c0 100644 --- a/packages/engine/src/run-audit.ts +++ b/packages/engine/src/run-audit.ts @@ -587,6 +587,47 @@ export type DatabaseMutationType = * ``` */ | "session:runtime-resolved" + /** + * FNXC:AgentReflectionTelemetry 2026-06-27-00:00: + * Agent performance reflection attempts must emit durable telemetry for every generated, skipped, or failed outcome. Metadata carries ids, trigger taxonomy, counts, and outcomes only; never persist reflection summaries, insight strings, suggested-improvement text, triggerDetail, or prompt text. + * + * Metadata shape for `reflection:generated`: + * ```ts + * { + * agentId: string; + * trigger: "manual" | "periodic" | "post-task" | "user-requested"; + * taskId?: string; + * reflectionId: string; + * tasksCompleted?: number; + * tasksFailed?: number; + * avgDurationMs?: number; + * commonErrorCount: number; + * insightCount: number; + * suggestedImprovementCount: number; + * } + * ``` + * Metadata shape for `reflection:skipped`: + * ```ts + * { + * agentId: string; + * trigger: "manual" | "periodic" | "post-task" | "user-requested"; + * taskId?: string; + * reason: "no-history"; + * } + * ``` + * Metadata shape for `reflection:failed`: + * ```ts + * { + * agentId: string; + * trigger: "manual" | "periodic" | "post-task" | "user-requested"; + * taskId?: string; + * errorClass: string; + * } + * ``` + */ + | "reflection:generated" + | "reflection:skipped" + | "reflection:failed" | "task:in-review-stall-deadlock-disposed" | "task:in-review-stall-terminal-provider-error" | "task:finalize-unproven-blocked"