feat(FN-1182): add agent reflection service and executor integration

- Extend project settings with reflection enablement, interval, and post-task trigger defaults
- Implement AgentReflectionService to gather agent/task history, generate structured AI reflections, and persist reflection metrics
- Add reflect_on_performance tool factory with optional focus area input and human-readable reflection output
- Wire reflection tool into TaskExecutor only when reflection is enabled and the task has an assigned agent
- Add comprehensive reflection service/tool tests and stabilize agent-store org tree expectation ordering
This commit is contained in:
gsxdsm
2026-04-08 05:55:36 -07:00
parent 7becb33972
commit a7eb95909a
7 changed files with 1086 additions and 2 deletions

View File

@@ -742,7 +742,7 @@ describe("AgentStore", () => {
}); });
const tree = await store.getOrgTree(); const tree = await store.getOrgTree();
expect(tree.map((node) => node.agent.id)).toEqual([root.id, orphan.id]); expect(tree.map((node) => node.agent.id).sort()).toEqual([root.id, orphan.id].sort());
}); });
}); });

View File

@@ -962,6 +962,12 @@ export interface ProjectSettings {
* When set, allows per-project customization of system prompts * When set, allows per-project customization of system prompts
* for different agent roles (executor, triage, reviewer, merger). */ * for different agent roles (executor, triage, reviewer, merger). */
agentPrompts?: AgentPromptsConfig; agentPrompts?: AgentPromptsConfig;
/** Enable/disable agent self-reflection workflows. Default: false. */
reflectionEnabled?: boolean;
/** How often periodic reflections occur in milliseconds. Default: 3_600_000 (1 hour). */
reflectionIntervalMs?: number;
/** When true, automatically trigger reflection after task completion. Default: true. */
reflectionAfterTask?: boolean;
} }
/** /**
@@ -1055,6 +1061,9 @@ export const DEFAULT_PROJECT_SETTINGS: ProjectSettings = {
runStepsInNewSessions: false, runStepsInNewSessions: false,
maxParallelSteps: 2, maxParallelSteps: 2,
agentPrompts: undefined, agentPrompts: undefined,
reflectionEnabled: false,
reflectionIntervalMs: 3_600_000,
reflectionAfterTask: true,
}; };
/** /**
@@ -1139,6 +1148,9 @@ export const PROJECT_SETTINGS_KEYS: ReadonlyArray<keyof ProjectSettings> = [
"runStepsInNewSessions", "runStepsInNewSessions",
"maxParallelSteps", "maxParallelSteps",
"agentPrompts", "agentPrompts",
"reflectionEnabled",
"reflectionIntervalMs",
"reflectionAfterTask",
] as const; ] as const;
export interface BoardConfig { export interface BoardConfig {

View File

@@ -0,0 +1,506 @@
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type {
Agent,
AgentHeartbeatRun,
AgentPerformanceSummary,
AgentReflection,
ReflectionMetrics,
Task,
} from "@fusion/core";
vi.mock("./pi.js", () => ({
createKbAgent: vi.fn(),
promptWithFallback: vi.fn(),
}));
import { createKbAgent, promptWithFallback } from "./pi.js";
import { AgentReflectionService } from "./agent-reflection.js";
import { createReflectOnPerformanceTool, reflectOnPerformanceParams } from "./agent-tools.js";
const mockedCreateKbAgent = vi.mocked(createKbAgent);
const mockedPromptWithFallback = vi.mocked(promptWithFallback);
function makeAgent(overrides: Partial<Agent> = {}): Agent {
return {
id: "agent-1",
name: "Execution Agent",
role: "executor",
state: "active",
createdAt: "2026-04-08T00:00:00.000Z",
updatedAt: "2026-04-08T00:00:00.000Z",
metadata: {},
...overrides,
};
}
function makeTask(overrides: Partial<Task> = {}): Task {
return {
id: "FN-001",
description: "Test task",
column: "done",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: "2026-04-08T00:00:00.000Z",
updatedAt: "2026-04-08T01:00:00.000Z",
assignedAgentId: "agent-1",
...overrides,
};
}
function makeSummary(overrides: Partial<AgentPerformanceSummary> = {}): AgentPerformanceSummary {
return {
agentId: "agent-1",
totalTasksCompleted: 3,
totalTasksFailed: 1,
avgDurationMs: 120_000,
successRate: 0.75,
commonErrors: ["timeout"],
strengths: ["Clear commit boundaries"],
weaknesses: ["Improve test coverage"],
recentReflectionCount: 2,
computedAt: "2026-04-08T01:00:00.000Z",
...overrides,
};
}
function makeReflection(overrides: Partial<AgentReflection> = {}): AgentReflection {
return {
id: "reflection-1",
agentId: "agent-1",
timestamp: "2026-04-08T01:00:00.000Z",
trigger: "manual",
metrics: {
tasksCompleted: 2,
tasksFailed: 1,
avgDurationMs: 60_000,
commonErrors: ["timeout"],
},
insights: ["Tends to stall on testing"],
suggestedImprovements: ["Run targeted tests earlier"],
summary: "Performance is generally solid with room to tighten test feedback loops.",
...overrides,
};
}
function makeRun(overrides: Partial<AgentHeartbeatRun> = {}): AgentHeartbeatRun {
return {
id: "run-1",
agentId: "agent-1",
startedAt: "2026-04-08T00:00:00.000Z",
endedAt: "2026-04-08T00:05:00.000Z",
status: "completed",
contextSnapshot: { taskId: "FN-001" },
...overrides,
};
}
function createMockDeps() {
const agentStore = {
getAgent: vi.fn().mockResolvedValue(makeAgent()),
getRecentRuns: vi.fn().mockResolvedValue([makeRun()]),
} as any;
const taskStore = {
listTasks: vi.fn().mockResolvedValue([makeTask()]),
} as any;
const reflectionStore = {
getPerformanceSummary: vi.fn().mockResolvedValue(makeSummary()),
getLatestReflection: vi.fn().mockResolvedValue(makeReflection()),
createReflection: vi.fn().mockImplementation(async ({
agentId,
trigger,
triggerDetail,
taskId,
metrics,
insights,
suggestedImprovements,
summary,
}: {
agentId: string;
trigger: AgentReflection["trigger"];
triggerDetail?: string;
taskId?: string;
metrics: ReflectionMetrics;
insights: string[];
suggestedImprovements: string[];
summary: string;
}) => makeReflection({
agentId,
trigger,
triggerDetail,
taskId,
metrics,
insights,
suggestedImprovements,
summary,
})),
} as any;
return { agentStore, taskStore, reflectionStore };
}
function createMockSession() {
return {
state: {},
dispose: vi.fn(),
} as any;
}
describe("AgentReflectionService", () => {
let tempRoot: string;
beforeEach(async () => {
vi.clearAllMocks();
tempRoot = await mkdtemp(join(tmpdir(), "agent-reflection-test-"));
});
describe("buildReflectionContext", () => {
it("returns context with recent outcomes, summary, and latest reflection", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const context = await service.buildReflectionContext("agent-1");
expect(context.agent.id).toBe("agent-1");
expect(context.recentOutcomes).toHaveLength(1);
expect(context.performanceSummary?.successRate).toBe(0.75);
expect(context.latestReflection?.id).toBe("reflection-1");
});
it("returns empty recentOutcomes when no task history exists", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
taskStore.listTasks.mockResolvedValue([]);
agentStore.getRecentRuns.mockResolvedValue([]);
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const context = await service.buildReflectionContext("agent-1");
expect(context.recentOutcomes).toEqual([]);
});
it("reads instructions from instructionsText when provided", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
agentStore.getAgent.mockResolvedValue(makeAgent({ instructionsText: "Always run tests before committing." }));
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const context = await service.buildReflectionContext("agent-1");
expect(context.instructions).toContain("Always run tests before committing.");
});
it("reads instructions from file when instructionsPath is set", async () => {
const instructionsDir = join(tempRoot, "agents");
const instructionsPath = join(instructionsDir, "executor.md");
await mkdir(instructionsDir, { recursive: true });
await writeFile(instructionsPath, "Prefer smaller commits with clear messages.", { encoding: "utf-8" });
const { agentStore, taskStore, reflectionStore } = createMockDeps();
agentStore.getAgent.mockResolvedValue(makeAgent({ instructionsPath: "agents/executor.md" }));
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const context = await service.buildReflectionContext("agent-1");
expect(context.instructions).toContain("Prefer smaller commits with clear messages.");
});
it("handles missing agent gracefully", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
agentStore.getAgent.mockResolvedValue(null);
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const context = await service.buildReflectionContext("agent-missing");
expect(context.agent.id).toBe("agent-missing");
expect(context.agent.name).toContain("Unknown Agent");
});
it("includes latest reflection when available", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
reflectionStore.getLatestReflection.mockResolvedValue(makeReflection({ id: "reflection-latest" }));
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const context = await service.buildReflectionContext("agent-1");
expect(context.latestReflection?.id).toBe("reflection-latest");
});
});
describe("getRecentTaskOutcomes", () => {
it("returns completed outcomes for done and in-review tasks", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
taskStore.listTasks.mockResolvedValue([
makeTask({ id: "FN-001", column: "done", assignedAgentId: "agent-1" }),
makeTask({ id: "FN-002", column: "in-review", assignedAgentId: "agent-1" }),
]);
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const outcomes = await service.getRecentTaskOutcomes("agent-1", 10);
expect(outcomes.map((outcome) => outcome.outcome)).toEqual(["completed", "completed"]);
});
it("returns failed outcomes when status includes failed", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
taskStore.listTasks.mockResolvedValue([
makeTask({ id: "FN-FAIL", column: "todo", status: "failed", assignedAgentId: "agent-1" }),
]);
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const outcomes = await service.getRecentTaskOutcomes("agent-1", 10);
expect(outcomes).toHaveLength(1);
expect(outcomes[0]?.outcome).toBe("failed");
});
it("returns stuck outcomes when task was killed by stuck detector", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
taskStore.listTasks.mockResolvedValue([
makeTask({
id: "FN-STUCK",
column: "todo",
status: "stuck-killed",
assignedAgentId: "agent-1",
log: [{ timestamp: "2026-04-08T00:00:00.000Z", action: "Task terminated due to stuck agent session" }],
}),
]);
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const outcomes = await service.getRecentTaskOutcomes("agent-1", 10);
expect(outcomes).toHaveLength(1);
expect(outcomes[0]?.outcome).toBe("stuck");
});
it("respects limit parameter", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
taskStore.listTasks.mockResolvedValue([
makeTask({ id: "FN-001", assignedAgentId: "agent-1" }),
makeTask({ id: "FN-002", assignedAgentId: "agent-1" }),
makeTask({ id: "FN-003", assignedAgentId: "agent-1" }),
]);
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const outcomes = await service.getRecentTaskOutcomes("agent-1", 2);
expect(outcomes).toHaveLength(2);
});
it("returns empty array when agent has no tasks", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
taskStore.listTasks.mockResolvedValue([makeTask({ assignedAgentId: "agent-2" })]);
agentStore.getRecentRuns.mockResolvedValue([]);
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const outcomes = await service.getRecentTaskOutcomes("agent-1", 10);
expect(outcomes).toEqual([]);
});
});
describe("extractErrorPatterns", () => {
it("returns common errors from performance summary", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
reflectionStore.getPerformanceSummary.mockResolvedValue(makeSummary({ commonErrors: ["timeout", "merge conflict"] }));
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const errors = await service.extractErrorPatterns("agent-1");
expect(errors).toEqual(["timeout", "merge conflict"]);
});
it("returns empty array when no summary exists", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
reflectionStore.getPerformanceSummary.mockResolvedValue(makeSummary({
totalTasksCompleted: 0,
totalTasksFailed: 0,
avgDurationMs: 0,
successRate: 0,
commonErrors: [],
strengths: [],
weaknesses: [],
recentReflectionCount: 0,
}));
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const errors = await service.extractErrorPatterns("agent-1");
expect(errors).toEqual([]);
});
it("returns empty array when summary has no errors", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
reflectionStore.getPerformanceSummary.mockResolvedValue(makeSummary({ commonErrors: [] }));
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const errors = await service.extractErrorPatterns("agent-1");
expect(errors).toEqual([]);
});
});
describe("generateReflection", () => {
it("creates reflection when data is available", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
const session = createMockSession();
mockedCreateKbAgent.mockImplementation(async (options: any) => {
options.onText?.(JSON.stringify({
insights: ["Strong execution on scoped changes"],
suggestedImprovements: ["Run tests earlier in the cycle"],
summary: "Execution quality is strong with room to tighten feedback loops.",
}));
return { session };
});
mockedPromptWithFallback.mockResolvedValue(undefined);
const service = new AgentReflectionService({
agentStore,
taskStore,
reflectionStore,
rootDir: tempRoot,
});
const reflection = await service.generateReflection("agent-1", "manual", {
taskId: "FN-001",
triggerDetail: "manual check",
});
expect(reflection).not.toBeNull();
expect(reflectionStore.createReflection).toHaveBeenCalledTimes(1);
expect(reflectionStore.createReflection).toHaveBeenCalledWith(expect.objectContaining({
agentId: "agent-1",
trigger: "manual",
triggerDetail: "manual check",
taskId: "FN-001",
}));
expect(mockedCreateKbAgent).toHaveBeenCalledWith(expect.objectContaining({
tools: "readonly",
}));
});
it("returns null when no meaningful data exists", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
taskStore.listTasks.mockResolvedValue([]);
agentStore.getRecentRuns.mockResolvedValue([]);
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const reflection = await service.generateReflection("agent-1", "manual");
expect(reflection).toBeNull();
expect(mockedCreateKbAgent).not.toHaveBeenCalled();
expect(reflectionStore.createReflection).not.toHaveBeenCalled();
});
it("returns null on AI session failure", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
mockedCreateKbAgent.mockRejectedValue(new Error("AI unavailable"));
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
const reflection = await service.generateReflection("agent-1", "manual");
expect(reflection).toBeNull();
expect(reflectionStore.createReflection).not.toHaveBeenCalled();
});
it("persists reflection via reflectionStore.createReflection", async () => {
const { agentStore, taskStore, reflectionStore } = createMockDeps();
const session = createMockSession();
mockedCreateKbAgent.mockImplementation(async (options: any) => {
options.onText?.(JSON.stringify({
insights: ["Insight A"],
suggestedImprovements: ["Improve B"],
summary: "Summary C",
}));
return { session };
});
mockedPromptWithFallback.mockResolvedValue(undefined);
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
await service.generateReflection("agent-1", "post-task", {
taskId: "FN-777",
triggerDetail: "post-task reflection",
});
expect(reflectionStore.createReflection).toHaveBeenCalledWith(expect.objectContaining({
trigger: "post-task",
taskId: "FN-777",
triggerDetail: "post-task reflection",
}));
});
});
describe("reflect_on_performance tool", () => {
it("returns formatted text when reflection succeeds", async () => {
const reflectionService = {
generateReflection: vi.fn().mockResolvedValue(makeReflection({
summary: "Reflection summary",
insights: ["Insight 1"],
suggestedImprovements: ["Improve 1"],
})),
} as any;
const tool = createReflectOnPerformanceTool(reflectionService, "agent-1");
const result = await (tool.execute as any)("tool-1", {}, {}, {}, undefined);
const content = result.content[0];
expect(content?.type).toBe("text");
if (!content || content.type !== "text") {
throw new Error("Expected text content");
}
expect(content.text).toContain("Summary: Reflection summary");
expect(content.text).toContain("Insights:");
expect(content.text).toContain("Suggested Improvements:");
});
it("returns no-data message when reflection returns null", async () => {
const reflectionService = {
generateReflection: vi.fn().mockResolvedValue(null),
} as any;
const tool = createReflectOnPerformanceTool(reflectionService, "agent-1");
const result = await (tool.execute as any)("tool-1", {}, {}, {}, undefined);
const content = result.content[0];
expect(content?.type).toBe("text");
if (!content || content.type !== "text") {
throw new Error("Expected text content");
}
expect(content.text).toBe("No reflection data available — not enough history yet.");
});
it("passes focus_area as triggerDetail", async () => {
const reflectionService = {
generateReflection: vi.fn().mockResolvedValue(makeReflection()),
} as any;
const tool = createReflectOnPerformanceTool(reflectionService, "agent-1");
await (tool.execute as any)("tool-1", { focus_area: "testing" }, {}, {}, undefined);
expect(reflectionService.generateReflection).toHaveBeenCalledWith(
"agent-1",
"manual",
{ triggerDetail: "Agent-initiated reflection focused on: testing" },
);
});
it("parameter schema accepts optional focus_area", () => {
const required = (reflectOnPerformanceParams as { required?: string[] }).required;
expect(required?.includes("focus_area") ?? false).toBe(false);
});
});
afterEach(async () => {
if (tempRoot) {
await rm(tempRoot, { recursive: true, force: true });
}
});
});

View File

@@ -0,0 +1,495 @@
import { readFile } from "node:fs/promises";
import { isAbsolute, resolve } from "node:path";
import type {
Agent,
AgentHeartbeatRun,
AgentPerformanceSummary,
AgentReflection,
AgentStore,
ReflectionMetrics,
ReflectionStore,
ReflectionTrigger,
Task,
TaskStore,
} from "@fusion/core";
import { createLogger } from "./logger.js";
import { createKbAgent, promptWithFallback } from "./pi.js";
const reflectionLog = createLogger("reflection");
const REFLECTION_SYSTEM_PROMPT = `You are an autonomous performance analyst reviewing an AI agent's recent execution history.
Your task is to identify concrete, actionable improvements from the provided metrics and outcomes.
Return STRICT JSON with this exact shape:
{
"insights": ["short, specific observation"],
"suggestedImprovements": ["actionable improvement"],
"summary": "2-4 sentence synthesis"
}
Rules:
- Output valid JSON only (no markdown fences, no prose outside JSON).
- Keep insights specific to the provided evidence.
- Prefer improvements that can be applied in the agent's next run.
- Avoid generic advice unless strongly justified by data.`;
const DEFAULT_OUTCOME_LIMIT = 20;
interface ReflectionContext {
agent: Agent;
recentOutcomes: TaskOutcome[];
performanceSummary: AgentPerformanceSummary | null;
latestReflection: AgentReflection | null;
instructions?: string;
}
interface TaskOutcome {
taskId: string;
outcome: "completed" | "failed" | "stuck";
durationMs?: number;
completedAt?: string;
}
interface ReflectionPayload {
insights: string[];
suggestedImprovements: string[];
summary: string;
}
export interface AgentReflectionServiceOptions {
agentStore: AgentStore;
taskStore: TaskStore;
reflectionStore: ReflectionStore;
rootDir: string;
modelProvider?: string;
modelId?: string;
}
export class AgentReflectionService {
private readonly agentStore: AgentStore;
private readonly taskStore: TaskStore;
private readonly reflectionStore: ReflectionStore;
private readonly rootDir: string;
private readonly modelProvider?: string;
private readonly modelId?: string;
constructor(options: AgentReflectionServiceOptions) {
this.agentStore = options.agentStore;
this.taskStore = options.taskStore;
this.reflectionStore = options.reflectionStore;
this.rootDir = options.rootDir;
this.modelProvider = options.modelProvider;
this.modelId = options.modelId;
}
async generateReflection(
agentId: string,
trigger: ReflectionTrigger,
options: { taskId?: string; triggerDetail?: string } = {},
): Promise<AgentReflection | null> {
try {
const context = await this.buildReflectionContext(agentId);
const recentRuns = await this.agentStore.getRecentRuns(agentId, DEFAULT_OUTCOME_LIMIT);
if (context.recentOutcomes.length === 0 && recentRuns.length === 0) {
reflectionLog.log(`Skipping reflection for ${agentId}: no recent tasks or heartbeat runs`);
return null;
}
let responseText = "";
const { session } = await createKbAgent({
cwd: this.rootDir,
systemPrompt: REFLECTION_SYSTEM_PROMPT,
tools: "readonly",
defaultProvider: this.modelProvider,
defaultModelId: this.modelId,
onText: (delta: string) => {
responseText += delta;
},
});
try {
await promptWithFallback(session, this.buildReflectionPrompt(context, options.triggerDetail));
if (session.state?.error) {
throw new Error(session.state.error);
}
} finally {
try {
session.dispose();
} catch {
// best-effort cleanup
}
}
const parsed = this.parseReflectionResponse(responseText);
const metrics = this.buildReflectionMetrics(context.recentOutcomes, context.performanceSummary, recentRuns);
return await this.reflectionStore.createReflection({
agentId,
trigger,
triggerDetail: options.triggerDetail,
taskId: options.taskId,
metrics,
insights: parsed.insights,
suggestedImprovements: parsed.suggestedImprovements,
summary: parsed.summary,
});
} catch (error) {
reflectionLog.error(`Failed to generate reflection for ${agentId}: ${(error as Error).message}`);
return null;
}
}
async buildReflectionContext(agentId: string): Promise<ReflectionContext> {
const [agentRecord, recentOutcomes, performanceSummaryRaw, latestReflection] = await Promise.all([
this.agentStore.getAgent(agentId),
this.getRecentTaskOutcomes(agentId, DEFAULT_OUTCOME_LIMIT),
this.reflectionStore.getPerformanceSummary(agentId),
this.reflectionStore.getLatestReflection(agentId),
]);
const agent = agentRecord ?? this.createUnknownAgent(agentId);
if (!agentRecord) {
reflectionLog.warn(`Agent ${agentId} not found while building reflection context`);
}
const instructions = await this.resolveInstructions(agentRecord);
const performanceSummary = this.isMeaningfulSummary(performanceSummaryRaw)
? performanceSummaryRaw
: null;
return {
agent,
recentOutcomes,
performanceSummary,
latestReflection,
instructions,
};
}
async getRecentTaskOutcomes(agentId: string, limit = DEFAULT_OUTCOME_LIMIT): Promise<TaskOutcome[]> {
const effectiveLimit = Math.max(1, limit);
const [tasks, recentRuns, agent] = await Promise.all([
this.taskStore.listTasks(),
this.agentStore.getRecentRuns(agentId, effectiveLimit * 4),
this.agentStore.getAgent(agentId),
]);
const recentTaskIdsFromRuns = this.extractTaskIdsFromRuns(recentRuns);
const sortedByRecency = [...tasks].sort((a, b) => this.getTaskTimestampMs(b) - this.getTaskTimestampMs(a));
const tasksToScan = sortedByRecency.slice(0, effectiveLimit * 2);
const agentMentions = [agentId, agent?.name].filter((value): value is string => Boolean(value?.trim()));
const outcomes: TaskOutcome[] = [];
for (const task of tasksToScan) {
if (!this.isTaskLinkedToAgent(task, agentId, recentTaskIdsFromRuns, agentMentions)) {
continue;
}
const outcome = this.classifyOutcome(task);
if (!outcome) {
continue;
}
const durationMs = this.calculateDurationMs(task);
const completedAt = this.resolveCompletedAt(task);
outcomes.push({
taskId: task.id,
outcome,
durationMs,
completedAt,
});
if (outcomes.length >= effectiveLimit) {
break;
}
}
return outcomes;
}
async extractErrorPatterns(agentId: string): Promise<string[]> {
const summary = await this.reflectionStore.getPerformanceSummary(agentId);
if (!this.isMeaningfulSummary(summary)) {
return [];
}
return summary.commonErrors ?? [];
}
private buildReflectionPrompt(context: ReflectionContext, triggerDetail?: string): string {
const summary = {
agent: {
id: context.agent.id,
name: context.agent.name,
role: context.agent.role,
state: context.agent.state,
},
triggerDetail,
recentOutcomes: context.recentOutcomes,
performanceSummary: context.performanceSummary,
latestReflection: context.latestReflection
? {
timestamp: context.latestReflection.timestamp,
summary: context.latestReflection.summary,
insights: context.latestReflection.insights,
suggestedImprovements: context.latestReflection.suggestedImprovements,
}
: null,
instructions: context.instructions,
};
return [
"Analyze the following agent performance context and propose concrete improvements.",
"Respond with strict JSON matching the required schema.",
JSON.stringify(summary, null, 2),
].join("\n\n");
}
private parseReflectionResponse(rawResponse: string): ReflectionPayload {
const candidate = this.extractJsonCandidate(rawResponse);
let parsed: unknown;
try {
parsed = JSON.parse(candidate);
} catch {
reflectionLog.warn("Reflection response was not valid JSON; using fallback reflection payload");
return {
insights: ["Insufficient structured output from reflection model."],
suggestedImprovements: ["Retry reflection with clearer historical context."],
summary: "The reflection model did not return valid structured JSON.",
};
}
const record = parsed as Partial<ReflectionPayload>;
const insights = Array.isArray(record.insights)
? record.insights.map((value) => String(value).trim()).filter(Boolean)
: [];
const suggestedImprovements = Array.isArray(record.suggestedImprovements)
? record.suggestedImprovements.map((value) => String(value).trim()).filter(Boolean)
: [];
const summary = typeof record.summary === "string" && record.summary.trim().length > 0
? record.summary.trim()
: "No summary was provided by the reflection model.";
return {
insights: insights.length > 0 ? insights : ["No specific insights were identified."],
suggestedImprovements: suggestedImprovements.length > 0
? suggestedImprovements
: ["No concrete improvements were suggested."],
summary,
};
}
private extractJsonCandidate(rawResponse: string): string {
const trimmed = rawResponse.trim();
if (!trimmed) {
return "{}";
}
if (trimmed.startsWith("```") && trimmed.endsWith("```")) {
const withoutFences = trimmed
.replace(/^```(?:json)?\s*/i, "")
.replace(/\s*```$/, "")
.trim();
return withoutFences || "{}";
}
const firstBrace = trimmed.indexOf("{");
const lastBrace = trimmed.lastIndexOf("}");
if (firstBrace >= 0 && lastBrace > firstBrace) {
return trimmed.slice(firstBrace, lastBrace + 1);
}
return trimmed;
}
private buildReflectionMetrics(
outcomes: TaskOutcome[],
performanceSummary: AgentPerformanceSummary | null,
recentRuns: AgentHeartbeatRun[],
): ReflectionMetrics {
const tasksCompleted = outcomes.filter((outcome) => outcome.outcome === "completed").length;
const tasksFailed = outcomes.filter((outcome) => outcome.outcome !== "completed").length;
const durations = outcomes
.map((outcome) => outcome.durationMs)
.filter((duration): duration is number => typeof duration === "number" && Number.isFinite(duration));
const avgDurationMs = durations.length > 0
? Math.round(durations.reduce((sum, duration) => sum + duration, 0) / durations.length)
: performanceSummary?.avgDurationMs ?? 0;
const runErrors = recentRuns
.map((run) => this.extractRunError(run))
.filter((value): value is string => Boolean(value));
const mergedErrors = [
...(performanceSummary?.commonErrors ?? []),
...outcomes.filter((outcome) => outcome.outcome !== "completed").map((outcome) => `${outcome.outcome}: ${outcome.taskId}`),
...runErrors,
];
const commonErrors = Array.from(new Set(mergedErrors.map((error) => error.trim()).filter(Boolean))).slice(0, 10);
return {
tasksCompleted,
tasksFailed,
avgDurationMs,
commonErrors,
};
}
private extractRunError(run: AgentHeartbeatRun): string | null {
if (typeof run.stderrExcerpt === "string" && run.stderrExcerpt.trim()) {
return run.stderrExcerpt.trim().split("\n")[0] ?? null;
}
const resultError = run.resultJson && typeof run.resultJson.error === "string"
? run.resultJson.error.trim()
: "";
if (resultError) {
return resultError;
}
return null;
}
private extractTaskIdsFromRuns(runs: AgentHeartbeatRun[]): Set<string> {
const ids = new Set<string>();
for (const run of runs) {
const taskId = run.contextSnapshot?.taskId;
if (typeof taskId === "string" && taskId.trim()) {
ids.add(taskId.trim());
}
}
return ids;
}
private classifyOutcome(task: Task): TaskOutcome["outcome"] | null {
const normalizedStatus = task.status?.toLowerCase() ?? "";
const hasStuckSignal =
normalizedStatus.includes("stuck")
|| task.log.some((entry) => {
const action = entry.action.toLowerCase();
return action.includes("stuck") || action.includes("terminated due to stuck");
});
if (hasStuckSignal) {
return "stuck";
}
if (normalizedStatus.includes("failed")) {
return "failed";
}
if (task.column === "done" || task.column === "in-review") {
return "completed";
}
return null;
}
private isTaskLinkedToAgent(
task: Task,
agentId: string,
recentTaskIdsFromRuns: Set<string>,
agentMentions: string[],
): boolean {
if (task.assignedAgentId === agentId) {
return true;
}
if (recentTaskIdsFromRuns.has(task.id)) {
return true;
}
if (agentMentions.length === 0) {
return false;
}
return task.log.some((entry) => {
const content = `${entry.action} ${entry.outcome ?? ""}`.toLowerCase();
return agentMentions.some((mention) => content.includes(mention.toLowerCase()));
});
}
private calculateDurationMs(task: Task): number | undefined {
const startedAtMs = Date.parse(task.createdAt);
const completedAtIso = this.resolveCompletedAt(task);
const completedAtMs = completedAtIso ? Date.parse(completedAtIso) : Date.parse(task.updatedAt);
if (!Number.isFinite(startedAtMs) || !Number.isFinite(completedAtMs) || completedAtMs <= startedAtMs) {
return undefined;
}
return completedAtMs - startedAtMs;
}
private resolveCompletedAt(task: Task): string | undefined {
return task.columnMovedAt ?? task.updatedAt;
}
private async resolveInstructions(agent: Agent | null): Promise<string | undefined> {
if (!agent) {
return undefined;
}
const pieces: string[] = [];
if (agent.instructionsText?.trim()) {
pieces.push(agent.instructionsText.trim());
}
if (agent.instructionsPath?.trim()) {
const resolvedPath = isAbsolute(agent.instructionsPath)
? agent.instructionsPath
: resolve(this.rootDir, agent.instructionsPath);
try {
const content = await readFile(resolvedPath, "utf-8");
if (content.trim()) {
pieces.push(content.trim());
}
} catch (error) {
reflectionLog.warn(
`Unable to read instructions file for ${agent.id} at ${agent.instructionsPath}: ${(error as Error).message}`,
);
}
}
return pieces.length > 0 ? pieces.join("\n\n") : undefined;
}
private isMeaningfulSummary(summary: AgentPerformanceSummary): boolean {
return summary.recentReflectionCount > 0
|| summary.totalTasksCompleted > 0
|| summary.totalTasksFailed > 0
|| summary.commonErrors.length > 0
|| summary.strengths.length > 0
|| summary.weaknesses.length > 0;
}
private createUnknownAgent(agentId: string): Agent {
const now = new Date().toISOString();
return {
id: agentId,
name: `Unknown Agent (${agentId})`,
role: "custom",
state: "idle",
createdAt: now,
updatedAt: now,
metadata: {},
};
}
private getTaskTimestampMs(task: Task): number {
const candidate = task.columnMovedAt ?? task.updatedAt ?? task.createdAt;
const timestamp = Date.parse(candidate);
return Number.isFinite(timestamp) ? timestamp : 0;
}
}

View File

@@ -10,6 +10,7 @@
import type { TaskStore } from "@fusion/core"; import type { TaskStore } from "@fusion/core";
import type { ToolDefinition } from "@mariozechner/pi-coding-agent"; import type { ToolDefinition } from "@mariozechner/pi-coding-agent";
import { Type, type Static } from "@mariozechner/pi-ai"; import { Type, type Static } from "@mariozechner/pi-ai";
import type { AgentReflectionService } from "./agent-reflection.js";
// ── Tool parameter schemas (canonical definitions) ──────────────────────── // ── Tool parameter schemas (canonical definitions) ────────────────────────
@@ -25,6 +26,12 @@ export const taskLogParams = Type.Object({
outcome: Type.Optional(Type.String({ description: "Result or consequence (optional)" })), outcome: Type.Optional(Type.String({ description: "Result or consequence (optional)" })),
}); });
export const reflectOnPerformanceParams = Type.Object({
focus_area: Type.Optional(
Type.String({ description: "Optional focus area for reflection (e.g., 'code quality', 'speed', 'testing')" }),
),
});
// ── Tool factory functions ──────────────────────────────────────────────── // ── Tool factory functions ────────────────────────────────────────────────
/** /**
@@ -85,3 +92,51 @@ export function createTaskLogTool(store: TaskStore, taskId: string): ToolDefinit
}, },
}; };
} }
/**
* Create a `reflect_on_performance` tool that asks the reflection service to
* analyze recent agent performance and return actionable insights.
*/
export function createReflectOnPerformanceTool(
reflectionService: AgentReflectionService,
agentId: string,
): ToolDefinition {
return {
name: "reflect_on_performance",
label: "Reflect on Performance",
description:
'Review your past task performance and generate insights for improvement. Optionally focus on a specific area like "code quality", "speed", or "testing".',
parameters: reflectOnPerformanceParams,
execute: async (_id: string, params: Static<typeof reflectOnPerformanceParams>) => {
const triggerDetail = params.focus_area
? `Agent-initiated reflection focused on: ${params.focus_area}`
: "Agent-initiated reflection";
const reflection = await reflectionService.generateReflection(agentId, "manual", {
triggerDetail,
});
if (!reflection) {
return {
content: [{ type: "text" as const, text: "No reflection data available — not enough history yet." }],
details: {},
};
}
const formattedText = [
`Summary: ${reflection.summary}`,
"",
"Insights:",
...reflection.insights.map((insight, index) => `${index + 1}. ${insight}`),
"",
"Suggested Improvements:",
...reflection.suggestedImprovements.map((improvement, index) => `${index + 1}. ${improvement}`),
].join("\n");
return {
content: [{ type: "text" as const, text: formattedText }],
details: {},
};
},
};
}

View File

@@ -23,7 +23,14 @@ import type { StuckTaskDetector, StuckTaskEvent } from "./stuck-task-detector.js
import { isContextLimitError } from "./context-limit-detector.js"; import { isContextLimitError } from "./context-limit-detector.js";
import { StepSessionExecutor, type StepSessionExecutorOptions, type StepResult } from "./step-session-executor.js"; import { StepSessionExecutor, type StepSessionExecutorOptions, type StepResult } from "./step-session-executor.js";
import { resolveAgentInstructions, buildSystemPromptWithInstructions } from "./agent-instructions.js"; import { resolveAgentInstructions, buildSystemPromptWithInstructions } from "./agent-instructions.js";
import { createTaskCreateTool as sharedCreateTaskCreateTool, createTaskLogTool as sharedCreateTaskLogTool, taskCreateParams, taskLogParams } from "./agent-tools.js"; import type { AgentReflectionService } from "./agent-reflection.js";
import {
createReflectOnPerformanceTool,
createTaskCreateTool as sharedCreateTaskCreateTool,
createTaskLogTool as sharedCreateTaskLogTool,
taskCreateParams,
taskLogParams,
} from "./agent-tools.js";
// Re-export for backward compatibility (tests import from executor.ts) // Re-export for backward compatibility (tests import from executor.ts)
export { summarizeToolArgs } from "./agent-logger.js"; export { summarizeToolArgs } from "./agent-logger.js";
@@ -225,6 +232,8 @@ export interface TaskExecutorOptions {
stuckTaskDetector?: StuckTaskDetector; stuckTaskDetector?: StuckTaskDetector;
/** AgentStore for tracking spawned child agents. If not provided, spawning is disabled. */ /** AgentStore for tracking spawned child agents. If not provided, spawning is disabled. */
agentStore?: import("@fusion/core").AgentStore; agentStore?: import("@fusion/core").AgentStore;
/** Reflection service used to generate self-reflection insights for agents. */
reflectionService?: AgentReflectionService;
missionStore?: MissionStore; missionStore?: MissionStore;
onSliceComplete?: (slice: Slice) => void; onSliceComplete?: (slice: Slice) => void;
onStart?: (task: Task, worktreePath: string) => void; onStart?: (task: Task, worktreePath: string) => void;
@@ -1012,6 +1021,10 @@ export class TaskExecutor {
const stepCheckpoints = new Map<number, string>(); const stepCheckpoints = new Map<number, string>();
const stuckDetector = this.options.stuckTaskDetector; const stuckDetector = this.options.stuckTaskDetector;
const assignedAgentId = detail.assignedAgentId?.trim();
const reflectionTools = this.options.reflectionService && settings.reflectionEnabled && assignedAgentId
? [createReflectOnPerformanceTool(this.options.reflectionService, assignedAgentId)]
: [];
const customTools = [ const customTools = [
this.createTaskUpdateTool(task.id, codeReviewVerdicts, sessionRef, stepCheckpoints, stuckDetector), this.createTaskUpdateTool(task.id, codeReviewVerdicts, sessionRef, stepCheckpoints, stuckDetector),
@@ -1021,6 +1034,8 @@ export class TaskExecutor {
this.createTaskDoneTool(task.id, () => { taskDone = true; }), this.createTaskDoneTool(task.id, () => { taskDone = true; }),
this.createReviewStepTool(task.id, worktreePath, detail.prompt, codeReviewVerdicts, sessionRef, stepCheckpoints, detail, stuckDetector), this.createReviewStepTool(task.id, worktreePath, detail.prompt, codeReviewVerdicts, sessionRef, stepCheckpoints, detail, stuckDetector),
this.createSpawnAgentTool(task.id, worktreePath, settings), this.createSpawnAgentTool(task.id, worktreePath, settings),
// Conditionally add agent self-reflection when enabled and task has an assigned agent.
...reflectionTools,
]; ];
const agentLogger = new AgentLogger({ const agentLogger = new AgentLogger({

View File

@@ -7,6 +7,7 @@ export { MissionAutopilot, type MissionAutopilotOptions } from "./mission-autopi
export { aiMergeTask, type MergerOptions } from "./merger.js"; export { aiMergeTask, type MergerOptions } from "./merger.js";
export { reviewStep, type ReviewType, type ReviewVerdict, type ReviewResult, type ReviewOptions } from "./reviewer.js"; export { reviewStep, type ReviewType, type ReviewVerdict, type ReviewResult, type ReviewOptions } from "./reviewer.js";
export { createKbAgent, type AgentOptions, type AgentResult } from "./pi.js"; export { createKbAgent, type AgentOptions, type AgentResult } from "./pi.js";
export { AgentReflectionService, type AgentReflectionServiceOptions } from "./agent-reflection.js";
export { WorktreePool, scanIdleWorktrees, cleanupOrphanedWorktrees } from "./worktree-pool.js"; export { WorktreePool, scanIdleWorktrees, cleanupOrphanedWorktrees } from "./worktree-pool.js";
export { createLogger, type Logger } from "./logger.js"; export { createLogger, type Logger } from "./logger.js";
export { isUsageLimitError, UsageLimitPauser } from "./usage-limit-detector.js"; export { isUsageLimitError, UsageLimitPauser } from "./usage-limit-detector.js";