feat(FN-1182): add agent reflection service and executor integration
- Extend project settings with reflection enablement, interval, and post-task trigger defaults - Implement AgentReflectionService to gather agent/task history, generate structured AI reflections, and persist reflection metrics - Add reflect_on_performance tool factory with optional focus area input and human-readable reflection output - Wire reflection tool into TaskExecutor only when reflection is enabled and the task has an assigned agent - Add comprehensive reflection service/tool tests and stabilize agent-store org tree expectation ordering
This commit is contained in:
@@ -742,7 +742,7 @@ describe("AgentStore", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const tree = await store.getOrgTree();
|
const tree = await store.getOrgTree();
|
||||||
expect(tree.map((node) => node.agent.id)).toEqual([root.id, orphan.id]);
|
expect(tree.map((node) => node.agent.id).sort()).toEqual([root.id, orphan.id].sort());
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -962,6 +962,12 @@ export interface ProjectSettings {
|
|||||||
* When set, allows per-project customization of system prompts
|
* When set, allows per-project customization of system prompts
|
||||||
* for different agent roles (executor, triage, reviewer, merger). */
|
* for different agent roles (executor, triage, reviewer, merger). */
|
||||||
agentPrompts?: AgentPromptsConfig;
|
agentPrompts?: AgentPromptsConfig;
|
||||||
|
/** Enable/disable agent self-reflection workflows. Default: false. */
|
||||||
|
reflectionEnabled?: boolean;
|
||||||
|
/** How often periodic reflections occur in milliseconds. Default: 3_600_000 (1 hour). */
|
||||||
|
reflectionIntervalMs?: number;
|
||||||
|
/** When true, automatically trigger reflection after task completion. Default: true. */
|
||||||
|
reflectionAfterTask?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1055,6 +1061,9 @@ export const DEFAULT_PROJECT_SETTINGS: ProjectSettings = {
|
|||||||
runStepsInNewSessions: false,
|
runStepsInNewSessions: false,
|
||||||
maxParallelSteps: 2,
|
maxParallelSteps: 2,
|
||||||
agentPrompts: undefined,
|
agentPrompts: undefined,
|
||||||
|
reflectionEnabled: false,
|
||||||
|
reflectionIntervalMs: 3_600_000,
|
||||||
|
reflectionAfterTask: true,
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1139,6 +1148,9 @@ export const PROJECT_SETTINGS_KEYS: ReadonlyArray<keyof ProjectSettings> = [
|
|||||||
"runStepsInNewSessions",
|
"runStepsInNewSessions",
|
||||||
"maxParallelSteps",
|
"maxParallelSteps",
|
||||||
"agentPrompts",
|
"agentPrompts",
|
||||||
|
"reflectionEnabled",
|
||||||
|
"reflectionIntervalMs",
|
||||||
|
"reflectionAfterTask",
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
export interface BoardConfig {
|
export interface BoardConfig {
|
||||||
|
|||||||
506
packages/engine/src/agent-reflection.test.ts
Normal file
506
packages/engine/src/agent-reflection.test.ts
Normal file
@@ -0,0 +1,506 @@
|
|||||||
|
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import type {
|
||||||
|
Agent,
|
||||||
|
AgentHeartbeatRun,
|
||||||
|
AgentPerformanceSummary,
|
||||||
|
AgentReflection,
|
||||||
|
ReflectionMetrics,
|
||||||
|
Task,
|
||||||
|
} from "@fusion/core";
|
||||||
|
|
||||||
|
vi.mock("./pi.js", () => ({
|
||||||
|
createKbAgent: vi.fn(),
|
||||||
|
promptWithFallback: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { createKbAgent, promptWithFallback } from "./pi.js";
|
||||||
|
import { AgentReflectionService } from "./agent-reflection.js";
|
||||||
|
import { createReflectOnPerformanceTool, reflectOnPerformanceParams } from "./agent-tools.js";
|
||||||
|
|
||||||
|
const mockedCreateKbAgent = vi.mocked(createKbAgent);
|
||||||
|
const mockedPromptWithFallback = vi.mocked(promptWithFallback);
|
||||||
|
|
||||||
|
function makeAgent(overrides: Partial<Agent> = {}): Agent {
|
||||||
|
return {
|
||||||
|
id: "agent-1",
|
||||||
|
name: "Execution Agent",
|
||||||
|
role: "executor",
|
||||||
|
state: "active",
|
||||||
|
createdAt: "2026-04-08T00:00:00.000Z",
|
||||||
|
updatedAt: "2026-04-08T00:00:00.000Z",
|
||||||
|
metadata: {},
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeTask(overrides: Partial<Task> = {}): Task {
|
||||||
|
return {
|
||||||
|
id: "FN-001",
|
||||||
|
description: "Test task",
|
||||||
|
column: "done",
|
||||||
|
dependencies: [],
|
||||||
|
steps: [],
|
||||||
|
currentStep: 0,
|
||||||
|
log: [],
|
||||||
|
createdAt: "2026-04-08T00:00:00.000Z",
|
||||||
|
updatedAt: "2026-04-08T01:00:00.000Z",
|
||||||
|
assignedAgentId: "agent-1",
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeSummary(overrides: Partial<AgentPerformanceSummary> = {}): AgentPerformanceSummary {
|
||||||
|
return {
|
||||||
|
agentId: "agent-1",
|
||||||
|
totalTasksCompleted: 3,
|
||||||
|
totalTasksFailed: 1,
|
||||||
|
avgDurationMs: 120_000,
|
||||||
|
successRate: 0.75,
|
||||||
|
commonErrors: ["timeout"],
|
||||||
|
strengths: ["Clear commit boundaries"],
|
||||||
|
weaknesses: ["Improve test coverage"],
|
||||||
|
recentReflectionCount: 2,
|
||||||
|
computedAt: "2026-04-08T01:00:00.000Z",
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeReflection(overrides: Partial<AgentReflection> = {}): AgentReflection {
|
||||||
|
return {
|
||||||
|
id: "reflection-1",
|
||||||
|
agentId: "agent-1",
|
||||||
|
timestamp: "2026-04-08T01:00:00.000Z",
|
||||||
|
trigger: "manual",
|
||||||
|
metrics: {
|
||||||
|
tasksCompleted: 2,
|
||||||
|
tasksFailed: 1,
|
||||||
|
avgDurationMs: 60_000,
|
||||||
|
commonErrors: ["timeout"],
|
||||||
|
},
|
||||||
|
insights: ["Tends to stall on testing"],
|
||||||
|
suggestedImprovements: ["Run targeted tests earlier"],
|
||||||
|
summary: "Performance is generally solid with room to tighten test feedback loops.",
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeRun(overrides: Partial<AgentHeartbeatRun> = {}): AgentHeartbeatRun {
|
||||||
|
return {
|
||||||
|
id: "run-1",
|
||||||
|
agentId: "agent-1",
|
||||||
|
startedAt: "2026-04-08T00:00:00.000Z",
|
||||||
|
endedAt: "2026-04-08T00:05:00.000Z",
|
||||||
|
status: "completed",
|
||||||
|
contextSnapshot: { taskId: "FN-001" },
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createMockDeps() {
|
||||||
|
const agentStore = {
|
||||||
|
getAgent: vi.fn().mockResolvedValue(makeAgent()),
|
||||||
|
getRecentRuns: vi.fn().mockResolvedValue([makeRun()]),
|
||||||
|
} as any;
|
||||||
|
|
||||||
|
const taskStore = {
|
||||||
|
listTasks: vi.fn().mockResolvedValue([makeTask()]),
|
||||||
|
} as any;
|
||||||
|
|
||||||
|
const reflectionStore = {
|
||||||
|
getPerformanceSummary: vi.fn().mockResolvedValue(makeSummary()),
|
||||||
|
getLatestReflection: vi.fn().mockResolvedValue(makeReflection()),
|
||||||
|
createReflection: vi.fn().mockImplementation(async ({
|
||||||
|
agentId,
|
||||||
|
trigger,
|
||||||
|
triggerDetail,
|
||||||
|
taskId,
|
||||||
|
metrics,
|
||||||
|
insights,
|
||||||
|
suggestedImprovements,
|
||||||
|
summary,
|
||||||
|
}: {
|
||||||
|
agentId: string;
|
||||||
|
trigger: AgentReflection["trigger"];
|
||||||
|
triggerDetail?: string;
|
||||||
|
taskId?: string;
|
||||||
|
metrics: ReflectionMetrics;
|
||||||
|
insights: string[];
|
||||||
|
suggestedImprovements: string[];
|
||||||
|
summary: string;
|
||||||
|
}) => makeReflection({
|
||||||
|
agentId,
|
||||||
|
trigger,
|
||||||
|
triggerDetail,
|
||||||
|
taskId,
|
||||||
|
metrics,
|
||||||
|
insights,
|
||||||
|
suggestedImprovements,
|
||||||
|
summary,
|
||||||
|
})),
|
||||||
|
} as any;
|
||||||
|
|
||||||
|
return { agentStore, taskStore, reflectionStore };
|
||||||
|
}
|
||||||
|
|
||||||
|
function createMockSession() {
|
||||||
|
return {
|
||||||
|
state: {},
|
||||||
|
dispose: vi.fn(),
|
||||||
|
} as any;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("AgentReflectionService", () => {
|
||||||
|
let tempRoot: string;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
tempRoot = await mkdtemp(join(tmpdir(), "agent-reflection-test-"));
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("buildReflectionContext", () => {
|
||||||
|
it("returns context with recent outcomes, summary, and latest reflection", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
|
||||||
|
const context = await service.buildReflectionContext("agent-1");
|
||||||
|
|
||||||
|
expect(context.agent.id).toBe("agent-1");
|
||||||
|
expect(context.recentOutcomes).toHaveLength(1);
|
||||||
|
expect(context.performanceSummary?.successRate).toBe(0.75);
|
||||||
|
expect(context.latestReflection?.id).toBe("reflection-1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns empty recentOutcomes when no task history exists", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
taskStore.listTasks.mockResolvedValue([]);
|
||||||
|
agentStore.getRecentRuns.mockResolvedValue([]);
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const context = await service.buildReflectionContext("agent-1");
|
||||||
|
|
||||||
|
expect(context.recentOutcomes).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads instructions from instructionsText when provided", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
agentStore.getAgent.mockResolvedValue(makeAgent({ instructionsText: "Always run tests before committing." }));
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const context = await service.buildReflectionContext("agent-1");
|
||||||
|
|
||||||
|
expect(context.instructions).toContain("Always run tests before committing.");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads instructions from file when instructionsPath is set", async () => {
|
||||||
|
const instructionsDir = join(tempRoot, "agents");
|
||||||
|
const instructionsPath = join(instructionsDir, "executor.md");
|
||||||
|
await mkdir(instructionsDir, { recursive: true });
|
||||||
|
await writeFile(instructionsPath, "Prefer smaller commits with clear messages.", { encoding: "utf-8" });
|
||||||
|
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
agentStore.getAgent.mockResolvedValue(makeAgent({ instructionsPath: "agents/executor.md" }));
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const context = await service.buildReflectionContext("agent-1");
|
||||||
|
|
||||||
|
expect(context.instructions).toContain("Prefer smaller commits with clear messages.");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("handles missing agent gracefully", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
agentStore.getAgent.mockResolvedValue(null);
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const context = await service.buildReflectionContext("agent-missing");
|
||||||
|
|
||||||
|
expect(context.agent.id).toBe("agent-missing");
|
||||||
|
expect(context.agent.name).toContain("Unknown Agent");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("includes latest reflection when available", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
reflectionStore.getLatestReflection.mockResolvedValue(makeReflection({ id: "reflection-latest" }));
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const context = await service.buildReflectionContext("agent-1");
|
||||||
|
|
||||||
|
expect(context.latestReflection?.id).toBe("reflection-latest");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("getRecentTaskOutcomes", () => {
|
||||||
|
it("returns completed outcomes for done and in-review tasks", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
taskStore.listTasks.mockResolvedValue([
|
||||||
|
makeTask({ id: "FN-001", column: "done", assignedAgentId: "agent-1" }),
|
||||||
|
makeTask({ id: "FN-002", column: "in-review", assignedAgentId: "agent-1" }),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const outcomes = await service.getRecentTaskOutcomes("agent-1", 10);
|
||||||
|
|
||||||
|
expect(outcomes.map((outcome) => outcome.outcome)).toEqual(["completed", "completed"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns failed outcomes when status includes failed", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
taskStore.listTasks.mockResolvedValue([
|
||||||
|
makeTask({ id: "FN-FAIL", column: "todo", status: "failed", assignedAgentId: "agent-1" }),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const outcomes = await service.getRecentTaskOutcomes("agent-1", 10);
|
||||||
|
|
||||||
|
expect(outcomes).toHaveLength(1);
|
||||||
|
expect(outcomes[0]?.outcome).toBe("failed");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns stuck outcomes when task was killed by stuck detector", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
taskStore.listTasks.mockResolvedValue([
|
||||||
|
makeTask({
|
||||||
|
id: "FN-STUCK",
|
||||||
|
column: "todo",
|
||||||
|
status: "stuck-killed",
|
||||||
|
assignedAgentId: "agent-1",
|
||||||
|
log: [{ timestamp: "2026-04-08T00:00:00.000Z", action: "Task terminated due to stuck agent session" }],
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const outcomes = await service.getRecentTaskOutcomes("agent-1", 10);
|
||||||
|
|
||||||
|
expect(outcomes).toHaveLength(1);
|
||||||
|
expect(outcomes[0]?.outcome).toBe("stuck");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("respects limit parameter", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
taskStore.listTasks.mockResolvedValue([
|
||||||
|
makeTask({ id: "FN-001", assignedAgentId: "agent-1" }),
|
||||||
|
makeTask({ id: "FN-002", assignedAgentId: "agent-1" }),
|
||||||
|
makeTask({ id: "FN-003", assignedAgentId: "agent-1" }),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const outcomes = await service.getRecentTaskOutcomes("agent-1", 2);
|
||||||
|
|
||||||
|
expect(outcomes).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns empty array when agent has no tasks", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
taskStore.listTasks.mockResolvedValue([makeTask({ assignedAgentId: "agent-2" })]);
|
||||||
|
agentStore.getRecentRuns.mockResolvedValue([]);
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const outcomes = await service.getRecentTaskOutcomes("agent-1", 10);
|
||||||
|
|
||||||
|
expect(outcomes).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("extractErrorPatterns", () => {
|
||||||
|
it("returns common errors from performance summary", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
reflectionStore.getPerformanceSummary.mockResolvedValue(makeSummary({ commonErrors: ["timeout", "merge conflict"] }));
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const errors = await service.extractErrorPatterns("agent-1");
|
||||||
|
|
||||||
|
expect(errors).toEqual(["timeout", "merge conflict"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns empty array when no summary exists", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
reflectionStore.getPerformanceSummary.mockResolvedValue(makeSummary({
|
||||||
|
totalTasksCompleted: 0,
|
||||||
|
totalTasksFailed: 0,
|
||||||
|
avgDurationMs: 0,
|
||||||
|
successRate: 0,
|
||||||
|
commonErrors: [],
|
||||||
|
strengths: [],
|
||||||
|
weaknesses: [],
|
||||||
|
recentReflectionCount: 0,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const errors = await service.extractErrorPatterns("agent-1");
|
||||||
|
|
||||||
|
expect(errors).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns empty array when summary has no errors", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
reflectionStore.getPerformanceSummary.mockResolvedValue(makeSummary({ commonErrors: [] }));
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const errors = await service.extractErrorPatterns("agent-1");
|
||||||
|
|
||||||
|
expect(errors).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("generateReflection", () => {
|
||||||
|
it("creates reflection when data is available", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
const session = createMockSession();
|
||||||
|
|
||||||
|
mockedCreateKbAgent.mockImplementation(async (options: any) => {
|
||||||
|
options.onText?.(JSON.stringify({
|
||||||
|
insights: ["Strong execution on scoped changes"],
|
||||||
|
suggestedImprovements: ["Run tests earlier in the cycle"],
|
||||||
|
summary: "Execution quality is strong with room to tighten feedback loops.",
|
||||||
|
}));
|
||||||
|
return { session };
|
||||||
|
});
|
||||||
|
mockedPromptWithFallback.mockResolvedValue(undefined);
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({
|
||||||
|
agentStore,
|
||||||
|
taskStore,
|
||||||
|
reflectionStore,
|
||||||
|
rootDir: tempRoot,
|
||||||
|
});
|
||||||
|
|
||||||
|
const reflection = await service.generateReflection("agent-1", "manual", {
|
||||||
|
taskId: "FN-001",
|
||||||
|
triggerDetail: "manual check",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(reflection).not.toBeNull();
|
||||||
|
expect(reflectionStore.createReflection).toHaveBeenCalledTimes(1);
|
||||||
|
expect(reflectionStore.createReflection).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
agentId: "agent-1",
|
||||||
|
trigger: "manual",
|
||||||
|
triggerDetail: "manual check",
|
||||||
|
taskId: "FN-001",
|
||||||
|
}));
|
||||||
|
expect(mockedCreateKbAgent).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
tools: "readonly",
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null when no meaningful data exists", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
taskStore.listTasks.mockResolvedValue([]);
|
||||||
|
agentStore.getRecentRuns.mockResolvedValue([]);
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const reflection = await service.generateReflection("agent-1", "manual");
|
||||||
|
|
||||||
|
expect(reflection).toBeNull();
|
||||||
|
expect(mockedCreateKbAgent).not.toHaveBeenCalled();
|
||||||
|
expect(reflectionStore.createReflection).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns null on AI session failure", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
mockedCreateKbAgent.mockRejectedValue(new Error("AI unavailable"));
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
const reflection = await service.generateReflection("agent-1", "manual");
|
||||||
|
|
||||||
|
expect(reflection).toBeNull();
|
||||||
|
expect(reflectionStore.createReflection).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("persists reflection via reflectionStore.createReflection", async () => {
|
||||||
|
const { agentStore, taskStore, reflectionStore } = createMockDeps();
|
||||||
|
const session = createMockSession();
|
||||||
|
|
||||||
|
mockedCreateKbAgent.mockImplementation(async (options: any) => {
|
||||||
|
options.onText?.(JSON.stringify({
|
||||||
|
insights: ["Insight A"],
|
||||||
|
suggestedImprovements: ["Improve B"],
|
||||||
|
summary: "Summary C",
|
||||||
|
}));
|
||||||
|
return { session };
|
||||||
|
});
|
||||||
|
mockedPromptWithFallback.mockResolvedValue(undefined);
|
||||||
|
|
||||||
|
const service = new AgentReflectionService({ agentStore, taskStore, reflectionStore, rootDir: tempRoot });
|
||||||
|
await service.generateReflection("agent-1", "post-task", {
|
||||||
|
taskId: "FN-777",
|
||||||
|
triggerDetail: "post-task reflection",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(reflectionStore.createReflection).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
trigger: "post-task",
|
||||||
|
taskId: "FN-777",
|
||||||
|
triggerDetail: "post-task reflection",
|
||||||
|
}));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("reflect_on_performance tool", () => {
|
||||||
|
it("returns formatted text when reflection succeeds", async () => {
|
||||||
|
const reflectionService = {
|
||||||
|
generateReflection: vi.fn().mockResolvedValue(makeReflection({
|
||||||
|
summary: "Reflection summary",
|
||||||
|
insights: ["Insight 1"],
|
||||||
|
suggestedImprovements: ["Improve 1"],
|
||||||
|
})),
|
||||||
|
} as any;
|
||||||
|
|
||||||
|
const tool = createReflectOnPerformanceTool(reflectionService, "agent-1");
|
||||||
|
const result = await (tool.execute as any)("tool-1", {}, {}, {}, undefined);
|
||||||
|
|
||||||
|
const content = result.content[0];
|
||||||
|
expect(content?.type).toBe("text");
|
||||||
|
if (!content || content.type !== "text") {
|
||||||
|
throw new Error("Expected text content");
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(content.text).toContain("Summary: Reflection summary");
|
||||||
|
expect(content.text).toContain("Insights:");
|
||||||
|
expect(content.text).toContain("Suggested Improvements:");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns no-data message when reflection returns null", async () => {
|
||||||
|
const reflectionService = {
|
||||||
|
generateReflection: vi.fn().mockResolvedValue(null),
|
||||||
|
} as any;
|
||||||
|
|
||||||
|
const tool = createReflectOnPerformanceTool(reflectionService, "agent-1");
|
||||||
|
const result = await (tool.execute as any)("tool-1", {}, {}, {}, undefined);
|
||||||
|
|
||||||
|
const content = result.content[0];
|
||||||
|
expect(content?.type).toBe("text");
|
||||||
|
if (!content || content.type !== "text") {
|
||||||
|
throw new Error("Expected text content");
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(content.text).toBe("No reflection data available — not enough history yet.");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes focus_area as triggerDetail", async () => {
|
||||||
|
const reflectionService = {
|
||||||
|
generateReflection: vi.fn().mockResolvedValue(makeReflection()),
|
||||||
|
} as any;
|
||||||
|
|
||||||
|
const tool = createReflectOnPerformanceTool(reflectionService, "agent-1");
|
||||||
|
await (tool.execute as any)("tool-1", { focus_area: "testing" }, {}, {}, undefined);
|
||||||
|
|
||||||
|
expect(reflectionService.generateReflection).toHaveBeenCalledWith(
|
||||||
|
"agent-1",
|
||||||
|
"manual",
|
||||||
|
{ triggerDetail: "Agent-initiated reflection focused on: testing" },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("parameter schema accepts optional focus_area", () => {
|
||||||
|
const required = (reflectOnPerformanceParams as { required?: string[] }).required;
|
||||||
|
expect(required?.includes("focus_area") ?? false).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
if (tempRoot) {
|
||||||
|
await rm(tempRoot, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
495
packages/engine/src/agent-reflection.ts
Normal file
495
packages/engine/src/agent-reflection.ts
Normal file
@@ -0,0 +1,495 @@
|
|||||||
|
import { readFile } from "node:fs/promises";
|
||||||
|
import { isAbsolute, resolve } from "node:path";
|
||||||
|
import type {
|
||||||
|
Agent,
|
||||||
|
AgentHeartbeatRun,
|
||||||
|
AgentPerformanceSummary,
|
||||||
|
AgentReflection,
|
||||||
|
AgentStore,
|
||||||
|
ReflectionMetrics,
|
||||||
|
ReflectionStore,
|
||||||
|
ReflectionTrigger,
|
||||||
|
Task,
|
||||||
|
TaskStore,
|
||||||
|
} from "@fusion/core";
|
||||||
|
import { createLogger } from "./logger.js";
|
||||||
|
import { createKbAgent, promptWithFallback } from "./pi.js";
|
||||||
|
|
||||||
|
const reflectionLog = createLogger("reflection");
|
||||||
|
|
||||||
|
const REFLECTION_SYSTEM_PROMPT = `You are an autonomous performance analyst reviewing an AI agent's recent execution history.
|
||||||
|
|
||||||
|
Your task is to identify concrete, actionable improvements from the provided metrics and outcomes.
|
||||||
|
Return STRICT JSON with this exact shape:
|
||||||
|
{
|
||||||
|
"insights": ["short, specific observation"],
|
||||||
|
"suggestedImprovements": ["actionable improvement"],
|
||||||
|
"summary": "2-4 sentence synthesis"
|
||||||
|
}
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- Output valid JSON only (no markdown fences, no prose outside JSON).
|
||||||
|
- Keep insights specific to the provided evidence.
|
||||||
|
- Prefer improvements that can be applied in the agent's next run.
|
||||||
|
- Avoid generic advice unless strongly justified by data.`;
|
||||||
|
|
||||||
|
const DEFAULT_OUTCOME_LIMIT = 20;
|
||||||
|
|
||||||
|
interface ReflectionContext {
|
||||||
|
agent: Agent;
|
||||||
|
recentOutcomes: TaskOutcome[];
|
||||||
|
performanceSummary: AgentPerformanceSummary | null;
|
||||||
|
latestReflection: AgentReflection | null;
|
||||||
|
instructions?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TaskOutcome {
|
||||||
|
taskId: string;
|
||||||
|
outcome: "completed" | "failed" | "stuck";
|
||||||
|
durationMs?: number;
|
||||||
|
completedAt?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ReflectionPayload {
|
||||||
|
insights: string[];
|
||||||
|
suggestedImprovements: string[];
|
||||||
|
summary: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AgentReflectionServiceOptions {
|
||||||
|
agentStore: AgentStore;
|
||||||
|
taskStore: TaskStore;
|
||||||
|
reflectionStore: ReflectionStore;
|
||||||
|
rootDir: string;
|
||||||
|
modelProvider?: string;
|
||||||
|
modelId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AgentReflectionService {
|
||||||
|
private readonly agentStore: AgentStore;
|
||||||
|
private readonly taskStore: TaskStore;
|
||||||
|
private readonly reflectionStore: ReflectionStore;
|
||||||
|
private readonly rootDir: string;
|
||||||
|
private readonly modelProvider?: string;
|
||||||
|
private readonly modelId?: string;
|
||||||
|
|
||||||
|
constructor(options: AgentReflectionServiceOptions) {
|
||||||
|
this.agentStore = options.agentStore;
|
||||||
|
this.taskStore = options.taskStore;
|
||||||
|
this.reflectionStore = options.reflectionStore;
|
||||||
|
this.rootDir = options.rootDir;
|
||||||
|
this.modelProvider = options.modelProvider;
|
||||||
|
this.modelId = options.modelId;
|
||||||
|
}
|
||||||
|
|
||||||
|
async generateReflection(
|
||||||
|
agentId: string,
|
||||||
|
trigger: ReflectionTrigger,
|
||||||
|
options: { taskId?: string; triggerDetail?: string } = {},
|
||||||
|
): Promise<AgentReflection | null> {
|
||||||
|
try {
|
||||||
|
const context = await this.buildReflectionContext(agentId);
|
||||||
|
const recentRuns = await this.agentStore.getRecentRuns(agentId, DEFAULT_OUTCOME_LIMIT);
|
||||||
|
|
||||||
|
if (context.recentOutcomes.length === 0 && recentRuns.length === 0) {
|
||||||
|
reflectionLog.log(`Skipping reflection for ${agentId}: no recent tasks or heartbeat runs`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
let responseText = "";
|
||||||
|
const { session } = await createKbAgent({
|
||||||
|
cwd: this.rootDir,
|
||||||
|
systemPrompt: REFLECTION_SYSTEM_PROMPT,
|
||||||
|
tools: "readonly",
|
||||||
|
defaultProvider: this.modelProvider,
|
||||||
|
defaultModelId: this.modelId,
|
||||||
|
onText: (delta: string) => {
|
||||||
|
responseText += delta;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await promptWithFallback(session, this.buildReflectionPrompt(context, options.triggerDetail));
|
||||||
|
|
||||||
|
if (session.state?.error) {
|
||||||
|
throw new Error(session.state.error);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
session.dispose();
|
||||||
|
} catch {
|
||||||
|
// best-effort cleanup
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = this.parseReflectionResponse(responseText);
|
||||||
|
const metrics = this.buildReflectionMetrics(context.recentOutcomes, context.performanceSummary, recentRuns);
|
||||||
|
|
||||||
|
return await this.reflectionStore.createReflection({
|
||||||
|
agentId,
|
||||||
|
trigger,
|
||||||
|
triggerDetail: options.triggerDetail,
|
||||||
|
taskId: options.taskId,
|
||||||
|
metrics,
|
||||||
|
insights: parsed.insights,
|
||||||
|
suggestedImprovements: parsed.suggestedImprovements,
|
||||||
|
summary: parsed.summary,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
reflectionLog.error(`Failed to generate reflection for ${agentId}: ${(error as Error).message}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async buildReflectionContext(agentId: string): Promise<ReflectionContext> {
|
||||||
|
const [agentRecord, recentOutcomes, performanceSummaryRaw, latestReflection] = await Promise.all([
|
||||||
|
this.agentStore.getAgent(agentId),
|
||||||
|
this.getRecentTaskOutcomes(agentId, DEFAULT_OUTCOME_LIMIT),
|
||||||
|
this.reflectionStore.getPerformanceSummary(agentId),
|
||||||
|
this.reflectionStore.getLatestReflection(agentId),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const agent = agentRecord ?? this.createUnknownAgent(agentId);
|
||||||
|
if (!agentRecord) {
|
||||||
|
reflectionLog.warn(`Agent ${agentId} not found while building reflection context`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const instructions = await this.resolveInstructions(agentRecord);
|
||||||
|
const performanceSummary = this.isMeaningfulSummary(performanceSummaryRaw)
|
||||||
|
? performanceSummaryRaw
|
||||||
|
: null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
agent,
|
||||||
|
recentOutcomes,
|
||||||
|
performanceSummary,
|
||||||
|
latestReflection,
|
||||||
|
instructions,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async getRecentTaskOutcomes(agentId: string, limit = DEFAULT_OUTCOME_LIMIT): Promise<TaskOutcome[]> {
|
||||||
|
const effectiveLimit = Math.max(1, limit);
|
||||||
|
|
||||||
|
const [tasks, recentRuns, agent] = await Promise.all([
|
||||||
|
this.taskStore.listTasks(),
|
||||||
|
this.agentStore.getRecentRuns(agentId, effectiveLimit * 4),
|
||||||
|
this.agentStore.getAgent(agentId),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const recentTaskIdsFromRuns = this.extractTaskIdsFromRuns(recentRuns);
|
||||||
|
const sortedByRecency = [...tasks].sort((a, b) => this.getTaskTimestampMs(b) - this.getTaskTimestampMs(a));
|
||||||
|
const tasksToScan = sortedByRecency.slice(0, effectiveLimit * 2);
|
||||||
|
|
||||||
|
const agentMentions = [agentId, agent?.name].filter((value): value is string => Boolean(value?.trim()));
|
||||||
|
|
||||||
|
const outcomes: TaskOutcome[] = [];
|
||||||
|
for (const task of tasksToScan) {
|
||||||
|
if (!this.isTaskLinkedToAgent(task, agentId, recentTaskIdsFromRuns, agentMentions)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const outcome = this.classifyOutcome(task);
|
||||||
|
if (!outcome) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const durationMs = this.calculateDurationMs(task);
|
||||||
|
const completedAt = this.resolveCompletedAt(task);
|
||||||
|
|
||||||
|
outcomes.push({
|
||||||
|
taskId: task.id,
|
||||||
|
outcome,
|
||||||
|
durationMs,
|
||||||
|
completedAt,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (outcomes.length >= effectiveLimit) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return outcomes;
|
||||||
|
}
|
||||||
|
|
||||||
|
async extractErrorPatterns(agentId: string): Promise<string[]> {
|
||||||
|
const summary = await this.reflectionStore.getPerformanceSummary(agentId);
|
||||||
|
if (!this.isMeaningfulSummary(summary)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return summary.commonErrors ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildReflectionPrompt(context: ReflectionContext, triggerDetail?: string): string {
|
||||||
|
const summary = {
|
||||||
|
agent: {
|
||||||
|
id: context.agent.id,
|
||||||
|
name: context.agent.name,
|
||||||
|
role: context.agent.role,
|
||||||
|
state: context.agent.state,
|
||||||
|
},
|
||||||
|
triggerDetail,
|
||||||
|
recentOutcomes: context.recentOutcomes,
|
||||||
|
performanceSummary: context.performanceSummary,
|
||||||
|
latestReflection: context.latestReflection
|
||||||
|
? {
|
||||||
|
timestamp: context.latestReflection.timestamp,
|
||||||
|
summary: context.latestReflection.summary,
|
||||||
|
insights: context.latestReflection.insights,
|
||||||
|
suggestedImprovements: context.latestReflection.suggestedImprovements,
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
instructions: context.instructions,
|
||||||
|
};
|
||||||
|
|
||||||
|
return [
|
||||||
|
"Analyze the following agent performance context and propose concrete improvements.",
|
||||||
|
"Respond with strict JSON matching the required schema.",
|
||||||
|
JSON.stringify(summary, null, 2),
|
||||||
|
].join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseReflectionResponse(rawResponse: string): ReflectionPayload {
|
||||||
|
const candidate = this.extractJsonCandidate(rawResponse);
|
||||||
|
|
||||||
|
let parsed: unknown;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(candidate);
|
||||||
|
} catch {
|
||||||
|
reflectionLog.warn("Reflection response was not valid JSON; using fallback reflection payload");
|
||||||
|
return {
|
||||||
|
insights: ["Insufficient structured output from reflection model."],
|
||||||
|
suggestedImprovements: ["Retry reflection with clearer historical context."],
|
||||||
|
summary: "The reflection model did not return valid structured JSON.",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const record = parsed as Partial<ReflectionPayload>;
|
||||||
|
const insights = Array.isArray(record.insights)
|
||||||
|
? record.insights.map((value) => String(value).trim()).filter(Boolean)
|
||||||
|
: [];
|
||||||
|
const suggestedImprovements = Array.isArray(record.suggestedImprovements)
|
||||||
|
? record.suggestedImprovements.map((value) => String(value).trim()).filter(Boolean)
|
||||||
|
: [];
|
||||||
|
const summary = typeof record.summary === "string" && record.summary.trim().length > 0
|
||||||
|
? record.summary.trim()
|
||||||
|
: "No summary was provided by the reflection model.";
|
||||||
|
|
||||||
|
return {
|
||||||
|
insights: insights.length > 0 ? insights : ["No specific insights were identified."],
|
||||||
|
suggestedImprovements: suggestedImprovements.length > 0
|
||||||
|
? suggestedImprovements
|
||||||
|
: ["No concrete improvements were suggested."],
|
||||||
|
summary,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private extractJsonCandidate(rawResponse: string): string {
|
||||||
|
const trimmed = rawResponse.trim();
|
||||||
|
if (!trimmed) {
|
||||||
|
return "{}";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (trimmed.startsWith("```") && trimmed.endsWith("```")) {
|
||||||
|
const withoutFences = trimmed
|
||||||
|
.replace(/^```(?:json)?\s*/i, "")
|
||||||
|
.replace(/\s*```$/, "")
|
||||||
|
.trim();
|
||||||
|
return withoutFences || "{}";
|
||||||
|
}
|
||||||
|
|
||||||
|
const firstBrace = trimmed.indexOf("{");
|
||||||
|
const lastBrace = trimmed.lastIndexOf("}");
|
||||||
|
if (firstBrace >= 0 && lastBrace > firstBrace) {
|
||||||
|
return trimmed.slice(firstBrace, lastBrace + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return trimmed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildReflectionMetrics(
|
||||||
|
outcomes: TaskOutcome[],
|
||||||
|
performanceSummary: AgentPerformanceSummary | null,
|
||||||
|
recentRuns: AgentHeartbeatRun[],
|
||||||
|
): ReflectionMetrics {
|
||||||
|
const tasksCompleted = outcomes.filter((outcome) => outcome.outcome === "completed").length;
|
||||||
|
const tasksFailed = outcomes.filter((outcome) => outcome.outcome !== "completed").length;
|
||||||
|
|
||||||
|
const durations = outcomes
|
||||||
|
.map((outcome) => outcome.durationMs)
|
||||||
|
.filter((duration): duration is number => typeof duration === "number" && Number.isFinite(duration));
|
||||||
|
|
||||||
|
const avgDurationMs = durations.length > 0
|
||||||
|
? Math.round(durations.reduce((sum, duration) => sum + duration, 0) / durations.length)
|
||||||
|
: performanceSummary?.avgDurationMs ?? 0;
|
||||||
|
|
||||||
|
const runErrors = recentRuns
|
||||||
|
.map((run) => this.extractRunError(run))
|
||||||
|
.filter((value): value is string => Boolean(value));
|
||||||
|
|
||||||
|
const mergedErrors = [
|
||||||
|
...(performanceSummary?.commonErrors ?? []),
|
||||||
|
...outcomes.filter((outcome) => outcome.outcome !== "completed").map((outcome) => `${outcome.outcome}: ${outcome.taskId}`),
|
||||||
|
...runErrors,
|
||||||
|
];
|
||||||
|
|
||||||
|
const commonErrors = Array.from(new Set(mergedErrors.map((error) => error.trim()).filter(Boolean))).slice(0, 10);
|
||||||
|
|
||||||
|
return {
|
||||||
|
tasksCompleted,
|
||||||
|
tasksFailed,
|
||||||
|
avgDurationMs,
|
||||||
|
commonErrors,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private extractRunError(run: AgentHeartbeatRun): string | null {
|
||||||
|
if (typeof run.stderrExcerpt === "string" && run.stderrExcerpt.trim()) {
|
||||||
|
return run.stderrExcerpt.trim().split("\n")[0] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const resultError = run.resultJson && typeof run.resultJson.error === "string"
|
||||||
|
? run.resultJson.error.trim()
|
||||||
|
: "";
|
||||||
|
|
||||||
|
if (resultError) {
|
||||||
|
return resultError;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private extractTaskIdsFromRuns(runs: AgentHeartbeatRun[]): Set<string> {
|
||||||
|
const ids = new Set<string>();
|
||||||
|
for (const run of runs) {
|
||||||
|
const taskId = run.contextSnapshot?.taskId;
|
||||||
|
if (typeof taskId === "string" && taskId.trim()) {
|
||||||
|
ids.add(taskId.trim());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ids;
|
||||||
|
}
|
||||||
|
|
||||||
|
private classifyOutcome(task: Task): TaskOutcome["outcome"] | null {
|
||||||
|
const normalizedStatus = task.status?.toLowerCase() ?? "";
|
||||||
|
const hasStuckSignal =
|
||||||
|
normalizedStatus.includes("stuck")
|
||||||
|
|| task.log.some((entry) => {
|
||||||
|
const action = entry.action.toLowerCase();
|
||||||
|
return action.includes("stuck") || action.includes("terminated due to stuck");
|
||||||
|
});
|
||||||
|
|
||||||
|
if (hasStuckSignal) {
|
||||||
|
return "stuck";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (normalizedStatus.includes("failed")) {
|
||||||
|
return "failed";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (task.column === "done" || task.column === "in-review") {
|
||||||
|
return "completed";
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private isTaskLinkedToAgent(
|
||||||
|
task: Task,
|
||||||
|
agentId: string,
|
||||||
|
recentTaskIdsFromRuns: Set<string>,
|
||||||
|
agentMentions: string[],
|
||||||
|
): boolean {
|
||||||
|
if (task.assignedAgentId === agentId) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (recentTaskIdsFromRuns.has(task.id)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (agentMentions.length === 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return task.log.some((entry) => {
|
||||||
|
const content = `${entry.action} ${entry.outcome ?? ""}`.toLowerCase();
|
||||||
|
return agentMentions.some((mention) => content.includes(mention.toLowerCase()));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private calculateDurationMs(task: Task): number | undefined {
|
||||||
|
const startedAtMs = Date.parse(task.createdAt);
|
||||||
|
const completedAtIso = this.resolveCompletedAt(task);
|
||||||
|
const completedAtMs = completedAtIso ? Date.parse(completedAtIso) : Date.parse(task.updatedAt);
|
||||||
|
|
||||||
|
if (!Number.isFinite(startedAtMs) || !Number.isFinite(completedAtMs) || completedAtMs <= startedAtMs) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
return completedAtMs - startedAtMs;
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolveCompletedAt(task: Task): string | undefined {
|
||||||
|
return task.columnMovedAt ?? task.updatedAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async resolveInstructions(agent: Agent | null): Promise<string | undefined> {
|
||||||
|
if (!agent) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
const pieces: string[] = [];
|
||||||
|
|
||||||
|
if (agent.instructionsText?.trim()) {
|
||||||
|
pieces.push(agent.instructionsText.trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (agent.instructionsPath?.trim()) {
|
||||||
|
const resolvedPath = isAbsolute(agent.instructionsPath)
|
||||||
|
? agent.instructionsPath
|
||||||
|
: resolve(this.rootDir, agent.instructionsPath);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const content = await readFile(resolvedPath, "utf-8");
|
||||||
|
if (content.trim()) {
|
||||||
|
pieces.push(content.trim());
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
reflectionLog.warn(
|
||||||
|
`Unable to read instructions file for ${agent.id} at ${agent.instructionsPath}: ${(error as Error).message}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return pieces.length > 0 ? pieces.join("\n\n") : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
private isMeaningfulSummary(summary: AgentPerformanceSummary): boolean {
|
||||||
|
return summary.recentReflectionCount > 0
|
||||||
|
|| summary.totalTasksCompleted > 0
|
||||||
|
|| summary.totalTasksFailed > 0
|
||||||
|
|| summary.commonErrors.length > 0
|
||||||
|
|| summary.strengths.length > 0
|
||||||
|
|| summary.weaknesses.length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private createUnknownAgent(agentId: string): Agent {
|
||||||
|
const now = new Date().toISOString();
|
||||||
|
return {
|
||||||
|
id: agentId,
|
||||||
|
name: `Unknown Agent (${agentId})`,
|
||||||
|
role: "custom",
|
||||||
|
state: "idle",
|
||||||
|
createdAt: now,
|
||||||
|
updatedAt: now,
|
||||||
|
metadata: {},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private getTaskTimestampMs(task: Task): number {
|
||||||
|
const candidate = task.columnMovedAt ?? task.updatedAt ?? task.createdAt;
|
||||||
|
const timestamp = Date.parse(candidate);
|
||||||
|
return Number.isFinite(timestamp) ? timestamp : 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@
|
|||||||
import type { TaskStore } from "@fusion/core";
|
import type { TaskStore } from "@fusion/core";
|
||||||
import type { ToolDefinition } from "@mariozechner/pi-coding-agent";
|
import type { ToolDefinition } from "@mariozechner/pi-coding-agent";
|
||||||
import { Type, type Static } from "@mariozechner/pi-ai";
|
import { Type, type Static } from "@mariozechner/pi-ai";
|
||||||
|
import type { AgentReflectionService } from "./agent-reflection.js";
|
||||||
|
|
||||||
// ── Tool parameter schemas (canonical definitions) ────────────────────────
|
// ── Tool parameter schemas (canonical definitions) ────────────────────────
|
||||||
|
|
||||||
@@ -25,6 +26,12 @@ export const taskLogParams = Type.Object({
|
|||||||
outcome: Type.Optional(Type.String({ description: "Result or consequence (optional)" })),
|
outcome: Type.Optional(Type.String({ description: "Result or consequence (optional)" })),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const reflectOnPerformanceParams = Type.Object({
|
||||||
|
focus_area: Type.Optional(
|
||||||
|
Type.String({ description: "Optional focus area for reflection (e.g., 'code quality', 'speed', 'testing')" }),
|
||||||
|
),
|
||||||
|
});
|
||||||
|
|
||||||
// ── Tool factory functions ────────────────────────────────────────────────
|
// ── Tool factory functions ────────────────────────────────────────────────
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -85,3 +92,51 @@ export function createTaskLogTool(store: TaskStore, taskId: string): ToolDefinit
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a `reflect_on_performance` tool that asks the reflection service to
|
||||||
|
* analyze recent agent performance and return actionable insights.
|
||||||
|
*/
|
||||||
|
export function createReflectOnPerformanceTool(
|
||||||
|
reflectionService: AgentReflectionService,
|
||||||
|
agentId: string,
|
||||||
|
): ToolDefinition {
|
||||||
|
return {
|
||||||
|
name: "reflect_on_performance",
|
||||||
|
label: "Reflect on Performance",
|
||||||
|
description:
|
||||||
|
'Review your past task performance and generate insights for improvement. Optionally focus on a specific area like "code quality", "speed", or "testing".',
|
||||||
|
parameters: reflectOnPerformanceParams,
|
||||||
|
execute: async (_id: string, params: Static<typeof reflectOnPerformanceParams>) => {
|
||||||
|
const triggerDetail = params.focus_area
|
||||||
|
? `Agent-initiated reflection focused on: ${params.focus_area}`
|
||||||
|
: "Agent-initiated reflection";
|
||||||
|
|
||||||
|
const reflection = await reflectionService.generateReflection(agentId, "manual", {
|
||||||
|
triggerDetail,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!reflection) {
|
||||||
|
return {
|
||||||
|
content: [{ type: "text" as const, text: "No reflection data available — not enough history yet." }],
|
||||||
|
details: {},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const formattedText = [
|
||||||
|
`Summary: ${reflection.summary}`,
|
||||||
|
"",
|
||||||
|
"Insights:",
|
||||||
|
...reflection.insights.map((insight, index) => `${index + 1}. ${insight}`),
|
||||||
|
"",
|
||||||
|
"Suggested Improvements:",
|
||||||
|
...reflection.suggestedImprovements.map((improvement, index) => `${index + 1}. ${improvement}`),
|
||||||
|
].join("\n");
|
||||||
|
|
||||||
|
return {
|
||||||
|
content: [{ type: "text" as const, text: formattedText }],
|
||||||
|
details: {},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,7 +23,14 @@ import type { StuckTaskDetector, StuckTaskEvent } from "./stuck-task-detector.js
|
|||||||
import { isContextLimitError } from "./context-limit-detector.js";
|
import { isContextLimitError } from "./context-limit-detector.js";
|
||||||
import { StepSessionExecutor, type StepSessionExecutorOptions, type StepResult } from "./step-session-executor.js";
|
import { StepSessionExecutor, type StepSessionExecutorOptions, type StepResult } from "./step-session-executor.js";
|
||||||
import { resolveAgentInstructions, buildSystemPromptWithInstructions } from "./agent-instructions.js";
|
import { resolveAgentInstructions, buildSystemPromptWithInstructions } from "./agent-instructions.js";
|
||||||
import { createTaskCreateTool as sharedCreateTaskCreateTool, createTaskLogTool as sharedCreateTaskLogTool, taskCreateParams, taskLogParams } from "./agent-tools.js";
|
import type { AgentReflectionService } from "./agent-reflection.js";
|
||||||
|
import {
|
||||||
|
createReflectOnPerformanceTool,
|
||||||
|
createTaskCreateTool as sharedCreateTaskCreateTool,
|
||||||
|
createTaskLogTool as sharedCreateTaskLogTool,
|
||||||
|
taskCreateParams,
|
||||||
|
taskLogParams,
|
||||||
|
} from "./agent-tools.js";
|
||||||
|
|
||||||
// Re-export for backward compatibility (tests import from executor.ts)
|
// Re-export for backward compatibility (tests import from executor.ts)
|
||||||
export { summarizeToolArgs } from "./agent-logger.js";
|
export { summarizeToolArgs } from "./agent-logger.js";
|
||||||
@@ -225,6 +232,8 @@ export interface TaskExecutorOptions {
|
|||||||
stuckTaskDetector?: StuckTaskDetector;
|
stuckTaskDetector?: StuckTaskDetector;
|
||||||
/** AgentStore for tracking spawned child agents. If not provided, spawning is disabled. */
|
/** AgentStore for tracking spawned child agents. If not provided, spawning is disabled. */
|
||||||
agentStore?: import("@fusion/core").AgentStore;
|
agentStore?: import("@fusion/core").AgentStore;
|
||||||
|
/** Reflection service used to generate self-reflection insights for agents. */
|
||||||
|
reflectionService?: AgentReflectionService;
|
||||||
missionStore?: MissionStore;
|
missionStore?: MissionStore;
|
||||||
onSliceComplete?: (slice: Slice) => void;
|
onSliceComplete?: (slice: Slice) => void;
|
||||||
onStart?: (task: Task, worktreePath: string) => void;
|
onStart?: (task: Task, worktreePath: string) => void;
|
||||||
@@ -1012,6 +1021,10 @@ export class TaskExecutor {
|
|||||||
const stepCheckpoints = new Map<number, string>();
|
const stepCheckpoints = new Map<number, string>();
|
||||||
|
|
||||||
const stuckDetector = this.options.stuckTaskDetector;
|
const stuckDetector = this.options.stuckTaskDetector;
|
||||||
|
const assignedAgentId = detail.assignedAgentId?.trim();
|
||||||
|
const reflectionTools = this.options.reflectionService && settings.reflectionEnabled && assignedAgentId
|
||||||
|
? [createReflectOnPerformanceTool(this.options.reflectionService, assignedAgentId)]
|
||||||
|
: [];
|
||||||
|
|
||||||
const customTools = [
|
const customTools = [
|
||||||
this.createTaskUpdateTool(task.id, codeReviewVerdicts, sessionRef, stepCheckpoints, stuckDetector),
|
this.createTaskUpdateTool(task.id, codeReviewVerdicts, sessionRef, stepCheckpoints, stuckDetector),
|
||||||
@@ -1021,6 +1034,8 @@ export class TaskExecutor {
|
|||||||
this.createTaskDoneTool(task.id, () => { taskDone = true; }),
|
this.createTaskDoneTool(task.id, () => { taskDone = true; }),
|
||||||
this.createReviewStepTool(task.id, worktreePath, detail.prompt, codeReviewVerdicts, sessionRef, stepCheckpoints, detail, stuckDetector),
|
this.createReviewStepTool(task.id, worktreePath, detail.prompt, codeReviewVerdicts, sessionRef, stepCheckpoints, detail, stuckDetector),
|
||||||
this.createSpawnAgentTool(task.id, worktreePath, settings),
|
this.createSpawnAgentTool(task.id, worktreePath, settings),
|
||||||
|
// Conditionally add agent self-reflection when enabled and task has an assigned agent.
|
||||||
|
...reflectionTools,
|
||||||
];
|
];
|
||||||
|
|
||||||
const agentLogger = new AgentLogger({
|
const agentLogger = new AgentLogger({
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ export { MissionAutopilot, type MissionAutopilotOptions } from "./mission-autopi
|
|||||||
export { aiMergeTask, type MergerOptions } from "./merger.js";
|
export { aiMergeTask, type MergerOptions } from "./merger.js";
|
||||||
export { reviewStep, type ReviewType, type ReviewVerdict, type ReviewResult, type ReviewOptions } from "./reviewer.js";
|
export { reviewStep, type ReviewType, type ReviewVerdict, type ReviewResult, type ReviewOptions } from "./reviewer.js";
|
||||||
export { createKbAgent, type AgentOptions, type AgentResult } from "./pi.js";
|
export { createKbAgent, type AgentOptions, type AgentResult } from "./pi.js";
|
||||||
|
export { AgentReflectionService, type AgentReflectionServiceOptions } from "./agent-reflection.js";
|
||||||
export { WorktreePool, scanIdleWorktrees, cleanupOrphanedWorktrees } from "./worktree-pool.js";
|
export { WorktreePool, scanIdleWorktrees, cleanupOrphanedWorktrees } from "./worktree-pool.js";
|
||||||
export { createLogger, type Logger } from "./logger.js";
|
export { createLogger, type Logger } from "./logger.js";
|
||||||
export { isUsageLimitError, UsageLimitPauser } from "./usage-limit-detector.js";
|
export { isUsageLimitError, UsageLimitPauser } from "./usage-limit-detector.js";
|
||||||
|
|||||||
Reference in New Issue
Block a user