feat(FN-1181): add ReflectionStore with agent reflection models
- Add reflection core types for triggers, metrics, persisted reflections, and aggregated performance summaries - Implement ReflectionStore with JSONL persistence, per-agent write locking, retrieval/latest helpers, summary aggregation, and cleanup APIs - Emit typed reflection events for creation and summary computation and export the new store/types from @fusion/core - Add comprehensive ReflectionStore tests covering persistence order, malformed JSONL handling, windowed summaries, events, deletion, and concurrent writes
This commit is contained in:
gsxdsm
@@ -1,5 +1,5 @@
|
||||
export { COLUMNS, COLUMN_LABELS, COLUMN_DESCRIPTIONS, VALID_TRANSITIONS, DEFAULT_SETTINGS, DEFAULT_GLOBAL_SETTINGS, DEFAULT_PROJECT_SETTINGS, GLOBAL_SETTINGS_KEYS, PROJECT_SETTINGS_KEYS, THINKING_LEVELS, THEME_MODES, COLOR_THEMES, WORKFLOW_STEP_TEMPLATES, AGENT_PERMISSIONS, agentToConfigSnapshot, diffConfigSnapshots } from "./types.js";
|
||||
export type { Column, IssueInfo, IssueState, PrInfo, PrStatus, Task, TaskAttachment, TaskComment, TaskCommentInput, TaskCreateInput, TaskDetail, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, MergeDetails, MergeResult, Settings, GlobalSettings, ProjectSettings, SettingsScope, TaskStep, StepStatus, TaskLogEntry, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, HeartbeatInvocationSource, AgentTaskSession, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, NtfyNotificationEvent, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, Mailbox } from "./types.js";
|
||||
export type { Column, IssueInfo, IssueState, PrInfo, PrStatus, Task, TaskAttachment, TaskComment, TaskCommentInput, TaskCreateInput, TaskDetail, AgentLogEntry, AgentLogType, AgentRole, BoardConfig, MergeDetails, MergeResult, Settings, GlobalSettings, ProjectSettings, SettingsScope, TaskStep, StepStatus, TaskLogEntry, ActivityLogEntry, ActivityEventType, ThinkingLevel, ThemeMode, ColorTheme, PlanningQuestion, PlanningSummary, PlanningResponse, PlanningQuestionType, ArchivedTaskEntry, BatchStatusRequest, BatchStatusResponse, BatchStatusEntry, BatchStatusResult, ModelPreset, WorkflowStep, WorkflowStepMode, WorkflowStepPhase, WorkflowStepInput, WorkflowStepResult, WorkflowStepTemplate, Agent, OrgTreeNode, AgentState, AgentDetail, AgentCreateInput, AgentUpdateInput, AgentApiKey, AgentApiKeyCreateResult, AgentCapability, AgentPromptTemplate, AgentPromptsConfig, AgentPermission, TaskAssignSource, AgentAccessState, AgentHeartbeatConfig, InstructionsBundleConfig, MessageResponseMode, AgentHeartbeatEvent, AgentHeartbeatRun, HeartbeatInvocationSource, AgentTaskSession, AgentConfigSnapshot, RevisionFieldDiff, AgentConfigRevision, AgentStats, ReflectionTrigger, ReflectionMetrics, AgentReflection, AgentPerformanceSummary, NtfyNotificationEvent, SteeringComment, ParticipantType, MessageType, Message, MessageCreateInput, MessageFilter, Mailbox } from "./types.js";
|
||||
export { AGENT_VALID_TRANSITIONS } from "./types.js";
|
||||
export {
|
||||
BUILTIN_AGENT_PROMPTS,
|
||||
@@ -15,6 +15,8 @@ export {
|
||||
} from "./agent-permissions.js";
|
||||
export { AgentStore } from "./agent-store.js";
|
||||
export type { AgentStoreEvents } from "./agent-store.js";
|
||||
export { ReflectionStore } from "./reflection-store.js";
|
||||
export type { ReflectionStoreEvents } from "./reflection-store.js";
|
||||
export { MessageStore } from "./message-store.js";
|
||||
export type { MessageStoreEvents } from "./message-store.js";
|
||||
export { TaskStore } from "./store.js";
|
||||
|
||||
511
packages/core/src/reflection-store.test.ts
Normal file
511
packages/core/src/reflection-store.test.ts
Normal file
@@ -0,0 +1,511 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { ReflectionStore } from "./reflection-store.js";
|
||||
import type { AgentReflection, ReflectionTrigger } from "./types.js";
|
||||
|
||||
function makeTmpDir(): string {
|
||||
return mkdtempSync(join(tmpdir(), "kb-reflection-store-test-"));
|
||||
}
|
||||
|
||||
function makeReflection(
|
||||
agentId: string,
|
||||
overrides: Partial<AgentReflection> = {},
|
||||
): AgentReflection {
|
||||
return {
|
||||
id: `reflection-${Math.random().toString(16).slice(2, 10)}`,
|
||||
agentId,
|
||||
timestamp: new Date().toISOString(),
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "summary",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("ReflectionStore", () => {
|
||||
let rootDir: string;
|
||||
let store: ReflectionStore;
|
||||
|
||||
beforeEach(async () => {
|
||||
rootDir = makeTmpDir();
|
||||
store = new ReflectionStore({ rootDir });
|
||||
await store.init();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(rootDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("init", () => {
|
||||
it("creates the agents/ directory inside rootDir", async () => {
|
||||
const agentsDir = join(rootDir, "agents");
|
||||
expect(existsSync(agentsDir)).toBe(true);
|
||||
});
|
||||
|
||||
it("is idempotent", async () => {
|
||||
await store.init();
|
||||
await store.init();
|
||||
expect(existsSync(join(rootDir, "agents"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createReflection", () => {
|
||||
it("creates a reflection with expected fields", async () => {
|
||||
const reflection = await store.createReflection({
|
||||
agentId: "agent-001",
|
||||
trigger: "post-task",
|
||||
triggerDetail: "after task FN-042 completion",
|
||||
taskId: "FN-042",
|
||||
metrics: { tasksCompleted: 1, avgDurationMs: 4200 },
|
||||
insights: ["Strong planning reduced context switching"],
|
||||
suggestedImprovements: ["Improve edge-case validation"],
|
||||
summary: "Solid execution with one validation gap.",
|
||||
});
|
||||
|
||||
expect(reflection.id).toMatch(/^reflection-/);
|
||||
expect(reflection.agentId).toBe("agent-001");
|
||||
expect(reflection.trigger).toBe("post-task");
|
||||
expect(reflection.triggerDetail).toBe("after task FN-042 completion");
|
||||
expect(reflection.taskId).toBe("FN-042");
|
||||
expect(reflection.metrics).toEqual({ tasksCompleted: 1, avgDurationMs: 4200 });
|
||||
expect(reflection.insights).toEqual(["Strong planning reduced context switching"]);
|
||||
expect(reflection.suggestedImprovements).toEqual(["Improve edge-case validation"]);
|
||||
expect(reflection.summary).toBe("Solid execution with one validation gap.");
|
||||
expect(Number.isNaN(Date.parse(reflection.timestamp))).toBe(false);
|
||||
});
|
||||
|
||||
it("generates unique reflection IDs", async () => {
|
||||
const first = await store.createReflection({
|
||||
agentId: "agent-001",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "first",
|
||||
});
|
||||
const second = await store.createReflection({
|
||||
agentId: "agent-001",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "second",
|
||||
});
|
||||
|
||||
expect(first.id).toMatch(/^reflection-/);
|
||||
expect(second.id).toMatch(/^reflection-/);
|
||||
expect(first.id).not.toBe(second.id);
|
||||
});
|
||||
|
||||
it("appends reflections to the JSONL log", async () => {
|
||||
const first = await store.createReflection({
|
||||
agentId: "agent-append",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["first"],
|
||||
suggestedImprovements: ["first improvement"],
|
||||
summary: "first",
|
||||
});
|
||||
const second = await store.createReflection({
|
||||
agentId: "agent-append",
|
||||
trigger: "periodic",
|
||||
metrics: {},
|
||||
insights: ["second"],
|
||||
suggestedImprovements: ["second improvement"],
|
||||
summary: "second",
|
||||
});
|
||||
|
||||
const filePath = join(rootDir, "agents", "agent-append-reflections.jsonl");
|
||||
const lines = readFileSync(filePath, "utf-8").trim().split("\n");
|
||||
|
||||
expect(lines).toHaveLength(2);
|
||||
expect((JSON.parse(lines[0]) as AgentReflection).id).toBe(first.id);
|
||||
expect((JSON.parse(lines[1]) as AgentReflection).id).toBe(second.id);
|
||||
});
|
||||
|
||||
it("emits reflection:created event", async () => {
|
||||
const handler = vi.fn();
|
||||
store.on("reflection:created", handler);
|
||||
|
||||
const reflection = await store.createReflection({
|
||||
agentId: "agent-events",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "event",
|
||||
});
|
||||
|
||||
expect(handler).toHaveBeenCalledOnce();
|
||||
expect(handler).toHaveBeenCalledWith(reflection);
|
||||
});
|
||||
|
||||
it("throws when agentId is empty", async () => {
|
||||
await expect(
|
||||
store.createReflection({
|
||||
agentId: " ",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "invalid",
|
||||
}),
|
||||
).rejects.toThrow("agentId is required");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getReflections", () => {
|
||||
it("returns reflections in reverse chronological (newest-first) order", async () => {
|
||||
const first = await store.createReflection({
|
||||
agentId: "agent-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["one"],
|
||||
suggestedImprovements: [],
|
||||
summary: "one",
|
||||
});
|
||||
const second = await store.createReflection({
|
||||
agentId: "agent-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["two"],
|
||||
suggestedImprovements: [],
|
||||
summary: "two",
|
||||
});
|
||||
const third = await store.createReflection({
|
||||
agentId: "agent-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["three"],
|
||||
suggestedImprovements: [],
|
||||
summary: "three",
|
||||
});
|
||||
|
||||
const reflections = await store.getReflections("agent-order");
|
||||
expect(reflections.map((reflection) => reflection.id)).toEqual([
|
||||
third.id,
|
||||
second.id,
|
||||
first.id,
|
||||
]);
|
||||
});
|
||||
|
||||
it("respects the limit parameter", async () => {
|
||||
for (let i = 0; i < 4; i += 1) {
|
||||
await store.createReflection({
|
||||
agentId: "agent-limit",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [`insight-${i}`],
|
||||
suggestedImprovements: [],
|
||||
summary: `summary-${i}`,
|
||||
});
|
||||
}
|
||||
|
||||
const reflections = await store.getReflections("agent-limit", 2);
|
||||
expect(reflections).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("returns an empty array when no reflection file exists", async () => {
|
||||
const reflections = await store.getReflections("agent-missing");
|
||||
expect(reflections).toEqual([]);
|
||||
});
|
||||
|
||||
it("skips malformed JSONL lines gracefully", async () => {
|
||||
const agentId = "agent-malformed";
|
||||
const filePath = join(rootDir, "agents", `${agentId}-reflections.jsonl`);
|
||||
const goodOne = makeReflection(agentId, { id: "reflection-good-1", summary: "good-1" });
|
||||
const goodTwo = makeReflection(agentId, { id: "reflection-good-2", summary: "good-2" });
|
||||
|
||||
writeFileSync(
|
||||
filePath,
|
||||
`${JSON.stringify(goodOne)}\n{not-json\n${JSON.stringify(goodTwo)}\n`,
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const reflections = await store.getReflections(agentId, 10);
|
||||
|
||||
expect(reflections).toHaveLength(2);
|
||||
expect(reflections.map((reflection) => reflection.id)).toEqual([
|
||||
"reflection-good-2",
|
||||
"reflection-good-1",
|
||||
]);
|
||||
expect(warnSpy).toHaveBeenCalledOnce();
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("returns all reflections when limit exceeds total count", async () => {
|
||||
await store.createReflection({
|
||||
agentId: "agent-all",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["one"],
|
||||
suggestedImprovements: [],
|
||||
summary: "one",
|
||||
});
|
||||
await store.createReflection({
|
||||
agentId: "agent-all",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["two"],
|
||||
suggestedImprovements: [],
|
||||
summary: "two",
|
||||
});
|
||||
|
||||
const reflections = await store.getReflections("agent-all", 100);
|
||||
expect(reflections).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getLatestReflection", () => {
|
||||
it("returns the most recent reflection", async () => {
|
||||
await store.createReflection({
|
||||
agentId: "agent-latest",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["older"],
|
||||
suggestedImprovements: [],
|
||||
summary: "older",
|
||||
});
|
||||
const newest = await store.createReflection({
|
||||
agentId: "agent-latest",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["newer"],
|
||||
suggestedImprovements: [],
|
||||
summary: "newer",
|
||||
});
|
||||
|
||||
const latest = await store.getLatestReflection("agent-latest");
|
||||
expect(latest?.id).toBe(newest.id);
|
||||
});
|
||||
|
||||
it("returns null when no reflections exist", async () => {
|
||||
const latest = await store.getLatestReflection("agent-empty");
|
||||
expect(latest).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("getPerformanceSummary", () => {
|
||||
it("aggregates metrics and derives strengths/weaknesses", async () => {
|
||||
await store.createReflection({
|
||||
agentId: "agent-summary",
|
||||
trigger: "post-task",
|
||||
taskId: "FN-100",
|
||||
metrics: {
|
||||
tasksCompleted: 2,
|
||||
tasksFailed: 1,
|
||||
avgDurationMs: 1000,
|
||||
commonErrors: ["timeout", "validation"],
|
||||
},
|
||||
insights: ["Great at debugging", "Clear task decomposition"],
|
||||
suggestedImprovements: ["Handle retries better", "Improve test coverage"],
|
||||
summary: "Older reflection",
|
||||
});
|
||||
|
||||
await store.createReflection({
|
||||
agentId: "agent-summary",
|
||||
trigger: "post-task",
|
||||
taskId: "FN-101",
|
||||
metrics: {
|
||||
tasksCompleted: 3,
|
||||
tasksFailed: 0,
|
||||
avgDurationMs: 3000,
|
||||
commonErrors: ["timeout", "rate limit"],
|
||||
},
|
||||
insights: ["Great at debugging", "Strong communication"],
|
||||
suggestedImprovements: ["Improve test coverage", "Tune model temperature"],
|
||||
summary: "Newer reflection",
|
||||
});
|
||||
|
||||
const summary = await store.getPerformanceSummary("agent-summary");
|
||||
|
||||
expect(summary.agentId).toBe("agent-summary");
|
||||
expect(summary.totalTasksCompleted).toBe(5);
|
||||
expect(summary.totalTasksFailed).toBe(1);
|
||||
expect(summary.avgDurationMs).toBe(2000);
|
||||
expect(summary.successRate).toBeCloseTo(5 / 6, 10);
|
||||
expect(summary.commonErrors).toEqual(["timeout", "rate limit", "validation"]);
|
||||
expect(summary.strengths).toEqual([
|
||||
"Great at debugging",
|
||||
"Strong communication",
|
||||
"Clear task decomposition",
|
||||
]);
|
||||
expect(summary.weaknesses).toEqual([
|
||||
"Improve test coverage",
|
||||
"Tune model temperature",
|
||||
"Handle retries better",
|
||||
]);
|
||||
expect(summary.recentReflectionCount).toBe(2);
|
||||
expect(Number.isNaN(Date.parse(summary.computedAt))).toBe(false);
|
||||
});
|
||||
|
||||
it("returns a zeroed summary when no reflections exist", async () => {
|
||||
const summary = await store.getPerformanceSummary("agent-none");
|
||||
|
||||
expect(summary).toMatchObject({
|
||||
agentId: "agent-none",
|
||||
totalTasksCompleted: 0,
|
||||
totalTasksFailed: 0,
|
||||
avgDurationMs: 0,
|
||||
successRate: 0,
|
||||
commonErrors: [],
|
||||
strengths: [],
|
||||
weaknesses: [],
|
||||
recentReflectionCount: 0,
|
||||
});
|
||||
expect(Number.isNaN(Date.parse(summary.computedAt))).toBe(false);
|
||||
});
|
||||
|
||||
it("excludes reflections outside the default 7-day window", async () => {
|
||||
const agentId = "agent-window-default";
|
||||
const filePath = join(rootDir, "agents", `${agentId}-reflections.jsonl`);
|
||||
const now = Date.now();
|
||||
|
||||
const oldReflection = makeReflection(agentId, {
|
||||
id: "reflection-old",
|
||||
timestamp: new Date(now - 10 * 24 * 60 * 60 * 1000).toISOString(),
|
||||
metrics: { tasksCompleted: 10 },
|
||||
});
|
||||
|
||||
const recentReflection = makeReflection(agentId, {
|
||||
id: "reflection-recent",
|
||||
timestamp: new Date(now - 2 * 24 * 60 * 60 * 1000).toISOString(),
|
||||
metrics: { tasksCompleted: 2, tasksFailed: 1 },
|
||||
});
|
||||
|
||||
writeFileSync(filePath, `${JSON.stringify(oldReflection)}\n${JSON.stringify(recentReflection)}\n`, "utf-8");
|
||||
|
||||
const summary = await store.getPerformanceSummary(agentId);
|
||||
expect(summary.totalTasksCompleted).toBe(2);
|
||||
expect(summary.totalTasksFailed).toBe(1);
|
||||
expect(summary.recentReflectionCount).toBe(1);
|
||||
});
|
||||
|
||||
it("respects a custom windowMs option", async () => {
|
||||
const agentId = "agent-window-custom";
|
||||
const filePath = join(rootDir, "agents", `${agentId}-reflections.jsonl`);
|
||||
const now = Date.now();
|
||||
|
||||
const older = makeReflection(agentId, {
|
||||
id: "reflection-older",
|
||||
timestamp: new Date(now - 10_000).toISOString(),
|
||||
metrics: { tasksCompleted: 1 },
|
||||
});
|
||||
const newest = makeReflection(agentId, {
|
||||
id: "reflection-newest",
|
||||
timestamp: new Date(now - 200).toISOString(),
|
||||
metrics: { tasksCompleted: 2 },
|
||||
});
|
||||
|
||||
writeFileSync(filePath, `${JSON.stringify(older)}\n${JSON.stringify(newest)}\n`, "utf-8");
|
||||
|
||||
const summary = await store.getPerformanceSummary(agentId, { windowMs: 1000 });
|
||||
expect(summary.totalTasksCompleted).toBe(2);
|
||||
expect(summary.recentReflectionCount).toBe(1);
|
||||
});
|
||||
|
||||
it("emits reflection:summary-computed", async () => {
|
||||
const handler = vi.fn();
|
||||
store.on("reflection:summary-computed", handler);
|
||||
|
||||
const summary = await store.getPerformanceSummary("agent-summary-event");
|
||||
|
||||
expect(handler).toHaveBeenCalledOnce();
|
||||
expect(handler).toHaveBeenCalledWith(summary);
|
||||
});
|
||||
});
|
||||
|
||||
describe("deleteReflections", () => {
|
||||
it("removes the agent reflection file", async () => {
|
||||
const agentId = "agent-delete";
|
||||
await store.createReflection({
|
||||
agentId,
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "to delete",
|
||||
});
|
||||
|
||||
const filePath = join(rootDir, "agents", `${agentId}-reflections.jsonl`);
|
||||
expect(existsSync(filePath)).toBe(true);
|
||||
|
||||
await store.deleteReflections(agentId);
|
||||
expect(existsSync(filePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("no-ops when the file does not exist", async () => {
|
||||
await expect(store.deleteReflections("agent-missing-delete")).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("concurrency", () => {
|
||||
it("allows concurrent createReflection calls for the same agent", async () => {
|
||||
const agentId = "agent-concurrent";
|
||||
const triggers: ReflectionTrigger[] = ["manual", "periodic", "post-task", "user-requested"];
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: 25 }, (_, i) =>
|
||||
store.createReflection({
|
||||
agentId,
|
||||
trigger: triggers[i % triggers.length],
|
||||
metrics: { tasksCompleted: 1 },
|
||||
insights: [`insight-${i}`],
|
||||
suggestedImprovements: [`improvement-${i}`],
|
||||
summary: `summary-${i}`,
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const reflections = await store.getReflections(agentId, 100);
|
||||
expect(reflections).toHaveLength(25);
|
||||
|
||||
const filePath = join(rootDir, "agents", `${agentId}-reflections.jsonl`);
|
||||
const lines = readFileSync(filePath, "utf-8").trim().split("\n");
|
||||
expect(lines).toHaveLength(25);
|
||||
});
|
||||
});
|
||||
|
||||
describe("append-only behavior", () => {
|
||||
it("preserves all reflections for the same agent in file order", async () => {
|
||||
const first = await store.createReflection({
|
||||
agentId: "agent-append-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["first"],
|
||||
suggestedImprovements: [],
|
||||
summary: "first",
|
||||
});
|
||||
const second = await store.createReflection({
|
||||
agentId: "agent-append-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["second"],
|
||||
suggestedImprovements: [],
|
||||
summary: "second",
|
||||
});
|
||||
const third = await store.createReflection({
|
||||
agentId: "agent-append-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["third"],
|
||||
suggestedImprovements: [],
|
||||
summary: "third",
|
||||
});
|
||||
|
||||
const filePath = join(rootDir, "agents", "agent-append-order-reflections.jsonl");
|
||||
const ids = readFileSync(filePath, "utf-8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => (JSON.parse(line) as AgentReflection).id);
|
||||
|
||||
expect(ids).toEqual([first.id, second.id, third.id]);
|
||||
});
|
||||
});
|
||||
});
|
||||
294
packages/core/src/reflection-store.ts
Normal file
294
packages/core/src/reflection-store.ts
Normal file
@@ -0,0 +1,294 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, readFile, unlink, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type {
|
||||
AgentPerformanceSummary,
|
||||
AgentReflection,
|
||||
ReflectionMetrics,
|
||||
ReflectionTrigger,
|
||||
} from "./types.js";
|
||||
|
||||
/** Events emitted by ReflectionStore. */
|
||||
export interface ReflectionStoreEvents {
|
||||
/** Emitted after a reflection is created and persisted. */
|
||||
"reflection:created": (reflection: AgentReflection) => void;
|
||||
/** Emitted when a performance summary is computed. */
|
||||
"reflection:summary-computed": (summary: AgentPerformanceSummary) => void;
|
||||
}
|
||||
|
||||
/** Constructor options for ReflectionStore. */
|
||||
export interface ReflectionStoreOptions {
|
||||
/** Root kb data directory (default: .fusion). */
|
||||
rootDir?: string;
|
||||
}
|
||||
|
||||
/** Input payload for creating a reflection. */
|
||||
export interface CreateReflectionInput {
|
||||
agentId: string;
|
||||
trigger: ReflectionTrigger;
|
||||
triggerDetail?: string;
|
||||
taskId?: string;
|
||||
metrics: ReflectionMetrics;
|
||||
insights: string[];
|
||||
suggestedImprovements: string[];
|
||||
summary: string;
|
||||
}
|
||||
|
||||
/** Options for computing a performance summary. */
|
||||
export interface PerformanceSummaryOptions {
|
||||
/** Time window in milliseconds to include reflections from. Defaults to 7 days. */
|
||||
windowMs?: number;
|
||||
}
|
||||
|
||||
interface AgentLock {
|
||||
promise: Promise<unknown>;
|
||||
}
|
||||
|
||||
const DEFAULT_REFLECTION_LIMIT = 50;
|
||||
const DEFAULT_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
const SUMMARY_LIST_LIMIT = 10;
|
||||
|
||||
/**
|
||||
* ReflectionStore persists agent self-reflection records in append-only JSONL files.
|
||||
*
|
||||
* Storage layout:
|
||||
* - `.fusion/agents/{agentId}-reflections.jsonl`
|
||||
*/
|
||||
export class ReflectionStore extends EventEmitter {
|
||||
private rootDir: string;
|
||||
private agentsDir: string;
|
||||
private locks: Map<string, AgentLock> = new Map();
|
||||
|
||||
constructor(options: ReflectionStoreOptions = {}) {
|
||||
super();
|
||||
this.rootDir = options.rootDir ?? ".fusion";
|
||||
this.agentsDir = join(this.rootDir, "agents");
|
||||
}
|
||||
|
||||
override on(event: "reflection:created", listener: ReflectionStoreEvents["reflection:created"]): this;
|
||||
override on(
|
||||
event: "reflection:summary-computed",
|
||||
listener: ReflectionStoreEvents["reflection:summary-computed"],
|
||||
): this;
|
||||
override on(event: string | symbol, listener: (...args: any[]) => void): this {
|
||||
return super.on(event, listener);
|
||||
}
|
||||
|
||||
override emit(event: "reflection:created", reflection: AgentReflection): boolean;
|
||||
override emit(event: "reflection:summary-computed", summary: AgentPerformanceSummary): boolean;
|
||||
override emit(event: string | symbol, ...args: any[]): boolean {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
|
||||
/** Ensure required directories exist. */
|
||||
async init(): Promise<void> {
|
||||
await mkdir(this.agentsDir, { recursive: true });
|
||||
}
|
||||
|
||||
/** Create and append a reflection for an agent. */
|
||||
async createReflection(input: CreateReflectionInput): Promise<AgentReflection> {
|
||||
if (!input.agentId?.trim()) {
|
||||
throw new Error("agentId is required");
|
||||
}
|
||||
|
||||
return this.withLock(input.agentId, async () => {
|
||||
const reflection: AgentReflection = {
|
||||
id: `reflection-${randomUUID().slice(0, 8)}`,
|
||||
agentId: input.agentId,
|
||||
timestamp: new Date().toISOString(),
|
||||
trigger: input.trigger,
|
||||
triggerDetail: input.triggerDetail,
|
||||
taskId: input.taskId,
|
||||
metrics: input.metrics,
|
||||
insights: input.insights,
|
||||
suggestedImprovements: input.suggestedImprovements,
|
||||
summary: input.summary,
|
||||
};
|
||||
|
||||
const line = `${JSON.stringify(reflection)}\n`;
|
||||
await writeFile(this.reflectionsPath(input.agentId), line, { flag: "a" });
|
||||
|
||||
this.emit("reflection:created", reflection);
|
||||
return reflection;
|
||||
});
|
||||
}
|
||||
|
||||
/** Get recent reflections for an agent (newest first). */
|
||||
async getReflections(agentId: string, limit = DEFAULT_REFLECTION_LIMIT): Promise<AgentReflection[]> {
|
||||
if (!agentId?.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const reflectionPath = this.reflectionsPath(agentId);
|
||||
if (!existsSync(reflectionPath)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const reflections = await this.readReflectionsFromFile(agentId);
|
||||
return reflections.slice(0, Math.max(0, limit));
|
||||
}
|
||||
|
||||
/** Get the most recent reflection for an agent. */
|
||||
async getLatestReflection(agentId: string): Promise<AgentReflection | null> {
|
||||
const reflections = await this.getReflections(agentId, 1);
|
||||
return reflections[0] ?? null;
|
||||
}
|
||||
|
||||
/** Compute an aggregate performance summary from recent reflections. */
|
||||
async getPerformanceSummary(
|
||||
agentId: string,
|
||||
options: PerformanceSummaryOptions = {},
|
||||
): Promise<AgentPerformanceSummary> {
|
||||
const windowMs = options.windowMs ?? DEFAULT_WINDOW_MS;
|
||||
const cutoff = Date.now() - windowMs;
|
||||
const allReflections = await this.getReflections(agentId, Number.MAX_SAFE_INTEGER);
|
||||
|
||||
const windowedReflections = allReflections.filter((reflection) => {
|
||||
const timestamp = Date.parse(reflection.timestamp);
|
||||
return Number.isFinite(timestamp) && timestamp >= cutoff;
|
||||
});
|
||||
|
||||
let totalTasksCompleted = 0;
|
||||
let totalTasksFailed = 0;
|
||||
|
||||
const durations: number[] = [];
|
||||
const errorCounts = new Map<string, number>();
|
||||
|
||||
for (const reflection of windowedReflections) {
|
||||
totalTasksCompleted += reflection.metrics.tasksCompleted ?? 0;
|
||||
totalTasksFailed += reflection.metrics.tasksFailed ?? 0;
|
||||
|
||||
if (typeof reflection.metrics.avgDurationMs === "number") {
|
||||
durations.push(reflection.metrics.avgDurationMs);
|
||||
}
|
||||
|
||||
for (const error of reflection.metrics.commonErrors ?? []) {
|
||||
const normalized = error.trim();
|
||||
if (!normalized) continue;
|
||||
errorCounts.set(normalized, (errorCounts.get(normalized) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
const avgDurationMs = durations.length > 0
|
||||
? durations.reduce((sum, value) => sum + value, 0) / durations.length
|
||||
: 0;
|
||||
|
||||
const totalTasks = totalTasksCompleted + totalTasksFailed;
|
||||
const successRate = totalTasks > 0 ? totalTasksCompleted / totalTasks : 0;
|
||||
|
||||
const commonErrors = Array.from(errorCounts.entries())
|
||||
.sort((a, b) => {
|
||||
if (b[1] !== a[1]) return b[1] - a[1];
|
||||
return a[0].localeCompare(b[0]);
|
||||
})
|
||||
.slice(0, SUMMARY_LIST_LIMIT)
|
||||
.map(([error]) => error);
|
||||
|
||||
const strengths = this.collectRecentUnique(
|
||||
windowedReflections.flatMap((reflection) => reflection.insights),
|
||||
SUMMARY_LIST_LIMIT,
|
||||
);
|
||||
|
||||
const weaknesses = this.collectRecentUnique(
|
||||
windowedReflections.flatMap((reflection) => reflection.suggestedImprovements),
|
||||
SUMMARY_LIST_LIMIT,
|
||||
);
|
||||
|
||||
const summary: AgentPerformanceSummary = {
|
||||
agentId,
|
||||
totalTasksCompleted,
|
||||
totalTasksFailed,
|
||||
avgDurationMs,
|
||||
successRate,
|
||||
commonErrors,
|
||||
strengths,
|
||||
weaknesses,
|
||||
recentReflectionCount: windowedReflections.length,
|
||||
computedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
this.emit("reflection:summary-computed", summary);
|
||||
return summary;
|
||||
}
|
||||
|
||||
/** Delete all persisted reflections for an agent. */
|
||||
async deleteReflections(agentId: string): Promise<void> {
|
||||
if (!agentId?.trim()) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.withLock(agentId, async () => {
|
||||
try {
|
||||
await unlink(this.reflectionsPath(agentId));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private reflectionsPath(agentId: string): string {
|
||||
return join(this.agentsDir, `${agentId}-reflections.jsonl`);
|
||||
}
|
||||
|
||||
private async readReflectionsFromFile(agentId: string): Promise<AgentReflection[]> {
|
||||
const reflectionPath = this.reflectionsPath(agentId);
|
||||
|
||||
try {
|
||||
const content = await readFile(reflectionPath, "utf-8");
|
||||
const lines = content.split("\n").filter(Boolean);
|
||||
|
||||
const reflections: AgentReflection[] = [];
|
||||
for (const [index, line] of lines.entries()) {
|
||||
try {
|
||||
reflections.push(JSON.parse(line) as AgentReflection);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[ReflectionStore] Skipping malformed reflection line ${index + 1} for ${agentId}`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return reflections.reverse();
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private collectRecentUnique(items: string[], maxItems: number): string[] {
|
||||
const seen = new Set<string>();
|
||||
const deduped: string[] = [];
|
||||
|
||||
for (const item of items) {
|
||||
const normalized = item.trim();
|
||||
if (!normalized || seen.has(normalized)) continue;
|
||||
seen.add(normalized);
|
||||
deduped.push(normalized);
|
||||
if (deduped.length >= maxItems) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return deduped;
|
||||
}
|
||||
|
||||
private async withLock<T>(agentId: string, fn: () => Promise<T>): Promise<T> {
|
||||
let lock = this.locks.get(agentId);
|
||||
if (!lock) {
|
||||
lock = { promise: Promise.resolve() };
|
||||
this.locks.set(agentId, lock);
|
||||
}
|
||||
|
||||
const operation = lock.promise.then(fn, fn);
|
||||
lock.promise = operation;
|
||||
|
||||
return operation as Promise<T>;
|
||||
}
|
||||
}
|
||||
@@ -1842,6 +1842,73 @@ export interface AgentStats {
|
||||
successRate: number;
|
||||
}
|
||||
|
||||
/** Trigger source for an agent self-reflection run */
|
||||
export type ReflectionTrigger = "periodic" | "post-task" | "manual" | "user-requested";
|
||||
|
||||
/** Quantitative snapshot captured by a reflection */
|
||||
export interface ReflectionMetrics {
|
||||
/** Tasks completed in the analysis window */
|
||||
tasksCompleted?: number;
|
||||
/** Tasks failed in the analysis window */
|
||||
tasksFailed?: number;
|
||||
/** Average task duration in milliseconds */
|
||||
avgDurationMs?: number;
|
||||
/** Total tokens consumed in the analysis window */
|
||||
totalTokensUsed?: number;
|
||||
/** Number of errors encountered */
|
||||
errorCount?: number;
|
||||
/** Recurring error patterns */
|
||||
commonErrors?: string[];
|
||||
}
|
||||
|
||||
/** A persisted self-reflection generated by an agent */
|
||||
export interface AgentReflection {
|
||||
/** Unique reflection ID */
|
||||
id: string;
|
||||
/** The agent this reflection belongs to */
|
||||
agentId: string;
|
||||
/** ISO-8601 timestamp when the reflection was created */
|
||||
timestamp: string;
|
||||
/** What caused this reflection */
|
||||
trigger: ReflectionTrigger;
|
||||
/** Optional trigger detail context */
|
||||
triggerDetail?: string;
|
||||
/** Associated task ID (for post-task reflections) */
|
||||
taskId?: string;
|
||||
/** Quantitative reflection metrics */
|
||||
metrics: ReflectionMetrics;
|
||||
/** Key observations from self-analysis */
|
||||
insights: string[];
|
||||
/** Suggested improvements for future runs */
|
||||
suggestedImprovements: string[];
|
||||
/** One-paragraph narrative summary */
|
||||
summary: string;
|
||||
}
|
||||
|
||||
/** Aggregated performance summary derived from recent reflections */
|
||||
export interface AgentPerformanceSummary {
|
||||
/** Agent identifier */
|
||||
agentId: string;
|
||||
/** Total tasks completed in the analysis window */
|
||||
totalTasksCompleted: number;
|
||||
/** Total tasks failed in the analysis window */
|
||||
totalTasksFailed: number;
|
||||
/** Average task duration in milliseconds */
|
||||
avgDurationMs: number;
|
||||
/** Success ratio from 0 to 1 */
|
||||
successRate: number;
|
||||
/** Top recurring errors */
|
||||
commonErrors: string[];
|
||||
/** Derived strengths from successful patterns */
|
||||
strengths: string[];
|
||||
/** Derived weaknesses from failure patterns */
|
||||
weaknesses: string[];
|
||||
/** Number of reflections considered in this summary */
|
||||
recentReflectionCount: number;
|
||||
/** ISO-8601 timestamp when summary was computed */
|
||||
computedAt: string;
|
||||
}
|
||||
|
||||
// ── Multi-Project First-Run & Migration Types ───────────────────────────────
|
||||
|
||||
/** Detected project for migration consideration */
|
||||
|
||||
Reference in New Issue
Block a user