feat(FN-1181): add ReflectionStore with agent reflection models
- Add reflection core types for triggers, metrics, persisted reflections, and aggregated performance summaries - Implement ReflectionStore with JSONL persistence, per-agent write locking, retrieval/latest helpers, summary aggregation, and cleanup APIs - Emit typed reflection events for creation and summary computation and export the new store/types from @fusion/core - Add comprehensive ReflectionStore tests covering persistence order, malformed JSONL handling, windowed summaries, events, deletion, and concurrent writes
This commit is contained in:
gsxdsm
511
packages/core/src/reflection-store.test.ts
Normal file
511
packages/core/src/reflection-store.test.ts
Normal file
@@ -0,0 +1,511 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { ReflectionStore } from "./reflection-store.js";
|
||||
import type { AgentReflection, ReflectionTrigger } from "./types.js";
|
||||
|
||||
function makeTmpDir(): string {
|
||||
return mkdtempSync(join(tmpdir(), "kb-reflection-store-test-"));
|
||||
}
|
||||
|
||||
function makeReflection(
|
||||
agentId: string,
|
||||
overrides: Partial<AgentReflection> = {},
|
||||
): AgentReflection {
|
||||
return {
|
||||
id: `reflection-${Math.random().toString(16).slice(2, 10)}`,
|
||||
agentId,
|
||||
timestamp: new Date().toISOString(),
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "summary",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("ReflectionStore", () => {
|
||||
let rootDir: string;
|
||||
let store: ReflectionStore;
|
||||
|
||||
beforeEach(async () => {
|
||||
rootDir = makeTmpDir();
|
||||
store = new ReflectionStore({ rootDir });
|
||||
await store.init();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(rootDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("init", () => {
|
||||
it("creates the agents/ directory inside rootDir", async () => {
|
||||
const agentsDir = join(rootDir, "agents");
|
||||
expect(existsSync(agentsDir)).toBe(true);
|
||||
});
|
||||
|
||||
it("is idempotent", async () => {
|
||||
await store.init();
|
||||
await store.init();
|
||||
expect(existsSync(join(rootDir, "agents"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createReflection", () => {
|
||||
it("creates a reflection with expected fields", async () => {
|
||||
const reflection = await store.createReflection({
|
||||
agentId: "agent-001",
|
||||
trigger: "post-task",
|
||||
triggerDetail: "after task FN-042 completion",
|
||||
taskId: "FN-042",
|
||||
metrics: { tasksCompleted: 1, avgDurationMs: 4200 },
|
||||
insights: ["Strong planning reduced context switching"],
|
||||
suggestedImprovements: ["Improve edge-case validation"],
|
||||
summary: "Solid execution with one validation gap.",
|
||||
});
|
||||
|
||||
expect(reflection.id).toMatch(/^reflection-/);
|
||||
expect(reflection.agentId).toBe("agent-001");
|
||||
expect(reflection.trigger).toBe("post-task");
|
||||
expect(reflection.triggerDetail).toBe("after task FN-042 completion");
|
||||
expect(reflection.taskId).toBe("FN-042");
|
||||
expect(reflection.metrics).toEqual({ tasksCompleted: 1, avgDurationMs: 4200 });
|
||||
expect(reflection.insights).toEqual(["Strong planning reduced context switching"]);
|
||||
expect(reflection.suggestedImprovements).toEqual(["Improve edge-case validation"]);
|
||||
expect(reflection.summary).toBe("Solid execution with one validation gap.");
|
||||
expect(Number.isNaN(Date.parse(reflection.timestamp))).toBe(false);
|
||||
});
|
||||
|
||||
it("generates unique reflection IDs", async () => {
|
||||
const first = await store.createReflection({
|
||||
agentId: "agent-001",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "first",
|
||||
});
|
||||
const second = await store.createReflection({
|
||||
agentId: "agent-001",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "second",
|
||||
});
|
||||
|
||||
expect(first.id).toMatch(/^reflection-/);
|
||||
expect(second.id).toMatch(/^reflection-/);
|
||||
expect(first.id).not.toBe(second.id);
|
||||
});
|
||||
|
||||
it("appends reflections to the JSONL log", async () => {
|
||||
const first = await store.createReflection({
|
||||
agentId: "agent-append",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["first"],
|
||||
suggestedImprovements: ["first improvement"],
|
||||
summary: "first",
|
||||
});
|
||||
const second = await store.createReflection({
|
||||
agentId: "agent-append",
|
||||
trigger: "periodic",
|
||||
metrics: {},
|
||||
insights: ["second"],
|
||||
suggestedImprovements: ["second improvement"],
|
||||
summary: "second",
|
||||
});
|
||||
|
||||
const filePath = join(rootDir, "agents", "agent-append-reflections.jsonl");
|
||||
const lines = readFileSync(filePath, "utf-8").trim().split("\n");
|
||||
|
||||
expect(lines).toHaveLength(2);
|
||||
expect((JSON.parse(lines[0]) as AgentReflection).id).toBe(first.id);
|
||||
expect((JSON.parse(lines[1]) as AgentReflection).id).toBe(second.id);
|
||||
});
|
||||
|
||||
it("emits reflection:created event", async () => {
|
||||
const handler = vi.fn();
|
||||
store.on("reflection:created", handler);
|
||||
|
||||
const reflection = await store.createReflection({
|
||||
agentId: "agent-events",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "event",
|
||||
});
|
||||
|
||||
expect(handler).toHaveBeenCalledOnce();
|
||||
expect(handler).toHaveBeenCalledWith(reflection);
|
||||
});
|
||||
|
||||
it("throws when agentId is empty", async () => {
|
||||
await expect(
|
||||
store.createReflection({
|
||||
agentId: " ",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "invalid",
|
||||
}),
|
||||
).rejects.toThrow("agentId is required");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getReflections", () => {
|
||||
it("returns reflections in reverse chronological (newest-first) order", async () => {
|
||||
const first = await store.createReflection({
|
||||
agentId: "agent-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["one"],
|
||||
suggestedImprovements: [],
|
||||
summary: "one",
|
||||
});
|
||||
const second = await store.createReflection({
|
||||
agentId: "agent-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["two"],
|
||||
suggestedImprovements: [],
|
||||
summary: "two",
|
||||
});
|
||||
const third = await store.createReflection({
|
||||
agentId: "agent-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["three"],
|
||||
suggestedImprovements: [],
|
||||
summary: "three",
|
||||
});
|
||||
|
||||
const reflections = await store.getReflections("agent-order");
|
||||
expect(reflections.map((reflection) => reflection.id)).toEqual([
|
||||
third.id,
|
||||
second.id,
|
||||
first.id,
|
||||
]);
|
||||
});
|
||||
|
||||
it("respects the limit parameter", async () => {
|
||||
for (let i = 0; i < 4; i += 1) {
|
||||
await store.createReflection({
|
||||
agentId: "agent-limit",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [`insight-${i}`],
|
||||
suggestedImprovements: [],
|
||||
summary: `summary-${i}`,
|
||||
});
|
||||
}
|
||||
|
||||
const reflections = await store.getReflections("agent-limit", 2);
|
||||
expect(reflections).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("returns an empty array when no reflection file exists", async () => {
|
||||
const reflections = await store.getReflections("agent-missing");
|
||||
expect(reflections).toEqual([]);
|
||||
});
|
||||
|
||||
it("skips malformed JSONL lines gracefully", async () => {
|
||||
const agentId = "agent-malformed";
|
||||
const filePath = join(rootDir, "agents", `${agentId}-reflections.jsonl`);
|
||||
const goodOne = makeReflection(agentId, { id: "reflection-good-1", summary: "good-1" });
|
||||
const goodTwo = makeReflection(agentId, { id: "reflection-good-2", summary: "good-2" });
|
||||
|
||||
writeFileSync(
|
||||
filePath,
|
||||
`${JSON.stringify(goodOne)}\n{not-json\n${JSON.stringify(goodTwo)}\n`,
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const reflections = await store.getReflections(agentId, 10);
|
||||
|
||||
expect(reflections).toHaveLength(2);
|
||||
expect(reflections.map((reflection) => reflection.id)).toEqual([
|
||||
"reflection-good-2",
|
||||
"reflection-good-1",
|
||||
]);
|
||||
expect(warnSpy).toHaveBeenCalledOnce();
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("returns all reflections when limit exceeds total count", async () => {
|
||||
await store.createReflection({
|
||||
agentId: "agent-all",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["one"],
|
||||
suggestedImprovements: [],
|
||||
summary: "one",
|
||||
});
|
||||
await store.createReflection({
|
||||
agentId: "agent-all",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["two"],
|
||||
suggestedImprovements: [],
|
||||
summary: "two",
|
||||
});
|
||||
|
||||
const reflections = await store.getReflections("agent-all", 100);
|
||||
expect(reflections).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getLatestReflection", () => {
|
||||
it("returns the most recent reflection", async () => {
|
||||
await store.createReflection({
|
||||
agentId: "agent-latest",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["older"],
|
||||
suggestedImprovements: [],
|
||||
summary: "older",
|
||||
});
|
||||
const newest = await store.createReflection({
|
||||
agentId: "agent-latest",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["newer"],
|
||||
suggestedImprovements: [],
|
||||
summary: "newer",
|
||||
});
|
||||
|
||||
const latest = await store.getLatestReflection("agent-latest");
|
||||
expect(latest?.id).toBe(newest.id);
|
||||
});
|
||||
|
||||
it("returns null when no reflections exist", async () => {
|
||||
const latest = await store.getLatestReflection("agent-empty");
|
||||
expect(latest).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("getPerformanceSummary", () => {
|
||||
it("aggregates metrics and derives strengths/weaknesses", async () => {
|
||||
await store.createReflection({
|
||||
agentId: "agent-summary",
|
||||
trigger: "post-task",
|
||||
taskId: "FN-100",
|
||||
metrics: {
|
||||
tasksCompleted: 2,
|
||||
tasksFailed: 1,
|
||||
avgDurationMs: 1000,
|
||||
commonErrors: ["timeout", "validation"],
|
||||
},
|
||||
insights: ["Great at debugging", "Clear task decomposition"],
|
||||
suggestedImprovements: ["Handle retries better", "Improve test coverage"],
|
||||
summary: "Older reflection",
|
||||
});
|
||||
|
||||
await store.createReflection({
|
||||
agentId: "agent-summary",
|
||||
trigger: "post-task",
|
||||
taskId: "FN-101",
|
||||
metrics: {
|
||||
tasksCompleted: 3,
|
||||
tasksFailed: 0,
|
||||
avgDurationMs: 3000,
|
||||
commonErrors: ["timeout", "rate limit"],
|
||||
},
|
||||
insights: ["Great at debugging", "Strong communication"],
|
||||
suggestedImprovements: ["Improve test coverage", "Tune model temperature"],
|
||||
summary: "Newer reflection",
|
||||
});
|
||||
|
||||
const summary = await store.getPerformanceSummary("agent-summary");
|
||||
|
||||
expect(summary.agentId).toBe("agent-summary");
|
||||
expect(summary.totalTasksCompleted).toBe(5);
|
||||
expect(summary.totalTasksFailed).toBe(1);
|
||||
expect(summary.avgDurationMs).toBe(2000);
|
||||
expect(summary.successRate).toBeCloseTo(5 / 6, 10);
|
||||
expect(summary.commonErrors).toEqual(["timeout", "rate limit", "validation"]);
|
||||
expect(summary.strengths).toEqual([
|
||||
"Great at debugging",
|
||||
"Strong communication",
|
||||
"Clear task decomposition",
|
||||
]);
|
||||
expect(summary.weaknesses).toEqual([
|
||||
"Improve test coverage",
|
||||
"Tune model temperature",
|
||||
"Handle retries better",
|
||||
]);
|
||||
expect(summary.recentReflectionCount).toBe(2);
|
||||
expect(Number.isNaN(Date.parse(summary.computedAt))).toBe(false);
|
||||
});
|
||||
|
||||
it("returns a zeroed summary when no reflections exist", async () => {
|
||||
const summary = await store.getPerformanceSummary("agent-none");
|
||||
|
||||
expect(summary).toMatchObject({
|
||||
agentId: "agent-none",
|
||||
totalTasksCompleted: 0,
|
||||
totalTasksFailed: 0,
|
||||
avgDurationMs: 0,
|
||||
successRate: 0,
|
||||
commonErrors: [],
|
||||
strengths: [],
|
||||
weaknesses: [],
|
||||
recentReflectionCount: 0,
|
||||
});
|
||||
expect(Number.isNaN(Date.parse(summary.computedAt))).toBe(false);
|
||||
});
|
||||
|
||||
it("excludes reflections outside the default 7-day window", async () => {
|
||||
const agentId = "agent-window-default";
|
||||
const filePath = join(rootDir, "agents", `${agentId}-reflections.jsonl`);
|
||||
const now = Date.now();
|
||||
|
||||
const oldReflection = makeReflection(agentId, {
|
||||
id: "reflection-old",
|
||||
timestamp: new Date(now - 10 * 24 * 60 * 60 * 1000).toISOString(),
|
||||
metrics: { tasksCompleted: 10 },
|
||||
});
|
||||
|
||||
const recentReflection = makeReflection(agentId, {
|
||||
id: "reflection-recent",
|
||||
timestamp: new Date(now - 2 * 24 * 60 * 60 * 1000).toISOString(),
|
||||
metrics: { tasksCompleted: 2, tasksFailed: 1 },
|
||||
});
|
||||
|
||||
writeFileSync(filePath, `${JSON.stringify(oldReflection)}\n${JSON.stringify(recentReflection)}\n`, "utf-8");
|
||||
|
||||
const summary = await store.getPerformanceSummary(agentId);
|
||||
expect(summary.totalTasksCompleted).toBe(2);
|
||||
expect(summary.totalTasksFailed).toBe(1);
|
||||
expect(summary.recentReflectionCount).toBe(1);
|
||||
});
|
||||
|
||||
it("respects a custom windowMs option", async () => {
|
||||
const agentId = "agent-window-custom";
|
||||
const filePath = join(rootDir, "agents", `${agentId}-reflections.jsonl`);
|
||||
const now = Date.now();
|
||||
|
||||
const older = makeReflection(agentId, {
|
||||
id: "reflection-older",
|
||||
timestamp: new Date(now - 10_000).toISOString(),
|
||||
metrics: { tasksCompleted: 1 },
|
||||
});
|
||||
const newest = makeReflection(agentId, {
|
||||
id: "reflection-newest",
|
||||
timestamp: new Date(now - 200).toISOString(),
|
||||
metrics: { tasksCompleted: 2 },
|
||||
});
|
||||
|
||||
writeFileSync(filePath, `${JSON.stringify(older)}\n${JSON.stringify(newest)}\n`, "utf-8");
|
||||
|
||||
const summary = await store.getPerformanceSummary(agentId, { windowMs: 1000 });
|
||||
expect(summary.totalTasksCompleted).toBe(2);
|
||||
expect(summary.recentReflectionCount).toBe(1);
|
||||
});
|
||||
|
||||
it("emits reflection:summary-computed", async () => {
|
||||
const handler = vi.fn();
|
||||
store.on("reflection:summary-computed", handler);
|
||||
|
||||
const summary = await store.getPerformanceSummary("agent-summary-event");
|
||||
|
||||
expect(handler).toHaveBeenCalledOnce();
|
||||
expect(handler).toHaveBeenCalledWith(summary);
|
||||
});
|
||||
});
|
||||
|
||||
describe("deleteReflections", () => {
|
||||
it("removes the agent reflection file", async () => {
|
||||
const agentId = "agent-delete";
|
||||
await store.createReflection({
|
||||
agentId,
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: [],
|
||||
suggestedImprovements: [],
|
||||
summary: "to delete",
|
||||
});
|
||||
|
||||
const filePath = join(rootDir, "agents", `${agentId}-reflections.jsonl`);
|
||||
expect(existsSync(filePath)).toBe(true);
|
||||
|
||||
await store.deleteReflections(agentId);
|
||||
expect(existsSync(filePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("no-ops when the file does not exist", async () => {
|
||||
await expect(store.deleteReflections("agent-missing-delete")).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("concurrency", () => {
|
||||
it("allows concurrent createReflection calls for the same agent", async () => {
|
||||
const agentId = "agent-concurrent";
|
||||
const triggers: ReflectionTrigger[] = ["manual", "periodic", "post-task", "user-requested"];
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: 25 }, (_, i) =>
|
||||
store.createReflection({
|
||||
agentId,
|
||||
trigger: triggers[i % triggers.length],
|
||||
metrics: { tasksCompleted: 1 },
|
||||
insights: [`insight-${i}`],
|
||||
suggestedImprovements: [`improvement-${i}`],
|
||||
summary: `summary-${i}`,
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const reflections = await store.getReflections(agentId, 100);
|
||||
expect(reflections).toHaveLength(25);
|
||||
|
||||
const filePath = join(rootDir, "agents", `${agentId}-reflections.jsonl`);
|
||||
const lines = readFileSync(filePath, "utf-8").trim().split("\n");
|
||||
expect(lines).toHaveLength(25);
|
||||
});
|
||||
});
|
||||
|
||||
describe("append-only behavior", () => {
|
||||
it("preserves all reflections for the same agent in file order", async () => {
|
||||
const first = await store.createReflection({
|
||||
agentId: "agent-append-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["first"],
|
||||
suggestedImprovements: [],
|
||||
summary: "first",
|
||||
});
|
||||
const second = await store.createReflection({
|
||||
agentId: "agent-append-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["second"],
|
||||
suggestedImprovements: [],
|
||||
summary: "second",
|
||||
});
|
||||
const third = await store.createReflection({
|
||||
agentId: "agent-append-order",
|
||||
trigger: "manual",
|
||||
metrics: {},
|
||||
insights: ["third"],
|
||||
suggestedImprovements: [],
|
||||
summary: "third",
|
||||
});
|
||||
|
||||
const filePath = join(rootDir, "agents", "agent-append-order-reflections.jsonl");
|
||||
const ids = readFileSync(filePath, "utf-8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => (JSON.parse(line) as AgentReflection).id);
|
||||
|
||||
expect(ids).toEqual([first.id, second.id, third.id]);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user