feat(FN-1185): inject agent rating feedback into runtime instructions

- Extend agent instruction resolution to append a Performance Feedback section with average score, trend, category breakdown, and recent comments when ratings exist
- Add rating-aware instruction resolution with graceful fallback when no store is configured, agent IDs are missing, or rating lookup fails
- Wire executor custom instruction loading to fetch agent rating summaries and include them in resolved instructions
- Expand agent-instructions tests to cover feedback formatting, trend indicators, comment limits, and fallback/error paths
This commit is contained in:
gsxdsm
2026-04-08 06:18:47 -07:00
parent b949fd01ff
commit 14c0041e61
3 changed files with 312 additions and 4 deletions

View File

@@ -2,8 +2,12 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
import type { Agent } from "@fusion/core";
import { resolveAgentInstructions, buildSystemPromptWithInstructions } from "../agent-instructions.js";
import type { Agent, AgentRating, AgentRatingSummary, AgentStore } from "@fusion/core";
import {
resolveAgentInstructions,
resolveAgentInstructionsWithRatings,
buildSystemPromptWithInstructions,
} from "../agent-instructions.js";
function makeAgent(overrides: Partial<Agent> = {}): Agent {
return {
@@ -18,6 +22,29 @@ function makeAgent(overrides: Partial<Agent> = {}): Agent {
} as Agent;
}
function makeRating(overrides: Partial<AgentRating> = {}): AgentRating {
return {
id: "rating-1",
agentId: "agent-test",
raterType: "user",
score: 4,
createdAt: new Date().toISOString(),
...overrides,
};
}
function makeRatingSummary(overrides: Partial<AgentRatingSummary> = {}): AgentRatingSummary {
return {
agentId: "agent-test",
averageScore: 4,
totalRatings: 1,
categoryAverages: {},
recentRatings: [makeRating()],
trend: "stable",
...overrides,
};
}
describe("resolveAgentInstructions", () => {
let testDir: string;
@@ -179,6 +206,206 @@ describe("resolveAgentInstructions", () => {
});
});
describe("resolveAgentInstructions with rating summary", () => {
let testDir: string;
beforeEach(async () => {
testDir = await mkdtemp(join(tmpdir(), "agent-instr-ratings-"));
});
afterEach(async () => {
await rm(testDir, { recursive: true, force: true });
});
it("adds a performance feedback section when ratings exist", async () => {
const agent = makeAgent({ instructionsText: "Follow the task prompt." });
const summary = makeRatingSummary({
averageScore: 4.26,
totalRatings: 4,
trend: "improving",
categoryAverages: { quality: 4.5, speed: 3.75 },
recentRatings: [
makeRating({ id: "r1", score: 5, comment: "Great debugging discipline" }),
makeRating({ id: "r2", score: 4, comment: "Could communicate blockers sooner" }),
],
});
const result = await resolveAgentInstructions(agent, testDir, summary);
expect(result).toContain("Follow the task prompt.");
expect(result).toContain("## Performance Feedback");
expect(result).toContain("- Average score: 4.3");
expect(result).toContain("- Trend: 📈 improving");
expect(result).toContain("- Category breakdown:");
expect(result).toContain(" - quality: 4.5");
expect(result).toContain(" - speed: 3.8");
expect(result).toContain(' - "Great debugging discipline" (score: 5.0)');
expect(result).toContain(' - "Could communicate blockers sooner" (score: 4.0)');
});
it("shows the correct trend indicator for all trend states", async () => {
const agent = makeAgent({ instructionsText: "Base instructions" });
const trends: Array<[AgentRatingSummary["trend"], string]> = [
["improving", "📈 improving"],
["declining", "📉 declining"],
["stable", "➡️ stable"],
["insufficient-data", "❓ insufficient-data"],
];
for (const [trend, expected] of trends) {
const result = await resolveAgentInstructions(
agent,
testDir,
makeRatingSummary({ trend, totalRatings: 10 }),
);
expect(result).toContain(`- Trend: ${expected}`);
}
});
it("limits recent feedback to 3 comments and skips unrated comments", async () => {
const agent = makeAgent();
const summary = makeRatingSummary({
totalRatings: 8,
recentRatings: [
makeRating({ id: "r1", score: 5, comment: "Most recent note" }),
makeRating({ id: "r2", score: 4, comment: "Second note" }),
makeRating({ id: "r3", score: 3 }),
makeRating({ id: "r4", score: 2, comment: "Third note" }),
makeRating({ id: "r5", score: 1, comment: "Should be trimmed" }),
],
});
const result = await resolveAgentInstructions(agent, testDir, summary);
expect(result).toContain("- Recent feedback:");
expect(result).toContain(' - "Most recent note" (score: 5.0)');
expect(result).toContain(' - "Second note" (score: 4.0)');
expect(result).toContain(' - "Third note" (score: 2.0)');
expect(result).not.toContain("Should be trimmed");
});
it("omits category breakdown when category averages are empty", async () => {
const result = await resolveAgentInstructions(
makeAgent({ instructionsText: "Do work" }),
testDir,
makeRatingSummary({ totalRatings: 2, categoryAverages: {} }),
);
expect(result).not.toContain("- Category breakdown:");
});
it("does not add performance feedback when totalRatings is zero", async () => {
const result = await resolveAgentInstructions(
makeAgent({ instructionsText: "Do work" }),
testDir,
makeRatingSummary({ totalRatings: 0 }),
);
expect(result).toBe("Do work");
expect(result).not.toContain("## Performance Feedback");
});
it("does not add performance feedback when rating summary is undefined", async () => {
const result = await resolveAgentInstructions(
makeAgent({ instructionsText: "Do work" }),
testDir,
undefined,
);
expect(result).toBe("Do work");
expect(result).not.toContain("## Performance Feedback");
});
});
describe("resolveAgentInstructionsWithRatings", () => {
let testDir: string;
beforeEach(async () => {
testDir = await mkdtemp(join(tmpdir(), "agent-instr-with-ratings-"));
});
afterEach(async () => {
await rm(testDir, { recursive: true, force: true });
});
it("returns empty string for null agent", async () => {
const store = {
getRatingSummary: vi.fn(),
} as unknown as AgentStore;
const result = await resolveAgentInstructionsWithRatings(null, testDir, store);
expect(result).toBe("");
expect(store.getRatingSummary).not.toHaveBeenCalled();
});
it("returns base instructions when no agent store is provided", async () => {
const result = await resolveAgentInstructionsWithRatings(
makeAgent({ instructionsText: "Inline instructions" }),
testDir,
undefined,
);
expect(result).toBe("Inline instructions");
});
it("injects performance feedback when store returns ratings", async () => {
const store = {
getRatingSummary: vi.fn().mockResolvedValue(
makeRatingSummary({
averageScore: 3.333,
totalRatings: 3,
trend: "stable",
categoryAverages: { codeQuality: 4.95 },
recentRatings: [makeRating({ score: 4, comment: "Solid implementation" })],
}),
),
} as unknown as AgentStore;
const result = await resolveAgentInstructionsWithRatings(
makeAgent({ instructionsText: "Inline instructions" }),
testDir,
store,
);
expect(store.getRatingSummary).toHaveBeenCalledWith("agent-test");
expect(result).toContain("Inline instructions");
expect(result).toContain("## Performance Feedback");
expect(result).toContain("- Average score: 3.3");
expect(result).toContain(" - codeQuality: 5.0");
});
it("falls back to base instructions when rating lookup fails", async () => {
const store = {
getRatingSummary: vi.fn().mockRejectedValue(new Error("db unavailable")),
} as unknown as AgentStore;
const result = await resolveAgentInstructionsWithRatings(
makeAgent({ instructionsText: "Fallback instructions" }),
testDir,
store,
);
expect(store.getRatingSummary).toHaveBeenCalledWith("agent-test");
expect(result).toBe("Fallback instructions");
});
it("does not query ratings when agent id is empty", async () => {
const store = {
getRatingSummary: vi.fn(),
} as unknown as AgentStore;
const result = await resolveAgentInstructionsWithRatings(
makeAgent({ id: "", instructionsText: "Fallback instructions" }),
testDir,
store,
);
expect(store.getRatingSummary).not.toHaveBeenCalled();
expect(result).toBe("Fallback instructions");
});
});
describe("buildSystemPromptWithInstructions", () => {
it("returns base prompt when instructions are empty", () => {
const result = buildSystemPromptWithInstructions("Base prompt", "");

View File

@@ -1,6 +1,6 @@
import { readFile } from "node:fs/promises";
import { isAbsolute, resolve, relative, normalize, sep } from "node:path";
import type { Agent } from "@fusion/core";
import type { Agent, AgentRatingSummary, AgentStore } from "@fusion/core";
const MAX_INSTRUCTIONS_PATH_LENGTH = 500;
const MAX_INSTRUCTIONS_TEXT_LENGTH = 50_000;
@@ -63,6 +63,50 @@ function resolveValidatedInstructionsPath(rawPath: string, rootDir: string, agen
return resolvedPath;
}
function getTrendLabel(trend: AgentRatingSummary["trend"]): string {
switch (trend) {
case "improving":
return "📈 improving";
case "declining":
return "📉 declining";
case "stable":
return "➡️ stable";
case "insufficient-data":
default:
return "❓ insufficient-data";
}
}
function formatPerformanceFeedbackSection(ratingSummary: AgentRatingSummary): string {
const lines: string[] = [
"## Performance Feedback",
"",
`- Average score: ${ratingSummary.averageScore.toFixed(1)}`,
`- Trend: ${getTrendLabel(ratingSummary.trend)}`,
];
const categoryEntries = Object.entries(ratingSummary.categoryAverages);
if (categoryEntries.length > 0) {
lines.push("- Category breakdown:");
for (const [category, average] of categoryEntries.sort(([a], [b]) => a.localeCompare(b))) {
lines.push(` - ${category}: ${average.toFixed(1)}`);
}
}
const recentComments = ratingSummary.recentRatings
.filter((rating) => typeof rating.comment === "string" && rating.comment.trim().length > 0)
.slice(0, 3);
if (recentComments.length > 0) {
lines.push("- Recent feedback:");
for (const rating of recentComments) {
lines.push(` - \"${rating.comment?.trim()}\" (score: ${rating.score.toFixed(1)})`);
}
}
return lines.join("\n");
}
/**
* Resolve custom instructions for an agent by combining inline text and/or
* file-based instructions.
@@ -74,6 +118,7 @@ function resolveValidatedInstructionsPath(rawPath: string, rootDir: string, agen
export async function resolveAgentInstructions(
agent: Agent | null | undefined,
rootDir: string,
ratingSummary?: AgentRatingSummary,
): Promise<string> {
if (!agent) return "";
@@ -125,9 +170,40 @@ export async function resolveAgentInstructions(
}
}
if (ratingSummary && ratingSummary.totalRatings > 0) {
parts.push(formatPerformanceFeedbackSection(ratingSummary));
}
return parts.join("\n\n");
}
/**
* Resolve agent instructions and include performance ratings when available.
* Falls back gracefully to base instructions if ratings lookup fails.
*/
export async function resolveAgentInstructionsWithRatings(
agent: Agent | null | undefined,
rootDir: string,
agentStore: AgentStore | undefined,
): Promise<string> {
if (!agent) {
return "";
}
const baseInstructions = await resolveAgentInstructions(agent, rootDir);
if (!agentStore || !agent.id) {
return baseInstructions;
}
try {
const ratingSummary = await agentStore.getRatingSummary(agent.id);
return await resolveAgentInstructions(agent, rootDir, ratingSummary);
} catch {
return baseInstructions;
}
}
/**
* Append a custom instructions block to a base system prompt.
* If instructions are empty, returns the base prompt unchanged.

View File

@@ -581,7 +581,12 @@ export class TaskExecutor {
const agents = await this.options.agentStore.listAgents({ role: role as AgentCapability });
for (const agent of agents) {
if (agent.instructionsText || agent.instructionsPath) {
return await resolveAgentInstructions(agent, this.rootDir);
try {
const ratingSummary = await this.options.agentStore.getRatingSummary(agent.id);
return await resolveAgentInstructions(agent, this.rootDir, ratingSummary);
} catch {
return await resolveAgentInstructions(agent, this.rootDir);
}
}
}
} catch {