root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(FN-1016): add comment-aware triage generation and stale approval invalidation

Browse Source 
- Include user comments as context during AI triage spec generation and spec review
- Invalidate stale spec approvals when new user comments are added after approval
- Add lastApprovedAt tracking to task metadata for approval freshness detection
- Add comprehensive tests for comment-aware triage, review, and stale approval logic
- Add changeset for published package and update README with feature documentation
This commit is contained in:
gsxdsm
2026-04-05 23:04:27 -07:00
parent ec33691033
commit 992e525648
8 changed files with 640 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
packages/engine/src/reviewer.test.ts
View File

@@ -360,4 +360,130 @@ describe("REVIEWER_SYSTEM_PROMPT", () => {
"3+ different packages but wasn't split",
);
});
it("includes user comment coverage criterion in spec review format", () => {
expect(REVIEWER_SYSTEM_PROMPT).toContain("User comment coverage");
expect(REVIEWER_SYSTEM_PROMPT).toContain("missing coverage is a blocking REVISE");
});
});
describe("reviewStep — user comments in spec review", () => {
let mockedCreateHaiAgent: ReturnType<typeof vi.fn>;
beforeEach(() => {
mockedCreateHaiAgent = vi.fn().mockResolvedValue({
session: {
prompt: vi.fn(),
subscribe: vi.fn().mockImplementation((cb: any) => {
cb({
type: "message_update",
assistantMessageEvent: { type: "text_delta", delta: "### Verdict: APPROVE\n### Summary\nOK" },
});
}),
dispose: vi.fn(),
sessionManager: { getLeafId: vi.fn() },
},
} as any);
vi.mocked(createKbAgent).mockImplementation(mockedCreateHaiAgent);
});
it("includes user comments in spec review request", async () => {
let capturedPrompt = "";
mockedCreateHaiAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockImplementation(async (prompt: string) => {
capturedPrompt = prompt;
}),
subscribe: vi.fn().mockImplementation((cb: any) => {
cb({
type: "message_update",
assistantMessageEvent: { type: "text_delta", delta: "### Verdict: APPROVE\n### Summary\nOK" },
});
}),
dispose: vi.fn(),
},
} as any);
const userComments = [
{
id: "c1",
text: "Make sure to handle the edge case",
author: "user",
createdAt: "2026-01-02T10:00:00.000Z",
},
];
await reviewStep(
"/tmp/worktree", "FN-050", 0, "Specification", "spec",
"# Task: FN-050\n\n## Mission\nDo something",
undefined,
{ userComments },
);
expect(capturedPrompt).toContain("User Comment Coverage (MANDATORY)");
expect(capturedPrompt).toContain("Make sure to handle the edge case");
expect(capturedPrompt).toContain("issue a REVISE verdict");
});
it("does not include user comments section when no comments provided", async () => {
let capturedPrompt = "";
mockedCreateHaiAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockImplementation(async (prompt: string) => {
capturedPrompt = prompt;
}),
subscribe: vi.fn().mockImplementation((cb: any) => {
cb({
type: "message_update",
assistantMessageEvent: { type: "text_delta", delta: "### Verdict: APPROVE\n### Summary\nOK" },
});
}),
dispose: vi.fn(),
},
} as any);
await reviewStep(
"/tmp/worktree", "FN-050", 0, "Specification", "spec",
"# Task: FN-050\n\n## Mission\nDo something",
);
expect(capturedPrompt).not.toContain("User Comment Coverage");
});
it("does not include user comments for non-spec review types", async () => {
let capturedPrompt = "";
mockedCreateHaiAgent.mockResolvedValue({
session: {
prompt: vi.fn().mockImplementation(async (prompt: string) => {
capturedPrompt = prompt;
}),
subscribe: vi.fn().mockImplementation((cb: any) => {
cb({
type: "message_update",
assistantMessageEvent: { type: "text_delta", delta: "### Verdict: APPROVE\n### Summary\nOK" },
});
}),
dispose: vi.fn(),
},
} as any);
const userComments = [
{
id: "c1",
text: "Some user feedback",
author: "user",
createdAt: "2026-01-02T10:00:00.000Z",
},
];
await reviewStep(
"/tmp/worktree", "FN-050", 1, "Implementation", "code",
"# Task: FN-050\n\n## Mission\nDo something",
"abc123",
{ userComments },
);
// Code reviews should not have user comment coverage checks
expect(capturedPrompt).not.toContain("User Comment Coverage");
});
});

27
packages/engine/src/reviewer.ts
View File

@@ -8,7 +8,7 @@
* - Verdict + feedback is returned to the worker
*/
import type { TaskStore } from "@fusion/core";
import type { TaskStore, TaskComment } from "@fusion/core";
import { createKbAgent, describeModel, promptWithFallback } from "./pi.js";
import { AgentLogger } from "./agent-logger.js";
import { reviewerLog } from "./logger.js";
@@ -113,6 +113,7 @@ access to the codebase and can run commands to inspect code.
- **Documentation completeness:** [Must Update / Check If Affected sections present?]
- **Sizing & review level:** [Size and review level appropriate for the work?]
- **Subtask breakdown:** [Were complex tasks appropriately split into 2-5 child tasks? A task with 8+ implementation steps, affecting 3+ packages, should have been divided]
- **User comment coverage:** [Were all user comments addressed? Every user comment must be reflected in the spec — missing coverage is a blocking REVISE]
### Suggestions
- [Optional improvements, not blocking]
@@ -185,6 +186,8 @@ export interface ReviewOptions {
store?: TaskStore;
/** Task ID for agent log persistence. Required alongside `store`. */
taskId?: string;
/** User comments on the task (author === "user"). For spec reviews, the reviewer explicitly checks that every comment is addressed. */
userComments?: TaskComment[];
}
/**
@@ -202,7 +205,7 @@ export async function reviewStep(
): Promise<ReviewResult> {
// Build the review request
const request = buildReviewRequest(
taskId, stepNumber, stepName, reviewType, promptContent, cwd, baseline,
taskId, stepNumber, stepName, reviewType, promptContent, cwd, baseline, options.userComments,
);
// Create AgentLogger for reviewer if store is available
@@ -289,6 +292,7 @@ function buildReviewRequest(
promptContent: string,
_cwd: string,
baseline?: string,
userComments?: TaskComment[],
): string {
const parts = [
`Review request for task ${taskId}, Step ${stepNumber}: ${stepName}`,
@@ -313,6 +317,25 @@ function buildReviewRequest(
"Check that steps have concrete, verifiable outcomes — not vague instructions.",
"Ensure testing requirements demand real automated tests with assertions.",
);
// Add user comment coverage check for spec reviews
if (userComments && userComments.length > 0) {
parts.push(
"",
"## User Comment Coverage (MANDATORY)",
"",
"The following user comments were posted on this task. You MUST verify that the spec addresses **every** comment. If any user comment is not reflected or addressed in the PROMPT.md, issue a REVISE verdict.",
"",
);
for (const comment of userComments) {
const date = comment.updatedAt || comment.createdAt;
parts.push(`- **[${date}]** ${comment.text}`);
}
parts.push(
"",
"Check each comment above against the spec content. Missing coverage for any user comment is a blocking issue.",
);
}
} else if (reviewType === "plan") {
parts.push(
"## What to review",

334
packages/engine/src/triage.test.ts
View File

@@ -1,10 +1,11 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import type { TaskStore, Task, TaskDetail, Settings } from "@fusion/core";
import {
TriageProcessor,
TRIAGE_SYSTEM_PROMPT,
buildSpecificationPrompt,
readAttachmentContents,
computeUserCommentFingerprint,
} from "./triage.js";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
@@ -260,6 +261,105 @@ describe("buildSpecificationPrompt", () => {
expect(prompt).toContain("Specify this task");
});
});
describe("user comments", () => {
it("includes user comments section when user comments exist", () => {
const taskWithComments: TaskDetail = {
...baseTask,
comments: [
{
id: "c1",
text: "Please add error handling for edge cases",
author: "user",
createdAt: "2026-01-02T10:00:00.000Z",
updatedAt: "2026-01-02T10:00:00.000Z",
},
{
id: "c2",
text: "Make sure to update the README too",
author: "user",
createdAt: "2026-01-02T11:00:00.000Z",
},
],
};
const prompt = buildSpecificationPrompt(
taskWithComments,
".fusion/tasks/KB-001/PROMPT.md",
);
expect(prompt).toContain("## User Comments");
expect(prompt).toContain("Please add error handling for edge cases");
expect(prompt).toContain("Make sure to update the README too");
expect(prompt).toContain("Address every comment");
expect(prompt).toContain("Missing comment coverage is a spec quality failure");
});
it("excludes agent/system comments from user comments section", () => {
const taskWithMixedComments: TaskDetail = {
...baseTask,
comments: [
{
id: "c1",
text: "User feedback here",
author: "user",
createdAt: "2026-01-02T10:00:00.000Z",
},
{
id: "c2",
text: "Agent system note",
author: "agent",
createdAt: "2026-01-02T11:00:00.000Z",
},
{
id: "c3",
text: "System auto-message",
author: "system",
createdAt: "2026-01-02T12:00:00.000Z",
},
],
};
const prompt = buildSpecificationPrompt(
taskWithMixedComments,
".fusion/tasks/KB-001/PROMPT.md",
);
expect(prompt).toContain("User feedback here");
expect(prompt).not.toContain("Agent system note");
expect(prompt).not.toContain("System auto-message");
});
it("does not include user comments section when no comments exist", () => {
const prompt = buildSpecificationPrompt(
baseTask,
".fusion/tasks/KB-001/PROMPT.md",
);
expect(prompt).not.toContain("## User Comments");
});
it("does not include user comments section when only agent comments exist", () => {
const taskWithOnlyAgentComments: TaskDetail = {
...baseTask,
comments: [
{
id: "c1",
text: "Agent note",
author: "agent",
createdAt: "2026-01-02T10:00:00.000Z",
},
],
};
const prompt = buildSpecificationPrompt(
taskWithOnlyAgentComments,
".fusion/tasks/KB-001/PROMPT.md",
);
expect(prompt).not.toContain("## User Comments");
});
});
});
describe("TRIAGE_SYSTEM_PROMPT", () => {
@@ -483,6 +583,11 @@ describe("TriageProcessor", () => {
store = createMockStore({
getSettings: vi.fn().mockResolvedValue(freshSettings),
getTask: vi.fn().mockResolvedValue({
...mockTaskDetail,
id: taskId,
comments: [],
}),
});
processor = new TriageProcessor(store, testRootDir);
@@ -522,6 +627,7 @@ describe("TriageProcessor", () => {
defaultModelId: "gpt-5.4",
validatorModelProvider: "zai",
validatorModelId: "glm-5.1",
userComments: undefined,
}),
);
@@ -1002,3 +1108,229 @@ describe("taskCreate tool model inheritance", () => {
});
});
});
describe("computeUserCommentFingerprint", () => {
it("returns empty string for undefined comments", () => {
expect(computeUserCommentFingerprint(undefined)).toBe("");
});
it("returns empty string for empty comments array", () => {
expect(computeUserCommentFingerprint([])).toBe("");
});
it("returns empty string when only agent comments exist", () => {
const comments = [
{ id: "c1", text: "agent note", author: "agent", createdAt: "2026-01-01T00:00:00.000Z" },
];
expect(computeUserCommentFingerprint(comments as any)).toBe("");
});
it("returns sorted semicolon-joined IDs for user comments", () => {
const comments = [
{ id: "c3", text: "user 3", author: "user", createdAt: "2026-01-01T00:00:00.000Z" },
{ id: "c1", text: "user 1", author: "user", createdAt: "2026-01-01T00:00:00.000Z" },
{ id: "c2", text: "agent", author: "agent", createdAt: "2026-01-01T00:00:00.000Z" },
];
// Should be sorted: c1;c3 (c2 is agent, excluded)
expect(computeUserCommentFingerprint(comments as any)).toBe("c1;c3");
});
it("detects changed fingerprint when new user comment is added", () => {
const before = [
{ id: "c1", text: "user 1", author: "user", createdAt: "2026-01-01T00:00:00.000Z" },
];
const after = [
{ id: "c1", text: "user 1", author: "user", createdAt: "2026-01-01T00:00:00.000Z" },
{ id: "c2", text: "user 2", author: "user", createdAt: "2026-01-01T00:00:00.000Z" },
];
expect(computeUserCommentFingerprint(before as any)).not.toBe(
computeUserCommentFingerprint(after as any),
);
});
});
describe("awaiting-approval poll exclusion", () => {
it("excludes awaiting-approval tasks from poll discovery", async () => {
const awaitingTask: Task = {
id: "FN-AW1",
description: "Awaiting approval task",
column: "triage",
status: "awaiting-approval",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-01T00:00:00.000Z",
};
const normalTask: Task = {
id: "FN-NT1",
description: "Normal triage task",
column: "triage",
dependencies: [],
steps: [],
currentStep: 0,
log: [],
createdAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-01T00:00:00.000Z",
};
const specifySpy = vi.fn();
const store = createMockStore({
listTasks: vi.fn().mockResolvedValue([awaitingTask, normalTask]),
getSettings: vi.fn().mockResolvedValue({
maxConcurrent: 2,
maxWorktrees: 4,
pollIntervalMs: 60000,
groupOverlappingFiles: false,
autoMerge: true,
}),
});
const processor = new TriageProcessor(store, "/tmp");
// Mark as running so poll() proceeds
(processor as any).running = true;
// Override specifyTask to spy on which tasks get dispatched
(processor as any).specifyTask = specifySpy;
// Trigger poll via private method
await (processor as any).poll();
// Only the normal task should have been dispatched
expect(specifySpy).toHaveBeenCalledTimes(1);
expect(specifySpy).toHaveBeenCalledWith(normalTask);
});
});
describe("stale approval detection", () => {
it("computeUserCommentFingerprint detects added user comment", () => {
const before = [
{ id: "c1", text: "First", author: "user", createdAt: "2026-01-01T00:00:00.000Z" },
];
const after = [
{ id: "c1", text: "First", author: "user", createdAt: "2026-01-01T00:00:00.000Z" },
{ id: "c2", text: "Second", author: "user", createdAt: "2026-01-02T00:00:00.000Z" },
];
const fpBefore = computeUserCommentFingerprint(before as any);
const fpAfter = computeUserCommentFingerprint(after as any);
expect(fpBefore).toBe("c1");
expect(fpAfter).toBe("c1;c2");
expect(fpBefore).not.toBe(fpAfter);
});
it("computeUserCommentFingerprint is stable when comments unchanged", () => {
const comments = [
{ id: "c1", text: "Same", author: "user", createdAt: "2026-01-01T00:00:00.000Z" },
];
const fp1 = computeUserCommentFingerprint(comments as any);
const fp2 = computeUserCommentFingerprint(comments as any);
expect(fp1).toBe(fp2);
});
it("captures fingerprint on review_spec APPROVE", async () => {
const rootDir = join(__dirname, "__test_stale_approval_capture__");
const taskId = "FN-CAP";
const taskDir = join(rootDir, ".fusion", "tasks", taskId);
await mkdir(taskDir, { recursive: true });
await writeFile(join(taskDir, "PROMPT.md"), "# Spec\n\nCurrent prompt");
const comments = [
{ id: "c1", text: "Feedback", author: "user", createdAt: "2026-01-01T00:00:00.000Z" },
];
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({
maxConcurrent: 2,
maxWorktrees: 4,
pollIntervalMs: 10000,
groupOverlappingFiles: false,
autoMerge: true,
} as Settings),
getTask: vi.fn().mockResolvedValue({
...mockTaskDetail,
id: taskId,
comments,
}),
});
const processor = new TriageProcessor(store, rootDir);
mockReviewStep.mockResolvedValue({
verdict: "APPROVE",
review: "Looks good.",
summary: "approved",
});
const approvedCommentFingerprintRef = { current: "" };
const tool = (processor as any).createReviewSpecTool(
taskId,
`.fusion/tasks/${taskId}/PROMPT.md`,
{ current: null },
{ current: null },
{ current: null },
approvedCommentFingerprintRef,
{},
);
// Execute review_spec — should capture fingerprint at APPROVE time
await tool.execute({});
// Verify fingerprint was captured from the user comments at approval time
expect(approvedCommentFingerprintRef.current).toBe("c1");
await rm(rootDir, { recursive: true, force: true });
});
it("fingerprint is empty string when review_spec returns REVISE (no capture)", async () => {
const rootDir = join(__dirname, "__test_stale_approval_revise__");
const taskId = "FN-REV";
const taskDir = join(rootDir, ".fusion", "tasks", taskId);
await mkdir(taskDir, { recursive: true });
await writeFile(join(taskDir, "PROMPT.md"), "# Spec\n\nCurrent prompt");
const store = createMockStore({
getSettings: vi.fn().mockResolvedValue({
maxConcurrent: 2,
maxWorktrees: 4,
pollIntervalMs: 10000,
groupOverlappingFiles: false,
autoMerge: true,
} as Settings),
getTask: vi.fn().mockResolvedValue({
...mockTaskDetail,
id: taskId,
comments: [],
}),
});
const processor = new TriageProcessor(store, rootDir);
mockReviewStep.mockResolvedValue({
verdict: "REVISE",
review: "Fix the spec.",
summary: "needs work",
});
const approvedCommentFingerprintRef = { current: "" };
const tool = (processor as any).createReviewSpecTool(
taskId,
`.fusion/tasks/${taskId}/PROMPT.md`,
{ current: null },
{ current: null },
{ current: null },
approvedCommentFingerprintRef,
{},
);
await tool.execute({});
// Fingerprint should NOT be captured on REVISE
expect(approvedCommentFingerprintRef.current).toBe("");
await rm(rootDir, { recursive: true, force: true });
});
});

79
packages/engine/src/triage.ts
View File

@@ -399,6 +399,8 @@ export class TriageProcessor {
const now = Date.now();
const triageTasks = tasks.filter(
(t) => t.column === "triage" && !this.processing.has(t.id) && !t.paused
// Skip tasks awaiting manual plan approval — they should not be auto-discovered
&& t.status !== "awaiting-approval"
// Skip tasks with a recovery backoff that hasn't elapsed yet
&& !(t.nextRecoveryAt && new Date(t.nextRecoveryAt).getTime() > now),
);
@@ -465,6 +467,10 @@ export class TriageProcessor {
const specReviewVerdictRef: { current: ReviewVerdict | null } = {
current: null,
};
// Track the user-comment fingerprint at the time of APPROVE for stale-approval detection
const approvedCommentFingerprintRef: { current: string } = {
current: "",
};
// Track subtasks created during triage when breakIntoSubtasks was requested.
const createdSubtasksRef: { current: string[] } = { current: [] };
@@ -480,6 +486,7 @@ export class TriageProcessor {
sessionRef,
checkpointRef,
specReviewVerdictRef,
approvedCommentFingerprintRef,
settings,
),
];
@@ -610,6 +617,25 @@ export class TriageProcessor {
// Check if the agent flagged a duplicate
const { readFile } = await import("node:fs/promises");
const { join } = await import("node:path");
// Stale-approval detection: re-read the task to check if new user
// comments arrived after the spec was approved. If the comment
// fingerprint changed, the approval is stale and the task needs
// re-specification.
const latestTask = await this.store.getTask(task.id);
const currentFingerprint = computeUserCommentFingerprint(latestTask.comments);
if (currentFingerprint !== approvedCommentFingerprintRef.current) {
triageLog.log(
`${task.id} stale approval detected — user comments changed after approval, triggering re-specification`,
);
await this.store.logEntry(
task.id,
"Spec approval invalidated — new user comments arrived after approval. Task needs re-specification.",
);
await this.store.updateTask(task.id, { status: "needs-respecify" });
return;
}
const written = await readFile(
join(this.rootDir, promptPath),
"utf-8",
@@ -949,6 +975,7 @@ export class TriageProcessor {
sessionRef: { current: AgentSession | null },
checkpointRef: { current: string | null },
specReviewVerdictRef: { current: ReviewVerdict | null },
approvedCommentFingerprintRef: { current: string },
settings: {
defaultProvider?: string;
defaultModelId?: string;
@@ -1005,6 +1032,12 @@ export class TriageProcessor {
// model changes made after the session started.
const currentSettings = await store.getSettings();
// Re-read task detail to get latest user comments for the reviewer
const currentDetail = await store.getTask(taskId);
const currentUserComments = (currentDetail.comments || []).filter(
(c: any) => c.author === "user",
);
const result = await reviewStep(
rootDir,
taskId,
@@ -1022,6 +1055,7 @@ export class TriageProcessor {
defaultThinkingLevel: currentSettings.defaultThinkingLevel,
store,
taskId,
userComments: currentUserComments.length > 0 ? currentUserComments : undefined,
},
);
@@ -1038,6 +1072,8 @@ export class TriageProcessor {
let text: string;
switch (result.verdict) {
case "APPROVE":
// Capture the user-comment fingerprint at approval time for stale-approval detection
approvedCommentFingerprintRef.current = computeUserCommentFingerprint(currentUserComments);
text = "APPROVE";
break;
case "REVISE":
@@ -1189,6 +1225,22 @@ export async function readAttachmentContents(
return { attachmentContents, imageContents };
}
/**
* Compute a deterministic fingerprint from user comments on a task.
* Returns a sorted, semicolon-joined string of comment IDs (user-authored only).
* Used to detect whether user comments changed after spec approval.
*/
export function computeUserCommentFingerprint(
comments?: import("@fusion/core").TaskComment[],
): string {
if (!comments || comments.length === 0) return "";
const userIds = comments
.filter((c) => c.author === "user")
.map((c) => c.id)
.sort();
return userIds.join(";");
}
export function buildSpecificationPrompt(
task: TaskDetail,
promptPath: string,
@@ -1239,6 +1291,31 @@ export function buildSpecificationPrompt(
attachmentsSection = "\n\n" + parts.join("\n");
}
// Include user comments as context for the triage agent
let userCommentsSection = "";
const userComments = (task.comments || []).filter(
(c) => c.author === "user",
);
if (userComments.length > 0) {
const parts = [
"## User Comments",
"",
"The following user comments have been posted on this task. **Address every comment** in the specification — each comment represents explicit user feedback or requirements that must be reflected in the PROMPT.md.",
"",
];
for (const comment of userComments) {
const date = comment.updatedAt || comment.createdAt;
parts.push(
`- **[${date}]** ${comment.text}`,
);
}
parts.push(
"",
"Ensure the specification addresses all of the above comments. Missing comment coverage is a spec quality failure.",
);
userCommentsSection = "\n\n" + parts.join("\n");
}
let revisionSection = "";
if (isRevision) {
revisionSection = `
@@ -1319,5 +1396,5 @@ ${task.dependencies.length > 0 ? `- **Dependencies:** ${task.dependencies.join("
## Instructions
${isRevision ? "1. Review the existing specification and user feedback carefully\n2. Revise the PROMPT.md to address the feedback while maintaining the structure\n3. Ensure the specification is detailed enough for an AI agent to execute" : "1. Read the project structure to understand context (package.json, source files, etc.)\n2. Write a complete PROMPT.md specification to the given path following the format in your system prompt\n3. The specification must be detailed enough for an autonomous AI agent to implement without asking questions\n4. Name actual files, functions, and patterns from the codebase — be specific"}
Use the write tool to write the specification file.${commandsSection}${memorySection}${attachmentsSection}`;
Use the write tool to write the specification file.${commandsSection}${memorySection}${attachmentsSection}${userCommentsSection}`;
}