feat(FN-1434): add prompt overrides for agent generation

- Add prompt-overrides module with template resolution and instruction injection
- Wire prompt overrides into agent generation flow via POST /api/agents route
- Support template-based prompt customization with role-based assignments
- Add agent generation tests and routes tests
- Document new prompt override settings in settings reference
This commit is contained in:
gsxdsm
2026-04-10 12:49:38 -07:00
parent d9acc10c49
commit d61e3c59fd
6 changed files with 333 additions and 20 deletions

View File

@@ -176,6 +176,8 @@ Fusion supports fine-grained customization of AI agent prompts through the `prom
| `triage-context` | triage | Context-gathering instructions |
| `reviewer-verdict` | reviewer | Verdict criteria and format |
| `merger-conflicts` | merger | Merge conflict resolution instructions |
| `agent-generation-system` | — | System prompt for AI-assisted agent specification generation |
| `workflow-step-refine` | — | System prompt for refining workflow step descriptions into detailed agent prompts |
### How It Works

View File

@@ -34,7 +34,9 @@ export type PromptKey =
| "triage-welcome"
| "triage-context"
| "reviewer-verdict"
| "merger-conflicts";
| "merger-conflicts"
| "agent-generation-system"
| "workflow-step-refine";
/**
* Metadata describing a prompt key including its purpose and default content.
@@ -172,6 +174,83 @@ If there are merge conflicts:
5. Run \`git add <file>\` for each resolved file
6. Do NOT change anything beyond what's needed to resolve the conflict`,
},
"agent-generation-system": {
key: "agent-generation-system",
name: "Agent Generation System",
roles: [],
description: "System prompt for the AI agent that generates agent specifications from role descriptions",
defaultContent: `You are an agent specification generator for the fn task board system.
Your job: given a user-provided role description, generate a complete agent specification suitable for creating an AI agent.
## Input
The user will provide a role description like:
- "Senior frontend code reviewer who specializes in React accessibility"
- "Security-focused DevOps engineer"
- "Performance optimization specialist for Node.js applications"
## Output
You MUST respond with ONLY valid JSON (no markdown, no explanation):
{
"title": "A concise display name (max 60 chars)",
"icon": "A single emoji representing the agent",
"role": "The most appropriate capability: triage | executor | reviewer | merger | scheduler | engineer | custom",
"description": "A brief 1-2 sentence description of the agent's purpose and expertise",
"systemPrompt": "A detailed markdown system prompt for the agent. This should be comprehensive and include:\\n- Role definition\\n- Core responsibilities\\n- Specific areas of expertise\\n- Behavioral guidelines\\n- Output format expectations\\n- Edge case handling instructions",
"thinkingLevel": "off | minimal | low | medium | high",
"maxTurns": 10
}
## Guidelines for System Prompt Generation
- Be specific about the agent's domain expertise
- Include concrete behavioral rules and constraints
- Define the expected output format clearly
- Add error handling and edge case guidance
- Keep the prompt focused and actionable (aim for 200-800 words)
- Use markdown formatting for readability
## Thinking Level Guidelines
- "off": For simple, well-defined tasks (basic CRUD, simple checks)
- "minimal": For straightforward tasks requiring some reasoning
- "low": For moderate complexity tasks
- "medium": For complex analysis, code review, architecture decisions
- "high": For critical decisions, security analysis, complex debugging
## Max Turns Guidelines
- 5-10: Simple, focused tasks (quick reviews, status checks)
- 10-25: Standard tasks (code review, feature planning)
- 25-50: Complex tasks (multi-file changes, architecture analysis)
- 50+: Extended tasks (large refactors, comprehensive audits)
## Role Selection Guidelines
- "reviewer": Agents focused on reviewing, auditing, analyzing
- "executor": Agents that perform implementation work
- "engineer": Agents that do engineering work with broader scope
- "triage": Agents focused on classification and routing
- "custom": Any agent that doesn't fit standard roles
- Default to "custom" if unclear`,
},
"workflow-step-refine": {
key: "workflow-step-refine",
name: "Workflow Step Refine",
roles: [],
description: "System prompt for refining workflow step descriptions into detailed agent prompts",
defaultContent: `You are an expert at creating detailed agent prompts for workflow steps.
A workflow step is a quality gate that runs after a task is implemented but before it's marked complete.
Given a rough description, create a detailed prompt that an AI agent can follow to execute this workflow step.
The prompt should:
1. Define the purpose clearly
2. Specify what files/context to examine
3. List specific criteria to check
4. Describe what "success" looks like
5. Include guidance on handling common edge cases
Output ONLY the prompt text (no markdown, no explanations).`,
},
};
/**

View File

@@ -335,4 +335,89 @@ describe("agent-generation module", () => {
expect(retrieved!.roleDescription).toBe("Security auditor role");
});
});
describe("prompt override support", () => {
// Mock createKbAgent to capture the systemPrompt passed to it
let capturedSystemPrompt: string | undefined;
beforeEach(async () => {
capturedSystemPrompt = undefined;
// Mock createKbAgent before tests run
vi.doMock("@fusion/engine", () => ({
createKbAgent: vi.fn(async (options: { cwd: string; systemPrompt: string; tools: string }) => {
capturedSystemPrompt = options.systemPrompt;
return {
session: {
state: { messages: [] },
prompt: vi.fn(async () => {}),
dispose: vi.fn(),
},
};
}),
}));
// Reset the module to pick up the mock
vi.resetModules();
});
afterEach(() => {
vi.restoreAllMocks();
vi.resetModules();
});
it("generates spec with default system prompt when no overrides provided", async () => {
const { generateAgentSpec: genSpec, startAgentGeneration: startGen } = await import("./agent-generation.js");
const session = await startGen(getUniqueIp(), "Test role");
const spec = await genSpec(session.id, "/tmp");
// The spec should be empty since we mocked an empty response
expect(spec).toBeDefined();
// The system prompt should be the default
expect(capturedSystemPrompt).toBeDefined();
expect(capturedSystemPrompt).toContain("agent specification generator");
});
it("generates spec with override system prompt when overrides provided", async () => {
const { generateAgentSpec: genSpec, startAgentGeneration: startGen } = await import("./agent-generation.js");
const customPrompt = "CUSTOM AGENT GENERATION PROMPT";
const overrides = { "agent-generation-system": customPrompt };
const session = await startGen(getUniqueIp(), "Test role");
const spec = await genSpec(session.id, "/tmp", overrides);
// The spec should be empty since we mocked an empty response
expect(spec).toBeDefined();
// The system prompt should be the custom override
expect(capturedSystemPrompt).toBe(customPrompt);
});
it("falls back to default when override key not recognized", async () => {
const { generateAgentSpec: genSpec, startAgentGeneration: startGen } = await import("./agent-generation.js");
// Provide an override with a non-existent key
const overrides = { "non-existent-key": "Some prompt" };
const session = await startGen(getUniqueIp(), "Test role");
const spec = await genSpec(session.id, "/tmp", overrides);
// Should fall back to default
expect(spec).toBeDefined();
expect(capturedSystemPrompt).toBeDefined();
expect(capturedSystemPrompt).toContain("agent specification generator");
});
it("falls back to AGENT_GENERATION_SYSTEM_PROMPT constant when resolvePrompt returns empty", async () => {
const { generateAgentSpec: genSpec, startAgentGeneration: startGen, AGENT_GENERATION_SYSTEM_PROMPT } = await import("./agent-generation.js");
// Empty overrides should still get the default constant
const overrides = { "agent-generation-system": "" };
const session = await startGen(getUniqueIp(), "Test role");
const spec = await genSpec(session.id, "/tmp", overrides);
expect(spec).toBeDefined();
expect(capturedSystemPrompt).toBe(AGENT_GENERATION_SYSTEM_PROMPT);
});
});
});

View File

@@ -14,6 +14,31 @@
import { randomUUID } from "node:crypto";
// Dynamic import for @fusion/core to get prompt override resolution
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type PromptOverrideMap = Record<string, string | undefined>;
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
type ResolvePromptFn = (key: string, overrides?: PromptOverrideMap) => string;
let resolvePrompt: ResolvePromptFn = () => "";
let promptCatalogReady = false;
async function initPromptCatalog() {
if (promptCatalogReady) return;
try {
const core = await import("@fusion/core");
resolvePrompt = (key: string, overrides?: PromptOverrideMap) =>
core.resolvePrompt(key as keyof typeof core.PROMPT_KEY_CATALOG, overrides);
promptCatalogReady = true;
} catch {
// Use fallback resolution when core is unavailable
resolvePrompt = () => "";
promptCatalogReady = true;
}
}
// Initialize prompt catalog (will be awaited in actual usage)
const promptCatalogReadyPromise = initPromptCatalog();
// Dynamic import for @fusion/engine to avoid resolution issues in test environment
// eslint-disable-next-line @typescript-eslint/consistent-type-imports, @typescript-eslint/no-explicit-any
type AgentResult = any;
@@ -424,11 +449,13 @@ export async function startAgentGeneration(
*
* @param sessionId - The session identifier
* @param rootDir - Project root directory for AI agent context
* @param promptOverrides - Optional prompt overrides from project settings
* @returns The generated agent specification
*/
export async function generateAgentSpec(
sessionId: string,
rootDir: string
rootDir: string,
promptOverrides?: PromptOverrideMap
): Promise<AgentGenerationSpec> {
const session = sessions.get(sessionId);
if (!session) {
@@ -437,7 +464,8 @@ export async function generateAgentSpec(
try {
await engineReady;
const spec = await generateSpecWithAI(session, rootDir);
await promptCatalogReadyPromise;
const spec = await generateSpecWithAI(session, rootDir, promptOverrides);
session.spec = spec;
session.updatedAt = new Date();
return spec;
@@ -450,14 +478,21 @@ export async function generateAgentSpec(
/**
* Generate an agent specification using the AI agent.
*/
async function generateSpecWithAI(session: Session, rootDir: string): Promise<AgentGenerationSpec> {
async function generateSpecWithAI(
session: Session,
rootDir: string,
promptOverrides?: PromptOverrideMap
): Promise<AgentGenerationSpec> {
if (!createKbAgent) {
throw new Error("AI agent not available. Ensure the engine is properly configured.");
}
// Resolve the system prompt using prompt overrides (with fallback to default)
const effectiveSystemPrompt = resolvePrompt("agent-generation-system", promptOverrides) || AGENT_GENERATION_SYSTEM_PROMPT;
const agent = await createKbAgent({
cwd: rootDir,
systemPrompt: AGENT_GENERATION_SYSTEM_PROMPT,
systemPrompt: effectiveSystemPrompt,
tools: "none",
});

View File

@@ -9946,6 +9946,89 @@ describe("POST /workflow-steps/:id/refine", () => {
expect(res.body.workflowStep.prompt).toBe("Check docs");
expect(store.updateWorkflowStep).toHaveBeenCalledWith("WS-001", { prompt: "Check docs" });
});
it("uses custom prompt from promptOverrides when provided", async () => {
const ws = { id: "WS-001", name: "Docs", description: "Check docs", mode: "prompt", prompt: "", enabled: true, createdAt: "2026-01-01", updatedAt: "2026-01-01" };
(store.getWorkflowStep as ReturnType<typeof vi.fn>).mockResolvedValueOnce(ws);
const customPrompt = "CUSTOM WORKFLOW STEP REFINE PROMPT";
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
promptOverrides: {
"workflow-step-refine": customPrompt,
},
});
const updatedWs = { ...ws, prompt: "Refined prompt from AI" };
(store.updateWorkflowStep as ReturnType<typeof vi.fn>).mockResolvedValueOnce(updatedWs);
let capturedSystemPrompt: string | undefined;
const session = {
on: vi.fn((event: string, cb: (delta: string) => void) => {
if (event === "text") {
cb("Refined ");
cb("prompt from AI");
}
}),
prompt: vi.fn(async () => {}),
dispose: vi.fn(),
};
const createKbAgentMock = vi.fn(async (options: { cwd: string; systemPrompt: string; tools: string }) => {
capturedSystemPrompt = options.systemPrompt;
return { session };
});
__setCreateKbAgentForRefine(createKbAgentMock);
const res = await REQUEST(buildApp(), "POST", "/api/workflow-steps/WS-001/refine", JSON.stringify({}), {
"Content-Type": "application/json",
});
expect(res.status).toBe(200);
expect(createKbAgentMock).toHaveBeenCalledTimes(1);
// Verify the custom prompt was passed
expect(capturedSystemPrompt).toBe(customPrompt);
});
it("uses default prompt when promptOverrides does not contain workflow-step-refine", async () => {
const ws = { id: "WS-001", name: "Docs", description: "Check docs", mode: "prompt", prompt: "", enabled: true, createdAt: "2026-01-01", updatedAt: "2026-01-01" };
(store.getWorkflowStep as ReturnType<typeof vi.fn>).mockResolvedValueOnce(ws);
// Settings with other overrides but not workflow-step-refine
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
promptOverrides: {
"executor-welcome": "Some other prompt",
},
});
const updatedWs = { ...ws, prompt: "Refined prompt from AI" };
(store.updateWorkflowStep as ReturnType<typeof vi.fn>).mockResolvedValueOnce(updatedWs);
let capturedSystemPrompt: string | undefined;
const session = {
on: vi.fn((event: string, cb: (delta: string) => void) => {
if (event === "text") {
cb("Refined ");
cb("prompt from AI");
}
}),
prompt: vi.fn(async () => {}),
dispose: vi.fn(),
};
const createKbAgentMock = vi.fn(async (options: { cwd: string; systemPrompt: string; tools: string }) => {
capturedSystemPrompt = options.systemPrompt;
return { session };
});
__setCreateKbAgentForRefine(createKbAgentMock);
const res = await REQUEST(buildApp(), "POST", "/api/workflow-steps/WS-001/refine", JSON.stringify({}), {
"Content-Type": "application/json",
});
expect(res.status).toBe(200);
expect(createKbAgentMock).toHaveBeenCalledTimes(1);
// Should use the default prompt (contains "You are an expert at creating")
expect(capturedSystemPrompt).toContain("You are an expert at creating");
expect(capturedSystemPrompt).toContain("workflow steps");
});
});
// ── Workflow Step Template Tests ──────────────────────────────────────────

View File

@@ -108,6 +108,43 @@ export function __setCreateKbAgentForRefine(mock: typeof createKbAgentForRefine)
createKbAgentForRefine = mock;
}
// Default system prompt for workflow step refinement (fallback when overrides unavailable)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let resolveWorkflowStepRefinePrompt: (key: string, overrides?: Record<string, string | undefined>) => string = () => DEFAULT_WORKFLOW_STEP_REFINE_PROMPT;
let promptOverridesReady = false;
async function initPromptOverrides() {
if (promptOverridesReady) return;
try {
const core = await import("@fusion/core");
resolveWorkflowStepRefinePrompt = (key: string, overrides?: Record<string, string | undefined>) =>
core.resolvePrompt(key as keyof typeof core.PROMPT_KEY_CATALOG, overrides);
promptOverridesReady = true;
} catch {
resolveWorkflowStepRefinePrompt = () => DEFAULT_WORKFLOW_STEP_REFINE_PROMPT;
promptOverridesReady = true;
}
}
// Initialize on module load
initPromptOverrides();
/** Default system prompt for workflow step refinement */
const DEFAULT_WORKFLOW_STEP_REFINE_PROMPT = `You are an expert at creating detailed agent prompts for workflow steps.
A workflow step is a quality gate that runs after a task is implemented but before it's marked complete.
Given a rough description, create a detailed prompt that an AI agent can follow to execute this workflow step.
The prompt should:
1. Define the purpose clearly
2. Specify what files/context to examine
3. List specific criteria to check
4. Describe what "success" looks like
5. Include guidance on handling common edge cases
Output ONLY the prompt text (no markdown, no explanations).`;
function validateOptionalModelField(value: unknown, name: string): string | undefined {
if (value === undefined || value === null) return undefined;
if (typeof value !== "string") {
@@ -8636,20 +8673,11 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout
const settings = await scopedStore.getSettings();
const systemPrompt = `You are an expert at creating detailed agent prompts for workflow steps.
A workflow step is a quality gate that runs after a task is implemented but before it's marked complete.
Given a rough description, create a detailed prompt that an AI agent can follow to execute this workflow step.
The prompt should:
1. Define the purpose clearly
2. Specify what files/context to examine
3. List specific criteria to check
4. Describe what "success" looks like
5. Include guidance on handling common edge cases
Output ONLY the prompt text (no markdown, no explanations).`;
// Resolve the system prompt using prompt overrides (with fallback to default)
const systemPrompt = resolveWorkflowStepRefinePrompt(
"workflow-step-refine",
settings.promptOverrides
) || DEFAULT_WORKFLOW_STEP_REFINE_PROMPT;
const { session } = await createKbAgent({
cwd: scopedStore.getRootDir(),
@@ -11116,8 +11144,9 @@ Output ONLY the prompt text (no markdown, no explanations).`;
const scopedStore = await getScopedStore(req);
const rootDir = scopedStore.getRootDir();
const settings = await scopedStore.getSettings();
const spec = await generateAgentSpec(sessionId, rootDir);
const spec = await generateAgentSpec(sessionId, rootDir, settings.promptOverrides);
res.json({ spec });
} catch (err: any) {
if (err instanceof ApiError) {