feat(FN-1434): add prompt overrides for agent generation
- Add prompt-overrides module with template resolution and instruction injection - Wire prompt overrides into agent generation flow via POST /api/agents route - Support template-based prompt customization with role-based assignments - Add agent generation tests and routes tests - Document new prompt override settings in settings reference
This commit is contained in:
@@ -176,6 +176,8 @@ Fusion supports fine-grained customization of AI agent prompts through the `prom
|
||||
| `triage-context` | triage | Context-gathering instructions |
|
||||
| `reviewer-verdict` | reviewer | Verdict criteria and format |
|
||||
| `merger-conflicts` | merger | Merge conflict resolution instructions |
|
||||
| `agent-generation-system` | — | System prompt for AI-assisted agent specification generation |
|
||||
| `workflow-step-refine` | — | System prompt for refining workflow step descriptions into detailed agent prompts |
|
||||
|
||||
### How It Works
|
||||
|
||||
|
||||
@@ -34,7 +34,9 @@ export type PromptKey =
|
||||
| "triage-welcome"
|
||||
| "triage-context"
|
||||
| "reviewer-verdict"
|
||||
| "merger-conflicts";
|
||||
| "merger-conflicts"
|
||||
| "agent-generation-system"
|
||||
| "workflow-step-refine";
|
||||
|
||||
/**
|
||||
* Metadata describing a prompt key including its purpose and default content.
|
||||
@@ -172,6 +174,83 @@ If there are merge conflicts:
|
||||
5. Run \`git add <file>\` for each resolved file
|
||||
6. Do NOT change anything beyond what's needed to resolve the conflict`,
|
||||
},
|
||||
"agent-generation-system": {
|
||||
key: "agent-generation-system",
|
||||
name: "Agent Generation System",
|
||||
roles: [],
|
||||
description: "System prompt for the AI agent that generates agent specifications from role descriptions",
|
||||
defaultContent: `You are an agent specification generator for the fn task board system.
|
||||
|
||||
Your job: given a user-provided role description, generate a complete agent specification suitable for creating an AI agent.
|
||||
|
||||
## Input
|
||||
The user will provide a role description like:
|
||||
- "Senior frontend code reviewer who specializes in React accessibility"
|
||||
- "Security-focused DevOps engineer"
|
||||
- "Performance optimization specialist for Node.js applications"
|
||||
|
||||
## Output
|
||||
You MUST respond with ONLY valid JSON (no markdown, no explanation):
|
||||
|
||||
{
|
||||
"title": "A concise display name (max 60 chars)",
|
||||
"icon": "A single emoji representing the agent",
|
||||
"role": "The most appropriate capability: triage | executor | reviewer | merger | scheduler | engineer | custom",
|
||||
"description": "A brief 1-2 sentence description of the agent's purpose and expertise",
|
||||
"systemPrompt": "A detailed markdown system prompt for the agent. This should be comprehensive and include:\\n- Role definition\\n- Core responsibilities\\n- Specific areas of expertise\\n- Behavioral guidelines\\n- Output format expectations\\n- Edge case handling instructions",
|
||||
"thinkingLevel": "off | minimal | low | medium | high",
|
||||
"maxTurns": 10
|
||||
}
|
||||
|
||||
## Guidelines for System Prompt Generation
|
||||
- Be specific about the agent's domain expertise
|
||||
- Include concrete behavioral rules and constraints
|
||||
- Define the expected output format clearly
|
||||
- Add error handling and edge case guidance
|
||||
- Keep the prompt focused and actionable (aim for 200-800 words)
|
||||
- Use markdown formatting for readability
|
||||
|
||||
## Thinking Level Guidelines
|
||||
- "off": For simple, well-defined tasks (basic CRUD, simple checks)
|
||||
- "minimal": For straightforward tasks requiring some reasoning
|
||||
- "low": For moderate complexity tasks
|
||||
- "medium": For complex analysis, code review, architecture decisions
|
||||
- "high": For critical decisions, security analysis, complex debugging
|
||||
|
||||
## Max Turns Guidelines
|
||||
- 5-10: Simple, focused tasks (quick reviews, status checks)
|
||||
- 10-25: Standard tasks (code review, feature planning)
|
||||
- 25-50: Complex tasks (multi-file changes, architecture analysis)
|
||||
- 50+: Extended tasks (large refactors, comprehensive audits)
|
||||
|
||||
## Role Selection Guidelines
|
||||
- "reviewer": Agents focused on reviewing, auditing, analyzing
|
||||
- "executor": Agents that perform implementation work
|
||||
- "engineer": Agents that do engineering work with broader scope
|
||||
- "triage": Agents focused on classification and routing
|
||||
- "custom": Any agent that doesn't fit standard roles
|
||||
- Default to "custom" if unclear`,
|
||||
},
|
||||
"workflow-step-refine": {
|
||||
key: "workflow-step-refine",
|
||||
name: "Workflow Step Refine",
|
||||
roles: [],
|
||||
description: "System prompt for refining workflow step descriptions into detailed agent prompts",
|
||||
defaultContent: `You are an expert at creating detailed agent prompts for workflow steps.
|
||||
|
||||
A workflow step is a quality gate that runs after a task is implemented but before it's marked complete.
|
||||
|
||||
Given a rough description, create a detailed prompt that an AI agent can follow to execute this workflow step.
|
||||
|
||||
The prompt should:
|
||||
1. Define the purpose clearly
|
||||
2. Specify what files/context to examine
|
||||
3. List specific criteria to check
|
||||
4. Describe what "success" looks like
|
||||
5. Include guidance on handling common edge cases
|
||||
|
||||
Output ONLY the prompt text (no markdown, no explanations).`,
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -335,4 +335,89 @@ describe("agent-generation module", () => {
|
||||
expect(retrieved!.roleDescription).toBe("Security auditor role");
|
||||
});
|
||||
});
|
||||
|
||||
describe("prompt override support", () => {
|
||||
// Mock createKbAgent to capture the systemPrompt passed to it
|
||||
let capturedSystemPrompt: string | undefined;
|
||||
|
||||
beforeEach(async () => {
|
||||
capturedSystemPrompt = undefined;
|
||||
// Mock createKbAgent before tests run
|
||||
vi.doMock("@fusion/engine", () => ({
|
||||
createKbAgent: vi.fn(async (options: { cwd: string; systemPrompt: string; tools: string }) => {
|
||||
capturedSystemPrompt = options.systemPrompt;
|
||||
return {
|
||||
session: {
|
||||
state: { messages: [] },
|
||||
prompt: vi.fn(async () => {}),
|
||||
dispose: vi.fn(),
|
||||
},
|
||||
};
|
||||
}),
|
||||
}));
|
||||
// Reset the module to pick up the mock
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("generates spec with default system prompt when no overrides provided", async () => {
|
||||
const { generateAgentSpec: genSpec, startAgentGeneration: startGen } = await import("./agent-generation.js");
|
||||
|
||||
const session = await startGen(getUniqueIp(), "Test role");
|
||||
const spec = await genSpec(session.id, "/tmp");
|
||||
|
||||
// The spec should be empty since we mocked an empty response
|
||||
expect(spec).toBeDefined();
|
||||
// The system prompt should be the default
|
||||
expect(capturedSystemPrompt).toBeDefined();
|
||||
expect(capturedSystemPrompt).toContain("agent specification generator");
|
||||
});
|
||||
|
||||
it("generates spec with override system prompt when overrides provided", async () => {
|
||||
const { generateAgentSpec: genSpec, startAgentGeneration: startGen } = await import("./agent-generation.js");
|
||||
|
||||
const customPrompt = "CUSTOM AGENT GENERATION PROMPT";
|
||||
const overrides = { "agent-generation-system": customPrompt };
|
||||
|
||||
const session = await startGen(getUniqueIp(), "Test role");
|
||||
const spec = await genSpec(session.id, "/tmp", overrides);
|
||||
|
||||
// The spec should be empty since we mocked an empty response
|
||||
expect(spec).toBeDefined();
|
||||
// The system prompt should be the custom override
|
||||
expect(capturedSystemPrompt).toBe(customPrompt);
|
||||
});
|
||||
|
||||
it("falls back to default when override key not recognized", async () => {
|
||||
const { generateAgentSpec: genSpec, startAgentGeneration: startGen } = await import("./agent-generation.js");
|
||||
|
||||
// Provide an override with a non-existent key
|
||||
const overrides = { "non-existent-key": "Some prompt" };
|
||||
|
||||
const session = await startGen(getUniqueIp(), "Test role");
|
||||
const spec = await genSpec(session.id, "/tmp", overrides);
|
||||
|
||||
// Should fall back to default
|
||||
expect(spec).toBeDefined();
|
||||
expect(capturedSystemPrompt).toBeDefined();
|
||||
expect(capturedSystemPrompt).toContain("agent specification generator");
|
||||
});
|
||||
|
||||
it("falls back to AGENT_GENERATION_SYSTEM_PROMPT constant when resolvePrompt returns empty", async () => {
|
||||
const { generateAgentSpec: genSpec, startAgentGeneration: startGen, AGENT_GENERATION_SYSTEM_PROMPT } = await import("./agent-generation.js");
|
||||
|
||||
// Empty overrides should still get the default constant
|
||||
const overrides = { "agent-generation-system": "" };
|
||||
|
||||
const session = await startGen(getUniqueIp(), "Test role");
|
||||
const spec = await genSpec(session.id, "/tmp", overrides);
|
||||
|
||||
expect(spec).toBeDefined();
|
||||
expect(capturedSystemPrompt).toBe(AGENT_GENERATION_SYSTEM_PROMPT);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,6 +14,31 @@
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
// Dynamic import for @fusion/core to get prompt override resolution
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type PromptOverrideMap = Record<string, string | undefined>;
|
||||
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
|
||||
type ResolvePromptFn = (key: string, overrides?: PromptOverrideMap) => string;
|
||||
let resolvePrompt: ResolvePromptFn = () => "";
|
||||
let promptCatalogReady = false;
|
||||
|
||||
async function initPromptCatalog() {
|
||||
if (promptCatalogReady) return;
|
||||
try {
|
||||
const core = await import("@fusion/core");
|
||||
resolvePrompt = (key: string, overrides?: PromptOverrideMap) =>
|
||||
core.resolvePrompt(key as keyof typeof core.PROMPT_KEY_CATALOG, overrides);
|
||||
promptCatalogReady = true;
|
||||
} catch {
|
||||
// Use fallback resolution when core is unavailable
|
||||
resolvePrompt = () => "";
|
||||
promptCatalogReady = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize prompt catalog (will be awaited in actual usage)
|
||||
const promptCatalogReadyPromise = initPromptCatalog();
|
||||
|
||||
// Dynamic import for @fusion/engine to avoid resolution issues in test environment
|
||||
// eslint-disable-next-line @typescript-eslint/consistent-type-imports, @typescript-eslint/no-explicit-any
|
||||
type AgentResult = any;
|
||||
@@ -424,11 +449,13 @@ export async function startAgentGeneration(
|
||||
*
|
||||
* @param sessionId - The session identifier
|
||||
* @param rootDir - Project root directory for AI agent context
|
||||
* @param promptOverrides - Optional prompt overrides from project settings
|
||||
* @returns The generated agent specification
|
||||
*/
|
||||
export async function generateAgentSpec(
|
||||
sessionId: string,
|
||||
rootDir: string
|
||||
rootDir: string,
|
||||
promptOverrides?: PromptOverrideMap
|
||||
): Promise<AgentGenerationSpec> {
|
||||
const session = sessions.get(sessionId);
|
||||
if (!session) {
|
||||
@@ -437,7 +464,8 @@ export async function generateAgentSpec(
|
||||
|
||||
try {
|
||||
await engineReady;
|
||||
const spec = await generateSpecWithAI(session, rootDir);
|
||||
await promptCatalogReadyPromise;
|
||||
const spec = await generateSpecWithAI(session, rootDir, promptOverrides);
|
||||
session.spec = spec;
|
||||
session.updatedAt = new Date();
|
||||
return spec;
|
||||
@@ -450,14 +478,21 @@ export async function generateAgentSpec(
|
||||
/**
|
||||
* Generate an agent specification using the AI agent.
|
||||
*/
|
||||
async function generateSpecWithAI(session: Session, rootDir: string): Promise<AgentGenerationSpec> {
|
||||
async function generateSpecWithAI(
|
||||
session: Session,
|
||||
rootDir: string,
|
||||
promptOverrides?: PromptOverrideMap
|
||||
): Promise<AgentGenerationSpec> {
|
||||
if (!createKbAgent) {
|
||||
throw new Error("AI agent not available. Ensure the engine is properly configured.");
|
||||
}
|
||||
|
||||
// Resolve the system prompt using prompt overrides (with fallback to default)
|
||||
const effectiveSystemPrompt = resolvePrompt("agent-generation-system", promptOverrides) || AGENT_GENERATION_SYSTEM_PROMPT;
|
||||
|
||||
const agent = await createKbAgent({
|
||||
cwd: rootDir,
|
||||
systemPrompt: AGENT_GENERATION_SYSTEM_PROMPT,
|
||||
systemPrompt: effectiveSystemPrompt,
|
||||
tools: "none",
|
||||
});
|
||||
|
||||
|
||||
@@ -9946,6 +9946,89 @@ describe("POST /workflow-steps/:id/refine", () => {
|
||||
expect(res.body.workflowStep.prompt).toBe("Check docs");
|
||||
expect(store.updateWorkflowStep).toHaveBeenCalledWith("WS-001", { prompt: "Check docs" });
|
||||
});
|
||||
|
||||
it("uses custom prompt from promptOverrides when provided", async () => {
|
||||
const ws = { id: "WS-001", name: "Docs", description: "Check docs", mode: "prompt", prompt: "", enabled: true, createdAt: "2026-01-01", updatedAt: "2026-01-01" };
|
||||
(store.getWorkflowStep as ReturnType<typeof vi.fn>).mockResolvedValueOnce(ws);
|
||||
const customPrompt = "CUSTOM WORKFLOW STEP REFINE PROMPT";
|
||||
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
|
||||
promptOverrides: {
|
||||
"workflow-step-refine": customPrompt,
|
||||
},
|
||||
});
|
||||
|
||||
const updatedWs = { ...ws, prompt: "Refined prompt from AI" };
|
||||
(store.updateWorkflowStep as ReturnType<typeof vi.fn>).mockResolvedValueOnce(updatedWs);
|
||||
|
||||
let capturedSystemPrompt: string | undefined;
|
||||
const session = {
|
||||
on: vi.fn((event: string, cb: (delta: string) => void) => {
|
||||
if (event === "text") {
|
||||
cb("Refined ");
|
||||
cb("prompt from AI");
|
||||
}
|
||||
}),
|
||||
prompt: vi.fn(async () => {}),
|
||||
dispose: vi.fn(),
|
||||
};
|
||||
|
||||
const createKbAgentMock = vi.fn(async (options: { cwd: string; systemPrompt: string; tools: string }) => {
|
||||
capturedSystemPrompt = options.systemPrompt;
|
||||
return { session };
|
||||
});
|
||||
__setCreateKbAgentForRefine(createKbAgentMock);
|
||||
|
||||
const res = await REQUEST(buildApp(), "POST", "/api/workflow-steps/WS-001/refine", JSON.stringify({}), {
|
||||
"Content-Type": "application/json",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(createKbAgentMock).toHaveBeenCalledTimes(1);
|
||||
// Verify the custom prompt was passed
|
||||
expect(capturedSystemPrompt).toBe(customPrompt);
|
||||
});
|
||||
|
||||
it("uses default prompt when promptOverrides does not contain workflow-step-refine", async () => {
|
||||
const ws = { id: "WS-001", name: "Docs", description: "Check docs", mode: "prompt", prompt: "", enabled: true, createdAt: "2026-01-01", updatedAt: "2026-01-01" };
|
||||
(store.getWorkflowStep as ReturnType<typeof vi.fn>).mockResolvedValueOnce(ws);
|
||||
// Settings with other overrides but not workflow-step-refine
|
||||
(store.getSettings as ReturnType<typeof vi.fn>).mockResolvedValueOnce({
|
||||
promptOverrides: {
|
||||
"executor-welcome": "Some other prompt",
|
||||
},
|
||||
});
|
||||
|
||||
const updatedWs = { ...ws, prompt: "Refined prompt from AI" };
|
||||
(store.updateWorkflowStep as ReturnType<typeof vi.fn>).mockResolvedValueOnce(updatedWs);
|
||||
|
||||
let capturedSystemPrompt: string | undefined;
|
||||
const session = {
|
||||
on: vi.fn((event: string, cb: (delta: string) => void) => {
|
||||
if (event === "text") {
|
||||
cb("Refined ");
|
||||
cb("prompt from AI");
|
||||
}
|
||||
}),
|
||||
prompt: vi.fn(async () => {}),
|
||||
dispose: vi.fn(),
|
||||
};
|
||||
|
||||
const createKbAgentMock = vi.fn(async (options: { cwd: string; systemPrompt: string; tools: string }) => {
|
||||
capturedSystemPrompt = options.systemPrompt;
|
||||
return { session };
|
||||
});
|
||||
__setCreateKbAgentForRefine(createKbAgentMock);
|
||||
|
||||
const res = await REQUEST(buildApp(), "POST", "/api/workflow-steps/WS-001/refine", JSON.stringify({}), {
|
||||
"Content-Type": "application/json",
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(createKbAgentMock).toHaveBeenCalledTimes(1);
|
||||
// Should use the default prompt (contains "You are an expert at creating")
|
||||
expect(capturedSystemPrompt).toContain("You are an expert at creating");
|
||||
expect(capturedSystemPrompt).toContain("workflow steps");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Workflow Step Template Tests ──────────────────────────────────────────
|
||||
|
||||
@@ -108,6 +108,43 @@ export function __setCreateKbAgentForRefine(mock: typeof createKbAgentForRefine)
|
||||
createKbAgentForRefine = mock;
|
||||
}
|
||||
|
||||
// Default system prompt for workflow step refinement (fallback when overrides unavailable)
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
let resolveWorkflowStepRefinePrompt: (key: string, overrides?: Record<string, string | undefined>) => string = () => DEFAULT_WORKFLOW_STEP_REFINE_PROMPT;
|
||||
let promptOverridesReady = false;
|
||||
|
||||
async function initPromptOverrides() {
|
||||
if (promptOverridesReady) return;
|
||||
try {
|
||||
const core = await import("@fusion/core");
|
||||
resolveWorkflowStepRefinePrompt = (key: string, overrides?: Record<string, string | undefined>) =>
|
||||
core.resolvePrompt(key as keyof typeof core.PROMPT_KEY_CATALOG, overrides);
|
||||
promptOverridesReady = true;
|
||||
} catch {
|
||||
resolveWorkflowStepRefinePrompt = () => DEFAULT_WORKFLOW_STEP_REFINE_PROMPT;
|
||||
promptOverridesReady = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize on module load
|
||||
initPromptOverrides();
|
||||
|
||||
/** Default system prompt for workflow step refinement */
|
||||
const DEFAULT_WORKFLOW_STEP_REFINE_PROMPT = `You are an expert at creating detailed agent prompts for workflow steps.
|
||||
|
||||
A workflow step is a quality gate that runs after a task is implemented but before it's marked complete.
|
||||
|
||||
Given a rough description, create a detailed prompt that an AI agent can follow to execute this workflow step.
|
||||
|
||||
The prompt should:
|
||||
1. Define the purpose clearly
|
||||
2. Specify what files/context to examine
|
||||
3. List specific criteria to check
|
||||
4. Describe what "success" looks like
|
||||
5. Include guidance on handling common edge cases
|
||||
|
||||
Output ONLY the prompt text (no markdown, no explanations).`;
|
||||
|
||||
function validateOptionalModelField(value: unknown, name: string): string | undefined {
|
||||
if (value === undefined || value === null) return undefined;
|
||||
if (typeof value !== "string") {
|
||||
@@ -8636,20 +8673,11 @@ export function createApiRoutes(store: TaskStore, options?: ServerOptions): Rout
|
||||
|
||||
const settings = await scopedStore.getSettings();
|
||||
|
||||
const systemPrompt = `You are an expert at creating detailed agent prompts for workflow steps.
|
||||
|
||||
A workflow step is a quality gate that runs after a task is implemented but before it's marked complete.
|
||||
|
||||
Given a rough description, create a detailed prompt that an AI agent can follow to execute this workflow step.
|
||||
|
||||
The prompt should:
|
||||
1. Define the purpose clearly
|
||||
2. Specify what files/context to examine
|
||||
3. List specific criteria to check
|
||||
4. Describe what "success" looks like
|
||||
5. Include guidance on handling common edge cases
|
||||
|
||||
Output ONLY the prompt text (no markdown, no explanations).`;
|
||||
// Resolve the system prompt using prompt overrides (with fallback to default)
|
||||
const systemPrompt = resolveWorkflowStepRefinePrompt(
|
||||
"workflow-step-refine",
|
||||
settings.promptOverrides
|
||||
) || DEFAULT_WORKFLOW_STEP_REFINE_PROMPT;
|
||||
|
||||
const { session } = await createKbAgent({
|
||||
cwd: scopedStore.getRootDir(),
|
||||
@@ -11116,8 +11144,9 @@ Output ONLY the prompt text (no markdown, no explanations).`;
|
||||
|
||||
const scopedStore = await getScopedStore(req);
|
||||
const rootDir = scopedStore.getRootDir();
|
||||
const settings = await scopedStore.getSettings();
|
||||
|
||||
const spec = await generateAgentSpec(sessionId, rootDir);
|
||||
const spec = await generateAgentSpec(sessionId, rootDir, settings.promptOverrides);
|
||||
res.json({ spec });
|
||||
} catch (err: any) {
|
||||
if (err instanceof ApiError) {
|
||||
|
||||
Reference in New Issue
Block a user