feat(dashboard): AI workflow design endpoint with validation, stripping, and rate limit

This commit is contained in:
gsxdsm
2026-06-04 23:55:05 -07:00
parent ef33d79e78
commit 776def02ea
2 changed files with 499 additions and 3 deletions

View File

@@ -0,0 +1,266 @@
// @vitest-environment node
//
// U7/R11/KTD-6 — HTTP integration coverage for POST /api/workflows/design.
// Exercises the route end-to-end against a REAL TaskStore (no store-method
// mocking) with a FAKE createFnAgent injected via __setCreateFnAgentForDesign:
// the fake captures the user prompt and returns canned text (NO model calls).
// The route must JSON-extract, parse, compile-triage, strip approval flags, and
// persist NOTHING — it returns the IR and the client decides.
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import express from "express";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { TaskStore, isBuiltinWorkflowId } from "@fusion/core";
import type { WorkflowIr } from "@fusion/core";
import {
registerWorkflowRoutes,
__setCreateFnAgentForDesign,
__resetCreateFnAgentForDesign,
__resetDesignRateLimit,
} from "../register-workflow-routes.js";
import { ApiError, sendErrorResponse } from "../../api-error.js";
import { request } from "../../test-request.js";
/** Captures the prompt the route fed the agent and returns canned `text`. */
function makeFakeAgent(text: string) {
const captured: { systemPrompt?: string; userPrompt?: string } = {};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const factory: any = async (opts: any) => {
captured.systemPrompt = opts.systemPrompt;
let textListener: ((delta: string) => void) | undefined;
const session = {
on(_event: "text", listener: (delta: string) => void) {
textListener = listener;
},
async prompt(userPrompt: string) {
captured.userPrompt = userPrompt;
textListener?.(text);
},
dispose() {},
};
return { session };
};
return { factory, captured };
}
/** A minimal valid v1 linear IR (start → prompt → end). */
function linearIr(overrides?: { nodeConfig?: Record<string, unknown> }): WorkflowIr {
return {
version: "v1",
name: "graph",
nodes: [
{ id: "start", kind: "start" },
{ id: "n1", kind: "prompt", config: { name: "Do it", prompt: "go", ...(overrides?.nodeConfig ?? {}) } },
{ id: "end", kind: "end" },
],
edges: [
{ from: "start", to: "n1", condition: "success" },
{ from: "n1", to: "end", condition: "success" },
],
} as WorkflowIr;
}
/** A branching IR: one node with two `success` edges → triggers the compiler's
* deferred-interpreter suffix (interpreterOnly). */
function branchingIr(): WorkflowIr {
return {
version: "v1",
name: "branchy",
nodes: [
{ id: "start", kind: "start" },
{ id: "a", kind: "prompt", config: { name: "A", prompt: "a" } },
{ id: "b", kind: "prompt", config: { name: "B", prompt: "b" } },
{ id: "c", kind: "prompt", config: { name: "C", prompt: "c" } },
{ id: "end", kind: "end" },
],
edges: [
{ from: "start", to: "a", condition: "success" },
{ from: "a", to: "b", condition: "success" },
{ from: "a", to: "c", condition: "success" },
{ from: "b", to: "end", condition: "success" },
{ from: "c", to: "end", condition: "success" },
],
} as WorkflowIr;
}
describe("POST /api/workflows/design (U7/R11/KTD-6)", () => {
let store: TaskStore;
let rootDir: string;
let globalDir: string;
let app: express.Express;
beforeEach(async () => {
rootDir = mkdtempSync(join(tmpdir(), "wf-design-root-"));
globalDir = mkdtempSync(join(tmpdir(), "wf-design-global-"));
store = new TaskStore(rootDir, globalDir, { inMemoryDb: true });
await store.init();
__resetDesignRateLimit();
app = express();
app.use(express.json());
const router = express.Router();
registerWorkflowRoutes({
router,
getProjectContext: async () => ({ store, engine: undefined, projectId: undefined }),
rethrowAsApiError: (err: unknown) => {
throw err instanceof ApiError ? err : new ApiError(500, err instanceof Error ? err.message : String(err));
},
} as unknown as Parameters<typeof registerWorkflowRoutes>[0]);
app.use("/api", router);
app.use((err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
if (err instanceof ApiError) sendErrorResponse(res, err.statusCode, err.message, { details: err.details });
else sendErrorResponse(res, 500, err instanceof Error ? err.message : String(err));
});
});
afterEach(() => {
__resetCreateFnAgentForDesign();
__resetDesignRateLimit();
store.close();
rmSync(rootDir, { recursive: true, force: true });
rmSync(globalDir, { recursive: true, force: true });
});
const postJson = (path: string, body: unknown) =>
request(app, "POST", path, JSON.stringify(body), { "Content-Type": "application/json" });
async function userDefCount() {
return (await store.listWorkflowDefinitions()).filter((w) => !isBuiltinWorkflowId(w.id)).length;
}
it("valid linear IR → 200 {ir, interpreterOnly:false} with layout", async () => {
const { factory } = makeFakeAgent(JSON.stringify(linearIr()));
__setCreateFnAgentForDesign(factory);
const res = await postJson("/api/workflows/design", { prompt: "a coding flow" });
expect(res.status).toBe(200);
expect(res.body.interpreterOnly).toBe(false);
expect(res.body.ir.nodes).toHaveLength(3);
expect(res.body.layout).toBeTruthy();
expect(Object.keys(res.body.layout).length).toBeGreaterThan(0);
expect(res.body.strippedApprovalFlags).toBe(false);
});
it("fenced + prose-wrapped JSON → still extracted and 200", async () => {
const wrapped = `Sure! Here is your workflow:\n\n\`\`\`json\n${JSON.stringify(linearIr())}\n\`\`\`\n\nLet me know if you want changes.`;
const { factory } = makeFakeAgent(wrapped);
__setCreateFnAgentForDesign(factory);
const res = await postJson("/api/workflows/design", { prompt: "make a flow" });
expect(res.status).toBe(200);
expect(res.body.ir.nodes).toHaveLength(3);
expect(res.body.interpreterOnly).toBe(false);
});
it("branching IR → 200 {interpreterOnly:true}", async () => {
const { factory } = makeFakeAgent(JSON.stringify(branchingIr()));
__setCreateFnAgentForDesign(factory);
const res = await postJson("/api/workflows/design", { prompt: "branch it" });
expect(res.status).toBe(200);
expect(res.body.interpreterOnly).toBe(true);
expect(res.body.ir.nodes).toHaveLength(5);
});
it("invalid JSON → 422, nothing persisted", async () => {
const { factory } = makeFakeAgent("this is not json at all");
__setCreateFnAgentForDesign(factory);
const before = await userDefCount();
const res = await postJson("/api/workflows/design", { prompt: "x" });
expect(res.status).toBe(422);
expect(await userDefCount()).toBe(before);
});
it("JSON failing parseWorkflowIr (missing start) → 422 with parser message, nothing persisted", async () => {
const noStart = {
version: "v1",
name: "broken",
nodes: [{ id: "end", kind: "end" }],
edges: [],
};
const { factory } = makeFakeAgent(JSON.stringify(noStart));
__setCreateFnAgentForDesign(factory);
const before = await userDefCount();
const res = await postJson("/api/workflows/design", { prompt: "x" });
expect(res.status).toBe(422);
expect(String(res.body.error ?? res.body.message ?? "")).toMatch(/start/i);
expect(await userDefCount()).toBe(before);
});
it("cliSkipApproval on a node → returned IR lacks it, strippedApprovalFlags:true", async () => {
const { factory } = makeFakeAgent(
JSON.stringify(linearIr({ nodeConfig: { cliSkipApproval: true } })),
);
__setCreateFnAgentForDesign(factory);
const res = await postJson("/api/workflows/design", { prompt: "x" });
expect(res.status).toBe(200);
expect(res.body.strippedApprovalFlags).toBe(true);
const node = res.body.ir.nodes.find((n: { id: string }) => n.id === "n1");
expect(node.config.cliSkipApproval).toBeUndefined();
});
it("workflowId flow: base IR is read server-side and folded into the agent prompt", async () => {
const seeded = await store.createWorkflowDefinition({
name: "Base flow",
description: "",
ir: linearIr(),
layout: {},
});
const { factory, captured } = makeFakeAgent(JSON.stringify(linearIr()));
__setCreateFnAgentForDesign(factory);
const res = await postJson("/api/workflows/design", {
prompt: "add a review step",
workflowId: seeded.id,
});
expect(res.status).toBe(200);
// The fake agent received the persisted base IR in its prompt.
expect(captured.userPrompt).toContain("BASE WORKFLOW IR");
expect(captured.userPrompt).toContain("\"kind\": \"start\"");
expect(captured.userPrompt).toContain("add a review step");
});
it("unknown workflowId → 404", async () => {
const { factory } = makeFakeAgent(JSON.stringify(linearIr()));
__setCreateFnAgentForDesign(factory);
const res = await postJson("/api/workflows/design", {
prompt: "x",
workflowId: "does-not-exist",
});
expect(res.status).toBe(404);
});
it("11th call within the window → 429", async () => {
const { factory } = makeFakeAgent(JSON.stringify(linearIr()));
__setCreateFnAgentForDesign(factory);
for (let i = 0; i < 10; i++) {
const ok = await postJson("/api/workflows/design", { prompt: `req ${i}` });
expect(ok.status).toBe(200);
}
const limited = await postJson("/api/workflows/design", { prompt: "one too many" });
expect(limited.status).toBe(429);
});
it("over-length prompt → 400", async () => {
const { factory } = makeFakeAgent(JSON.stringify(linearIr()));
__setCreateFnAgentForDesign(factory);
const res = await postJson("/api/workflows/design", { prompt: "x".repeat(4001) });
expect(res.status).toBe(400);
});
it("empty prompt → 400", async () => {
const { factory } = makeFakeAgent(JSON.stringify(linearIr()));
__setCreateFnAgentForDesign(factory);
const res = await postJson("/api/workflows/design", { prompt: " " });
expect(res.status).toBe(400);
});
});

View File

@@ -1,10 +1,121 @@
import type { WorkflowDefinition, WorkflowDefinitionKind, WorkflowIr, WorkflowIrNode } from "@fusion/core";
import { ColumnTraitValidationError, OccupiedColumnsError, InvalidRehomeTargetError, WorkflowCompileError, WorkflowIrError, SCHEMA_VERSION, assertColumnTraitsValid, compileWorkflowToSteps, listTraits, listStepParsers, parseWorkflowIr } from "@fusion/core";
import { validateCodeNodeSources } from "@fusion/engine";
import { ApiError, badRequest, conflict, notFound } from "../api-error.js";
import { ColumnTraitValidationError, OccupiedColumnsError, InvalidRehomeTargetError, WorkflowCompileError, WorkflowIrError, SCHEMA_VERSION, assertColumnTraitsValid, compileWorkflowToSteps, layoutForIr, listTraits, listStepParsers, parseWorkflowIr, resolvePlanningSettingsModel } from "@fusion/core";
import { createFnAgent as engineCreateFnAgent, validateCodeNodeSources } from "@fusion/engine";
import { ApiError, badRequest, conflict, notFound, rateLimited } from "../api-error.js";
import { emitWorkflowSseEvent } from "../sse.js";
import type { ApiRoutesContext } from "./types.js";
// ── AI design route DI seam + rate limiter (U7/R11/KTD-6) ─────────────────────
//
// Test-injectable createFnAgent factory, co-located with the route per KTD-6's
// "module-level __setCreateFnAgentForDesign DI seam co-located with the route".
// Defaults to the statically imported engine binding; tests inject a fake that
// captures the prompt and returns canned IR text (no model calls in tests).
let createFnAgentForDesign: typeof engineCreateFnAgent = engineCreateFnAgent;
/** @internal Inject a mock createFnAgent for the workflow-design route tests. */
export function __setCreateFnAgentForDesign(mock: typeof engineCreateFnAgent): void {
createFnAgentForDesign = mock;
}
/** @internal Reset the design route's createFnAgent to the real engine binding. */
export function __resetCreateFnAgentForDesign(): void {
createFnAgentForDesign = engineCreateFnAgent;
}
/** Minimal session shape used by the one-shot design turn (mirrors the refine
* route's RefineAgentSession): subscribe to text deltas, prompt once, dispose. */
interface DesignAgentSession {
on(event: "text", listener: (delta: string) => void): void;
prompt(text: string): Promise<void>;
dispose(): void;
}
/** Max design prompt length (chars). Over → 400 (mirrors the bounded prompts on
* the other AI routes; generous enough for an edit-with-context instruction). */
const MAX_DESIGN_PROMPT_LENGTH = 4000;
/** Rate limit: max design requests per IP per hour (mirrors /ai/refine-text). */
const MAX_DESIGN_REQUESTS_PER_HOUR = 10;
const DESIGN_RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000;
interface DesignRateLimitEntry {
count: number;
firstRequestAt: number;
}
// Dedicated window so design and refine-text don't share a counter. Module-level
// (per-process) exactly like ai-refine's limiter.
const designRateLimits = new Map<string, DesignRateLimitEntry>();
/** Returns true when the IP may make a design request (and records it); false
* when the 10/hour window is exhausted. Same shape as ai-refine.checkRateLimit. */
function checkDesignRateLimit(ip: string): boolean {
const now = Date.now();
const entry = designRateLimits.get(ip);
if (!entry || now - entry.firstRequestAt > DESIGN_RATE_LIMIT_WINDOW_MS) {
designRateLimits.set(ip, { count: 1, firstRequestAt: now });
return true;
}
if (entry.count >= MAX_DESIGN_REQUESTS_PER_HOUR) return false;
entry.count++;
return true;
}
/** @internal Reset the design rate-limit window (tests). */
export function __resetDesignRateLimit(): void {
designRateLimits.clear();
}
/** Extract a JSON object from possibly-fenced / prose-wrapped model output.
* Mirrors the evaluator/agent-generation precedent (packages/engine
* evaluator.ts extractJson): strip a leading ```json fence, else slice from the
* first `{` to the last `}`. Kept local — engine helper is not exported and the
* constraint forbids engine changes. */
function extractJsonFromText(raw: string): string {
const trimmed = raw.trim();
if (trimmed.startsWith("```")) {
return trimmed.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, "").trim();
}
const first = trimmed.indexOf("{");
const last = trimmed.lastIndexOf("}");
if (first >= 0 && last > first) return trimmed.slice(first, last + 1);
return trimmed;
}
/** System prompt for the design agent. Describes the WorkflowIr vocabulary
* concisely (adapted from fn_workflow_create's tool description) and constrains
* the model to emit ONLY a WorkflowIr JSON object. */
const WORKFLOW_DESIGN_SYSTEM_PROMPT = `You are a Fusion workflow architect. Given a description (and optionally a base graph to modify), output a single WorkflowIr JSON object that defines a board workflow.
OUTPUT CONTRACT
- Respond with ONLY the WorkflowIr JSON object. No prose, no explanation, no markdown fences.
WorkflowIr SHAPE
{ "version": "v1", "name": string, "nodes": Node[], "edges": Edge[] }
- Node: { "id": string (unique), "kind": NodeKind, "config"?: object }
- Edge: { "from": nodeId, "to": nodeId, "condition"?: "success" | "failure" }
NODE KINDS (v1)
- "start" — exactly one; the entry node. REQUIRED.
- "end" — exactly one; the terminal node. REQUIRED.
- "prompt" — an agent task. config: { "name": string, "prompt": string }. Encode the
workflow seam via config.seam = "execute" | "review" | "merge":
"execute" is the coding/work seam, "review" verifies, "merge" integrates.
- "script" — runs a configured project script. config: { "name": string, "scriptName": string }.
- "gate" — a manual/automatic checkpoint.
EDGES & SEAMS
- Every edge from a prompt seam node should carry a condition: "success" continues
the happy path; "failure" routes to "end".
- A standard linear coding workflow is: start → execute → review → merge → end, with
each seam's "success" advancing to the next and its "failure" going to "end".
- Keep it LINEAR unless the description clearly requires branching. The legacy engine
only runs linear graphs; branches are valid but flagged interpreter-only.
NEVER include "cliSkipApproval" or "autoApprove" in any node config — they are stripped
at this boundary regardless.`;
/**
* Routes for named workflow definitions, IR compilation preview, per-task
* workflow selection, and the project default workflow. All state changes flow
@@ -510,6 +621,125 @@ export function registerWorkflowRoutes(ctx: ApiRoutesContext): void {
rethrowAsApiError(err);
}
});
// POST /api/workflows/design — prompt → server-validated WorkflowIr (U7/R11/
// KTD-6). One-shot, tool-less agent on the planning lane; output is JSON-
// extracted, parsed, compile-triaged, and approval-flag-stripped. Persists
// NOTHING — the route returns the IR and the client decides. For the edit flow
// the client passes `workflowId` (never IR); the persisted base graph is read
// server-side and folded into the prompt. Rate-limited 10/hour/IP.
// prompt empty / > cap → 400
// rate limit exhausted → 429
// unknown workflowId → 404
// invalid JSON / parseWorkflowIr → 422 (parser message)
// compile deferred-suffix failure → 200 { interpreterOnly: true }
// other compile failure → 422 (graph unsound for both engines)
router.post("/workflows/design", async (req, res) => {
try {
const { store } = await getProjectContext(req);
const body = (req.body ?? {}) as { prompt?: unknown; workflowId?: unknown };
// Validate prompt (bounded length; non-empty).
if (typeof body.prompt !== "string" || !body.prompt.trim()) {
throw badRequest("prompt is required and must be a non-empty string");
}
const prompt = body.prompt;
if (prompt.length > MAX_DESIGN_PROMPT_LENGTH) {
throw badRequest(`prompt must not exceed ${MAX_DESIGN_PROMPT_LENGTH} characters`);
}
if (body.workflowId !== undefined && typeof body.workflowId !== "string") {
throw badRequest("workflowId must be a string when provided");
}
// Rate limit (mirrors /ai/refine-text: 10/hour/IP).
const ip = req.ip || req.socket?.remoteAddress || "unknown";
if (!checkDesignRateLimit(ip)) {
throw rateLimited(
`Rate limit exceeded. Maximum ${MAX_DESIGN_REQUESTS_PER_HOUR} workflow design requests per hour.`,
);
}
// Edit flow: read the persisted base IR server-side (client never posts IR).
let baseIrJson: string | undefined;
if (typeof body.workflowId === "string") {
const baseDef = await store.getWorkflowDefinition(body.workflowId);
if (!baseDef) throw notFound(`Workflow '${body.workflowId}' not found`);
baseIrJson = JSON.stringify(baseDef.ir, null, 2);
}
// One-shot, tool-less design turn on the planning lane.
const settings = await store.getSettings();
const planningModel = resolvePlanningSettingsModel(settings);
const { session } = await createFnAgentForDesign({
cwd: store.getRootDir(),
systemPrompt: WORKFLOW_DESIGN_SYSTEM_PROMPT,
tools: "readonly",
defaultProvider: planningModel.provider,
defaultModelId: planningModel.modelId,
defaultThinkingLevel: settings.defaultThinkingLevel,
});
const designSession = session as unknown as DesignAgentSession;
let output = "";
designSession.on("text", (delta: string) => {
output += delta;
});
const userPrompt = baseIrJson
? `Modify the following base workflow per the request below. Output the full updated WorkflowIr.\n\nBASE WORKFLOW IR:\n${baseIrJson}\n\nREQUEST:\n${prompt}`
: `Design a workflow for the following request:\n\n${prompt}`;
await designSession.prompt(userPrompt);
designSession.dispose();
// Extract JSON (handles fences/prose) → JSON.parse → parseWorkflowIr.
const candidate = extractJsonFromText(output);
let parsedJson: unknown;
try {
parsedJson = JSON.parse(candidate);
} catch {
throw new ApiError(422, "The AI response was not valid JSON.");
}
let ir: WorkflowIr;
try {
ir = parseWorkflowIr(parsedJson as WorkflowIr);
} catch (parseErr: unknown) {
if (parseErr instanceof WorkflowIrError) throw new ApiError(422, parseErr.message);
throw new ApiError(
422,
parseErr instanceof Error ? parseErr.message : "Invalid workflow IR",
);
}
// Compile triage: parseWorkflowIr is the validity gate. A compile failure
// whose message carries the deferred-interpreter suffix means the graph is
// structurally valid but only runnable on the (deferred) interpreter →
// interpreterOnly:true (NOT an error). Any OTHER compile failure means the
// graph is unsound for BOTH engines → 422.
let interpreterOnly = false;
try {
compileWorkflowToSteps(ir);
} catch (compileErr: unknown) {
const message = compileErr instanceof Error ? compileErr.message : String(compileErr);
if (message.includes("require the workflow interpreter (deferred)")) {
interpreterOnly = true;
} else {
throw new ApiError(422, message);
}
}
// Strip trust-escalating flags (shared helper; R11 trust boundary).
const strippedApprovalFlags = stripApprovalFlags(ir);
// Deterministic layout for the returned IR (server-side value import).
const layout = layoutForIr(ir);
res.json({ ir, layout, interpreterOnly, strippedApprovalFlags });
} catch (err: unknown) {
if (err instanceof ApiError) throw err;
rethrowAsApiError(err);
}
});
}
/** Validate trait availability for an imported IR exactly as the store does on