From 776def02ea1dd8c914c45386326fdd96f73c9a9f Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Thu, 4 Jun 2026 23:55:05 -0700 Subject: [PATCH] feat(dashboard): AI workflow design endpoint with validation, stripping, and rate limit --- .../__tests__/workflow-design-route.test.ts | 266 ++++++++++++++++++ .../src/routes/register-workflow-routes.ts | 236 +++++++++++++++- 2 files changed, 499 insertions(+), 3 deletions(-) create mode 100644 packages/dashboard/src/routes/__tests__/workflow-design-route.test.ts diff --git a/packages/dashboard/src/routes/__tests__/workflow-design-route.test.ts b/packages/dashboard/src/routes/__tests__/workflow-design-route.test.ts new file mode 100644 index 0000000000..db371b277e --- /dev/null +++ b/packages/dashboard/src/routes/__tests__/workflow-design-route.test.ts @@ -0,0 +1,266 @@ +// @vitest-environment node +// +// U7/R11/KTD-6 — HTTP integration coverage for POST /api/workflows/design. +// Exercises the route end-to-end against a REAL TaskStore (no store-method +// mocking) with a FAKE createFnAgent injected via __setCreateFnAgentForDesign: +// the fake captures the user prompt and returns canned text (NO model calls). +// The route must JSON-extract, parse, compile-triage, strip approval flags, and +// persist NOTHING — it returns the IR and the client decides. + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import express from "express"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { TaskStore, isBuiltinWorkflowId } from "@fusion/core"; +import type { WorkflowIr } from "@fusion/core"; +import { + registerWorkflowRoutes, + __setCreateFnAgentForDesign, + __resetCreateFnAgentForDesign, + __resetDesignRateLimit, +} from "../register-workflow-routes.js"; +import { ApiError, sendErrorResponse } from "../../api-error.js"; +import { request } from "../../test-request.js"; + +/** Captures the prompt the route fed the agent and returns canned `text`. */ +function makeFakeAgent(text: string) { + const captured: { systemPrompt?: string; userPrompt?: string } = {}; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const factory: any = async (opts: any) => { + captured.systemPrompt = opts.systemPrompt; + let textListener: ((delta: string) => void) | undefined; + const session = { + on(_event: "text", listener: (delta: string) => void) { + textListener = listener; + }, + async prompt(userPrompt: string) { + captured.userPrompt = userPrompt; + textListener?.(text); + }, + dispose() {}, + }; + return { session }; + }; + return { factory, captured }; +} + +/** A minimal valid v1 linear IR (start → prompt → end). */ +function linearIr(overrides?: { nodeConfig?: Record }): WorkflowIr { + return { + version: "v1", + name: "graph", + nodes: [ + { id: "start", kind: "start" }, + { id: "n1", kind: "prompt", config: { name: "Do it", prompt: "go", ...(overrides?.nodeConfig ?? {}) } }, + { id: "end", kind: "end" }, + ], + edges: [ + { from: "start", to: "n1", condition: "success" }, + { from: "n1", to: "end", condition: "success" }, + ], + } as WorkflowIr; +} + +/** A branching IR: one node with two `success` edges → triggers the compiler's + * deferred-interpreter suffix (interpreterOnly). */ +function branchingIr(): WorkflowIr { + return { + version: "v1", + name: "branchy", + nodes: [ + { id: "start", kind: "start" }, + { id: "a", kind: "prompt", config: { name: "A", prompt: "a" } }, + { id: "b", kind: "prompt", config: { name: "B", prompt: "b" } }, + { id: "c", kind: "prompt", config: { name: "C", prompt: "c" } }, + { id: "end", kind: "end" }, + ], + edges: [ + { from: "start", to: "a", condition: "success" }, + { from: "a", to: "b", condition: "success" }, + { from: "a", to: "c", condition: "success" }, + { from: "b", to: "end", condition: "success" }, + { from: "c", to: "end", condition: "success" }, + ], + } as WorkflowIr; +} + +describe("POST /api/workflows/design (U7/R11/KTD-6)", () => { + let store: TaskStore; + let rootDir: string; + let globalDir: string; + let app: express.Express; + + beforeEach(async () => { + rootDir = mkdtempSync(join(tmpdir(), "wf-design-root-")); + globalDir = mkdtempSync(join(tmpdir(), "wf-design-global-")); + store = new TaskStore(rootDir, globalDir, { inMemoryDb: true }); + await store.init(); + __resetDesignRateLimit(); + + app = express(); + app.use(express.json()); + const router = express.Router(); + registerWorkflowRoutes({ + router, + getProjectContext: async () => ({ store, engine: undefined, projectId: undefined }), + rethrowAsApiError: (err: unknown) => { + throw err instanceof ApiError ? err : new ApiError(500, err instanceof Error ? err.message : String(err)); + }, + } as unknown as Parameters[0]); + app.use("/api", router); + app.use((err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => { + if (err instanceof ApiError) sendErrorResponse(res, err.statusCode, err.message, { details: err.details }); + else sendErrorResponse(res, 500, err instanceof Error ? err.message : String(err)); + }); + }); + + afterEach(() => { + __resetCreateFnAgentForDesign(); + __resetDesignRateLimit(); + store.close(); + rmSync(rootDir, { recursive: true, force: true }); + rmSync(globalDir, { recursive: true, force: true }); + }); + + const postJson = (path: string, body: unknown) => + request(app, "POST", path, JSON.stringify(body), { "Content-Type": "application/json" }); + + async function userDefCount() { + return (await store.listWorkflowDefinitions()).filter((w) => !isBuiltinWorkflowId(w.id)).length; + } + + it("valid linear IR → 200 {ir, interpreterOnly:false} with layout", async () => { + const { factory } = makeFakeAgent(JSON.stringify(linearIr())); + __setCreateFnAgentForDesign(factory); + + const res = await postJson("/api/workflows/design", { prompt: "a coding flow" }); + expect(res.status).toBe(200); + expect(res.body.interpreterOnly).toBe(false); + expect(res.body.ir.nodes).toHaveLength(3); + expect(res.body.layout).toBeTruthy(); + expect(Object.keys(res.body.layout).length).toBeGreaterThan(0); + expect(res.body.strippedApprovalFlags).toBe(false); + }); + + it("fenced + prose-wrapped JSON → still extracted and 200", async () => { + const wrapped = `Sure! Here is your workflow:\n\n\`\`\`json\n${JSON.stringify(linearIr())}\n\`\`\`\n\nLet me know if you want changes.`; + const { factory } = makeFakeAgent(wrapped); + __setCreateFnAgentForDesign(factory); + + const res = await postJson("/api/workflows/design", { prompt: "make a flow" }); + expect(res.status).toBe(200); + expect(res.body.ir.nodes).toHaveLength(3); + expect(res.body.interpreterOnly).toBe(false); + }); + + it("branching IR → 200 {interpreterOnly:true}", async () => { + const { factory } = makeFakeAgent(JSON.stringify(branchingIr())); + __setCreateFnAgentForDesign(factory); + + const res = await postJson("/api/workflows/design", { prompt: "branch it" }); + expect(res.status).toBe(200); + expect(res.body.interpreterOnly).toBe(true); + expect(res.body.ir.nodes).toHaveLength(5); + }); + + it("invalid JSON → 422, nothing persisted", async () => { + const { factory } = makeFakeAgent("this is not json at all"); + __setCreateFnAgentForDesign(factory); + + const before = await userDefCount(); + const res = await postJson("/api/workflows/design", { prompt: "x" }); + expect(res.status).toBe(422); + expect(await userDefCount()).toBe(before); + }); + + it("JSON failing parseWorkflowIr (missing start) → 422 with parser message, nothing persisted", async () => { + const noStart = { + version: "v1", + name: "broken", + nodes: [{ id: "end", kind: "end" }], + edges: [], + }; + const { factory } = makeFakeAgent(JSON.stringify(noStart)); + __setCreateFnAgentForDesign(factory); + + const before = await userDefCount(); + const res = await postJson("/api/workflows/design", { prompt: "x" }); + expect(res.status).toBe(422); + expect(String(res.body.error ?? res.body.message ?? "")).toMatch(/start/i); + expect(await userDefCount()).toBe(before); + }); + + it("cliSkipApproval on a node → returned IR lacks it, strippedApprovalFlags:true", async () => { + const { factory } = makeFakeAgent( + JSON.stringify(linearIr({ nodeConfig: { cliSkipApproval: true } })), + ); + __setCreateFnAgentForDesign(factory); + + const res = await postJson("/api/workflows/design", { prompt: "x" }); + expect(res.status).toBe(200); + expect(res.body.strippedApprovalFlags).toBe(true); + const node = res.body.ir.nodes.find((n: { id: string }) => n.id === "n1"); + expect(node.config.cliSkipApproval).toBeUndefined(); + }); + + it("workflowId flow: base IR is read server-side and folded into the agent prompt", async () => { + const seeded = await store.createWorkflowDefinition({ + name: "Base flow", + description: "", + ir: linearIr(), + layout: {}, + }); + const { factory, captured } = makeFakeAgent(JSON.stringify(linearIr())); + __setCreateFnAgentForDesign(factory); + + const res = await postJson("/api/workflows/design", { + prompt: "add a review step", + workflowId: seeded.id, + }); + expect(res.status).toBe(200); + // The fake agent received the persisted base IR in its prompt. + expect(captured.userPrompt).toContain("BASE WORKFLOW IR"); + expect(captured.userPrompt).toContain("\"kind\": \"start\""); + expect(captured.userPrompt).toContain("add a review step"); + }); + + it("unknown workflowId → 404", async () => { + const { factory } = makeFakeAgent(JSON.stringify(linearIr())); + __setCreateFnAgentForDesign(factory); + + const res = await postJson("/api/workflows/design", { + prompt: "x", + workflowId: "does-not-exist", + }); + expect(res.status).toBe(404); + }); + + it("11th call within the window → 429", async () => { + const { factory } = makeFakeAgent(JSON.stringify(linearIr())); + __setCreateFnAgentForDesign(factory); + + for (let i = 0; i < 10; i++) { + const ok = await postJson("/api/workflows/design", { prompt: `req ${i}` }); + expect(ok.status).toBe(200); + } + const limited = await postJson("/api/workflows/design", { prompt: "one too many" }); + expect(limited.status).toBe(429); + }); + + it("over-length prompt → 400", async () => { + const { factory } = makeFakeAgent(JSON.stringify(linearIr())); + __setCreateFnAgentForDesign(factory); + + const res = await postJson("/api/workflows/design", { prompt: "x".repeat(4001) }); + expect(res.status).toBe(400); + }); + + it("empty prompt → 400", async () => { + const { factory } = makeFakeAgent(JSON.stringify(linearIr())); + __setCreateFnAgentForDesign(factory); + + const res = await postJson("/api/workflows/design", { prompt: " " }); + expect(res.status).toBe(400); + }); +}); diff --git a/packages/dashboard/src/routes/register-workflow-routes.ts b/packages/dashboard/src/routes/register-workflow-routes.ts index 73d1474670..1f01dad672 100644 --- a/packages/dashboard/src/routes/register-workflow-routes.ts +++ b/packages/dashboard/src/routes/register-workflow-routes.ts @@ -1,10 +1,121 @@ import type { WorkflowDefinition, WorkflowDefinitionKind, WorkflowIr, WorkflowIrNode } from "@fusion/core"; -import { ColumnTraitValidationError, OccupiedColumnsError, InvalidRehomeTargetError, WorkflowCompileError, WorkflowIrError, SCHEMA_VERSION, assertColumnTraitsValid, compileWorkflowToSteps, listTraits, listStepParsers, parseWorkflowIr } from "@fusion/core"; -import { validateCodeNodeSources } from "@fusion/engine"; -import { ApiError, badRequest, conflict, notFound } from "../api-error.js"; +import { ColumnTraitValidationError, OccupiedColumnsError, InvalidRehomeTargetError, WorkflowCompileError, WorkflowIrError, SCHEMA_VERSION, assertColumnTraitsValid, compileWorkflowToSteps, layoutForIr, listTraits, listStepParsers, parseWorkflowIr, resolvePlanningSettingsModel } from "@fusion/core"; +import { createFnAgent as engineCreateFnAgent, validateCodeNodeSources } from "@fusion/engine"; +import { ApiError, badRequest, conflict, notFound, rateLimited } from "../api-error.js"; import { emitWorkflowSseEvent } from "../sse.js"; import type { ApiRoutesContext } from "./types.js"; +// ── AI design route DI seam + rate limiter (U7/R11/KTD-6) ───────────────────── +// +// Test-injectable createFnAgent factory, co-located with the route per KTD-6's +// "module-level __setCreateFnAgentForDesign DI seam co-located with the route". +// Defaults to the statically imported engine binding; tests inject a fake that +// captures the prompt and returns canned IR text (no model calls in tests). +let createFnAgentForDesign: typeof engineCreateFnAgent = engineCreateFnAgent; + +/** @internal Inject a mock createFnAgent for the workflow-design route tests. */ +export function __setCreateFnAgentForDesign(mock: typeof engineCreateFnAgent): void { + createFnAgentForDesign = mock; +} + +/** @internal Reset the design route's createFnAgent to the real engine binding. */ +export function __resetCreateFnAgentForDesign(): void { + createFnAgentForDesign = engineCreateFnAgent; +} + +/** Minimal session shape used by the one-shot design turn (mirrors the refine + * route's RefineAgentSession): subscribe to text deltas, prompt once, dispose. */ +interface DesignAgentSession { + on(event: "text", listener: (delta: string) => void): void; + prompt(text: string): Promise; + dispose(): void; +} + +/** Max design prompt length (chars). Over → 400 (mirrors the bounded prompts on + * the other AI routes; generous enough for an edit-with-context instruction). */ +const MAX_DESIGN_PROMPT_LENGTH = 4000; + +/** Rate limit: max design requests per IP per hour (mirrors /ai/refine-text). */ +const MAX_DESIGN_REQUESTS_PER_HOUR = 10; +const DESIGN_RATE_LIMIT_WINDOW_MS = 60 * 60 * 1000; + +interface DesignRateLimitEntry { + count: number; + firstRequestAt: number; +} + +// Dedicated window so design and refine-text don't share a counter. Module-level +// (per-process) exactly like ai-refine's limiter. +const designRateLimits = new Map(); + +/** Returns true when the IP may make a design request (and records it); false + * when the 10/hour window is exhausted. Same shape as ai-refine.checkRateLimit. */ +function checkDesignRateLimit(ip: string): boolean { + const now = Date.now(); + const entry = designRateLimits.get(ip); + if (!entry || now - entry.firstRequestAt > DESIGN_RATE_LIMIT_WINDOW_MS) { + designRateLimits.set(ip, { count: 1, firstRequestAt: now }); + return true; + } + if (entry.count >= MAX_DESIGN_REQUESTS_PER_HOUR) return false; + entry.count++; + return true; +} + +/** @internal Reset the design rate-limit window (tests). */ +export function __resetDesignRateLimit(): void { + designRateLimits.clear(); +} + +/** Extract a JSON object from possibly-fenced / prose-wrapped model output. + * Mirrors the evaluator/agent-generation precedent (packages/engine + * evaluator.ts extractJson): strip a leading ```json fence, else slice from the + * first `{` to the last `}`. Kept local — engine helper is not exported and the + * constraint forbids engine changes. */ +function extractJsonFromText(raw: string): string { + const trimmed = raw.trim(); + if (trimmed.startsWith("```")) { + return trimmed.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, "").trim(); + } + const first = trimmed.indexOf("{"); + const last = trimmed.lastIndexOf("}"); + if (first >= 0 && last > first) return trimmed.slice(first, last + 1); + return trimmed; +} + +/** System prompt for the design agent. Describes the WorkflowIr vocabulary + * concisely (adapted from fn_workflow_create's tool description) and constrains + * the model to emit ONLY a WorkflowIr JSON object. */ +const WORKFLOW_DESIGN_SYSTEM_PROMPT = `You are a Fusion workflow architect. Given a description (and optionally a base graph to modify), output a single WorkflowIr JSON object that defines a board workflow. + +OUTPUT CONTRACT +- Respond with ONLY the WorkflowIr JSON object. No prose, no explanation, no markdown fences. + +WorkflowIr SHAPE +{ "version": "v1", "name": string, "nodes": Node[], "edges": Edge[] } +- Node: { "id": string (unique), "kind": NodeKind, "config"?: object } +- Edge: { "from": nodeId, "to": nodeId, "condition"?: "success" | "failure" } + +NODE KINDS (v1) +- "start" — exactly one; the entry node. REQUIRED. +- "end" — exactly one; the terminal node. REQUIRED. +- "prompt" — an agent task. config: { "name": string, "prompt": string }. Encode the + workflow seam via config.seam = "execute" | "review" | "merge": + "execute" is the coding/work seam, "review" verifies, "merge" integrates. +- "script" — runs a configured project script. config: { "name": string, "scriptName": string }. +- "gate" — a manual/automatic checkpoint. + +EDGES & SEAMS +- Every edge from a prompt seam node should carry a condition: "success" continues + the happy path; "failure" routes to "end". +- A standard linear coding workflow is: start → execute → review → merge → end, with + each seam's "success" advancing to the next and its "failure" going to "end". +- Keep it LINEAR unless the description clearly requires branching. The legacy engine + only runs linear graphs; branches are valid but flagged interpreter-only. + +NEVER include "cliSkipApproval" or "autoApprove" in any node config — they are stripped +at this boundary regardless.`; + /** * Routes for named workflow definitions, IR compilation preview, per-task * workflow selection, and the project default workflow. All state changes flow @@ -510,6 +621,125 @@ export function registerWorkflowRoutes(ctx: ApiRoutesContext): void { rethrowAsApiError(err); } }); + + // POST /api/workflows/design — prompt → server-validated WorkflowIr (U7/R11/ + // KTD-6). One-shot, tool-less agent on the planning lane; output is JSON- + // extracted, parsed, compile-triaged, and approval-flag-stripped. Persists + // NOTHING — the route returns the IR and the client decides. For the edit flow + // the client passes `workflowId` (never IR); the persisted base graph is read + // server-side and folded into the prompt. Rate-limited 10/hour/IP. + // prompt empty / > cap → 400 + // rate limit exhausted → 429 + // unknown workflowId → 404 + // invalid JSON / parseWorkflowIr → 422 (parser message) + // compile deferred-suffix failure → 200 { interpreterOnly: true } + // other compile failure → 422 (graph unsound for both engines) + router.post("/workflows/design", async (req, res) => { + try { + const { store } = await getProjectContext(req); + const body = (req.body ?? {}) as { prompt?: unknown; workflowId?: unknown }; + + // Validate prompt (bounded length; non-empty). + if (typeof body.prompt !== "string" || !body.prompt.trim()) { + throw badRequest("prompt is required and must be a non-empty string"); + } + const prompt = body.prompt; + if (prompt.length > MAX_DESIGN_PROMPT_LENGTH) { + throw badRequest(`prompt must not exceed ${MAX_DESIGN_PROMPT_LENGTH} characters`); + } + if (body.workflowId !== undefined && typeof body.workflowId !== "string") { + throw badRequest("workflowId must be a string when provided"); + } + + // Rate limit (mirrors /ai/refine-text: 10/hour/IP). + const ip = req.ip || req.socket?.remoteAddress || "unknown"; + if (!checkDesignRateLimit(ip)) { + throw rateLimited( + `Rate limit exceeded. Maximum ${MAX_DESIGN_REQUESTS_PER_HOUR} workflow design requests per hour.`, + ); + } + + // Edit flow: read the persisted base IR server-side (client never posts IR). + let baseIrJson: string | undefined; + if (typeof body.workflowId === "string") { + const baseDef = await store.getWorkflowDefinition(body.workflowId); + if (!baseDef) throw notFound(`Workflow '${body.workflowId}' not found`); + baseIrJson = JSON.stringify(baseDef.ir, null, 2); + } + + // One-shot, tool-less design turn on the planning lane. + const settings = await store.getSettings(); + const planningModel = resolvePlanningSettingsModel(settings); + const { session } = await createFnAgentForDesign({ + cwd: store.getRootDir(), + systemPrompt: WORKFLOW_DESIGN_SYSTEM_PROMPT, + tools: "readonly", + defaultProvider: planningModel.provider, + defaultModelId: planningModel.modelId, + defaultThinkingLevel: settings.defaultThinkingLevel, + }); + + const designSession = session as unknown as DesignAgentSession; + let output = ""; + designSession.on("text", (delta: string) => { + output += delta; + }); + + const userPrompt = baseIrJson + ? `Modify the following base workflow per the request below. Output the full updated WorkflowIr.\n\nBASE WORKFLOW IR:\n${baseIrJson}\n\nREQUEST:\n${prompt}` + : `Design a workflow for the following request:\n\n${prompt}`; + await designSession.prompt(userPrompt); + designSession.dispose(); + + // Extract JSON (handles fences/prose) → JSON.parse → parseWorkflowIr. + const candidate = extractJsonFromText(output); + let parsedJson: unknown; + try { + parsedJson = JSON.parse(candidate); + } catch { + throw new ApiError(422, "The AI response was not valid JSON."); + } + + let ir: WorkflowIr; + try { + ir = parseWorkflowIr(parsedJson as WorkflowIr); + } catch (parseErr: unknown) { + if (parseErr instanceof WorkflowIrError) throw new ApiError(422, parseErr.message); + throw new ApiError( + 422, + parseErr instanceof Error ? parseErr.message : "Invalid workflow IR", + ); + } + + // Compile triage: parseWorkflowIr is the validity gate. A compile failure + // whose message carries the deferred-interpreter suffix means the graph is + // structurally valid but only runnable on the (deferred) interpreter → + // interpreterOnly:true (NOT an error). Any OTHER compile failure means the + // graph is unsound for BOTH engines → 422. + let interpreterOnly = false; + try { + compileWorkflowToSteps(ir); + } catch (compileErr: unknown) { + const message = compileErr instanceof Error ? compileErr.message : String(compileErr); + if (message.includes("require the workflow interpreter (deferred)")) { + interpreterOnly = true; + } else { + throw new ApiError(422, message); + } + } + + // Strip trust-escalating flags (shared helper; R11 trust boundary). + const strippedApprovalFlags = stripApprovalFlags(ir); + + // Deterministic layout for the returned IR (server-side value import). + const layout = layoutForIr(ir); + + res.json({ ir, layout, interpreterOnly, strippedApprovalFlags }); + } catch (err: unknown) { + if (err instanceof ApiError) throw err; + rethrowAsApiError(err); + } + }); } /** Validate trait availability for an imported IR exactly as the store does on