From 883f38d68f7133e7ed6e967dcfe6ff232dc9c2c1 Mon Sep 17 00:00:00 2001 From: Phil Larson Date: Wed, 15 Jul 2026 13:57:15 -0700 Subject: [PATCH] Fix agent AI interview model routing (#2142) ## Summary - resolve the configured planning model when agent onboarding requests omit an explicit override - align the onboarding prompt with supported runtime/model hint fields, allowing AI-created agents to select runtimes such as Hermes - refresh the generated GitHub issue import limits required by the repository sync gate ## Root cause The agent onboarding route loaded project settings but passed only request-body model fields. The AI Interview UI omits those fields, so `createFnAgent` was called with `provider=undefined, model=undefined`; the session returned no usable assistant JSON. The prompt catalog also prohibited `runtimeHint` despite the parser and form already supporting it. ## Verification - targeted agent onboarding tests: 22 passed - `pnpm --filter @fusion/core typecheck` - `pnpm --filter @fusion/dashboard typecheck` - `pnpm lint` - `pnpm build` - `pnpm smoke:boot` - engine merge-gate subset: 294 passed Full `pnpm test` reached the PostgreSQL gate but this host has no `psql` binary, so 23 PostgreSQL suites could not start; this is an environment prerequisite failure, not a test assertion failure. ## Summary by CodeRabbit * **Bug Fixes** * Agent onboarding interviews now use the configured planning model when no override is provided. * Runtime suggestions and runtime-hint guidance are preserved during onboarding and reflected in generated configurations. * On onboarding start streaming, planning provider/model resolution now comes from settings with stricter override validation, and test mode continues to take priority. * **Documentation** * Updated onboarding prompt guidance to support additional configuration fields and optional runtime draft hints. * Reduced the maximum GitHub issue import/browse limit from 100 to 50. * **Tests** * Added coverage for runtime-hints prompting and planning-model override behavior. --- .changeset/bright-agent-interviews.md | 7 +++ .../fusion/references/extension-tools.md | 4 +- packages/core/src/prompt-overrides.ts | 8 ++- .../src/__tests__/agent-onboarding.test.ts | 22 +++++++ .../__tests__/agent-onboarding-routes.test.ts | 63 ++++++++++++++++++- ...r-agent-import-export-generation-routes.ts | 33 +++++++++- 6 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 .changeset/bright-agent-interviews.md diff --git a/.changeset/bright-agent-interviews.md b/.changeset/bright-agent-interviews.md new file mode 100644 index 0000000000..10d03badf9 --- /dev/null +++ b/.changeset/bright-agent-interviews.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fix agent AI interviews to use the configured planning model and preserve runtime suggestions. +category: fix +dev: Resolves onboarding model settings before session creation and aligns the prompt with supported runtime draft fields. diff --git a/packages/cli/skill/fusion/references/extension-tools.md b/packages/cli/skill/fusion/references/extension-tools.md index 291e513cbf..97509e0012 100644 --- a/packages/cli/skill/fusion/references/extension-tools.md +++ b/packages/cli/skill/fusion/references/extension-tools.md @@ -303,7 +303,7 @@ Import GitHub issues as Fusion tasks. Fetches open issues from a repository and | Parameter | Type | Required | Description | |-----------|------|----------|-------------| | `ownerRepo` | string | ✓ | Repository in owner/repo format (e.g., 'dustinbyrne/fusion') | -| `limit` | number | — | Max issues to import (default: 30, max: 100) | +| `limit` | number | — | Max issues to import (default: 30, max: 50) | | `labels` | array | — | Label names to filter by | ### fn_task_import_github_issue @@ -324,7 +324,7 @@ List open GitHub issues from a repository to browse before importing. Returns is |-----------|------|----------|-------------| | `owner` | string | ✓ | Repository owner (e.g., 'dustinbyrne') | | `repo` | string | ✓ | Repository name (e.g., 'fusion') | -| `limit` | number | — | Max issues to show (default: 30, max: 100) | +| `limit` | number | — | Max issues to show (default: 30, max: 50) | | `labels` | array | — | Label names to filter by | ## Mission Tools diff --git a/packages/core/src/prompt-overrides.ts b/packages/core/src/prompt-overrides.ts index a94ecc88ad..18848a9bb0 100644 --- a/packages/core/src/prompt-overrides.ts +++ b/packages/core/src/prompt-overrides.ts @@ -315,13 +315,17 @@ Ask targeted questions using this JSON format: {"type":"question","data":{"id":"q1","type":"text|single_select|multi_select|confirm","question":"...","description":"...","options":[{"id":"x","label":"X","description":"..."}]}} When ready, return a final summary JSON in this exact format: -{"type":"complete","data":{"name":"...","role":"executor","instructionsText":"...","thinkingLevel":"medium","maxTurns":25,"title":"...","icon":"🤖","reportsTo":"...","soul":"...","memory":"...","skills":["..."],"templateId":"...","patternAgentId":"...","rationale":"..."}} +{"type":"complete","data":{"name":"...","role":"executor","instructionsText":"...","thinkingLevel":"medium","maxTurns":25,"title":"...","icon":"🤖","reportsTo":"...","soul":"...","memory":"...","skills":["..."],"templateId":"...","patternAgentId":"...","rationale":"...","heartbeatProcedurePath":"...","heartbeatIntervalMs":30000,"heartbeatEnabled":true,"modelHint":"...","runtimeHint":"..."}} Rules: - role must be one of triage|executor|reviewer|merger|scheduler|engineer|custom - thinkingLevel must be off|minimal|low|medium|high - maxTurns must be a positive integer -- Do not include runtimeMode/model/runtimeHint; those are user review-time choices.`, +- Use instructionsText for starter operating guidance/playbook content; do not create a separate playbook field +- Prefer structuring instructionsText with these markdown sections when drafting: ## Description, ## Expertise, ## Priorities, ## Boundaries, ## Communication, ## Collaboration & Escalation +- Freeform instructionsText is still acceptable for compatibility; sectioned structure is preferred for new agents +- modelHint and runtimeHint are optional draft suggestions only (not final runtime selection) +- heartbeatProcedurePath, heartbeatIntervalMs, and heartbeatEnabled are optional draft hints only.`, }, "subtask-breakdown-system": { key: "subtask-breakdown-system", diff --git a/packages/dashboard/src/__tests__/agent-onboarding.test.ts b/packages/dashboard/src/__tests__/agent-onboarding.test.ts index 0f536febcd..572819e566 100644 --- a/packages/dashboard/src/__tests__/agent-onboarding.test.ts +++ b/packages/dashboard/src/__tests__/agent-onboarding.test.ts @@ -271,6 +271,28 @@ describe("agent-onboarding", () => { expect(prompt).toContain("messageResponseMode: immediate"); }); + it("asks the onboarding model for runtime hints that can select Hermes", async () => { + mockCreateFnAgent.mockResolvedValueOnce( + createMockAgent([ + JSON.stringify({ + type: "question", + data: { id: "goal", type: "text", question: "What is the primary goal?" }, + }), + ]), + ); + + await startAgentOnboardingSession( + "127.0.0.1", + { intent: "Use the Hermes runtime for computer-use testing", existingAgents: [], templates: [] }, + process.cwd(), + ); + + const options = mockCreateFnAgent.mock.calls.at(-1)?.[0] as { systemPrompt?: string }; + expect(options.systemPrompt).toContain('"runtimeHint"'); + expect(options.systemPrompt).toContain("optional draft suggestions"); + expect(options.systemPrompt).not.toContain("Do not include runtimeMode/model/runtimeHint"); + }); + it("requests role-fallback and enabled plugin skills for model-only onboarding agents", async () => { mockCreateFnAgent.mockResolvedValueOnce( createMockAgent([ diff --git a/packages/dashboard/src/routes/__tests__/agent-onboarding-routes.test.ts b/packages/dashboard/src/routes/__tests__/agent-onboarding-routes.test.ts index 520d09ad3a..6db146877a 100644 --- a/packages/dashboard/src/routes/__tests__/agent-onboarding-routes.test.ts +++ b/packages/dashboard/src/routes/__tests__/agent-onboarding-routes.test.ts @@ -63,10 +63,10 @@ function createMockStore(): TaskStore { } as unknown as TaskStore; } -function setupApp() { +function setupApp(store = createMockStore()) { const app = express(); app.use(express.json()); - app.use("/api", createApiRoutes(createMockStore())); + app.use("/api", createApiRoutes(store)); return app; } @@ -88,6 +88,46 @@ describe("agent onboarding routes", () => { expect(mockStartAgentOnboardingSession.mock.calls[0]?.[1]).toMatchObject({ mode: "create" }); }); + it("uses the configured planning model when the request omits an override", async () => { + const store = createMockStore(); + vi.mocked(store.getSettings).mockResolvedValue({ + planningProvider: "openai-codex", + planningModelId: "gpt-5.6-sol", + } as Awaited>); + const app = setupApp(store); + + const res = await request(app, "POST", "/api/agents/onboarding/start-streaming", JSON.stringify({ + intent: "Create a computer-use tester", + context: { existingAgents: [], templates: [] }, + }), { "Content-Type": "application/json" }); + + expect(res.status).toBe(201); + expect(mockStartAgentOnboardingSession).toHaveBeenCalledTimes(1); + expect(mockStartAgentOnboardingSession.mock.calls[0]?.[3]).toBe("openai-codex"); + expect(mockStartAgentOnboardingSession.mock.calls[0]?.[4]).toBe("gpt-5.6-sol"); + }); + + it("keeps test mode authoritative over explicit request model overrides", async () => { + const store = createMockStore(); + vi.mocked(store.getSettings).mockResolvedValue({ + testMode: true, + planningProvider: "openai-codex", + planningModelId: "gpt-5.6-sol", + } as Awaited>); + const app = setupApp(store); + + const res = await request(app, "POST", "/api/agents/onboarding/start-streaming", JSON.stringify({ + intent: "Create a test agent", + planningModelProvider: "anthropic", + planningModelId: "claude-sonnet-4-5", + context: { existingAgents: [], templates: [] }, + }), { "Content-Type": "application/json" }); + + expect(res.status).toBe(201); + expect(mockStartAgentOnboardingSession.mock.calls[0]?.[3]).toBe("mock"); + expect(mockStartAgentOnboardingSession.mock.calls[0]?.[4]).toBe("scripted"); + }); + it("accepts edit mode and forwards existingAgentConfig", async () => { const app = setupApp(); const res = await request(app, "POST", "/api/agents/onboarding/start-streaming", JSON.stringify({ @@ -113,6 +153,25 @@ describe("agent onboarding routes", () => { }); }); + it.each([ + { planningModelProvider: 42, planningModelId: "gpt-5.6-sol" }, + { planningModelProvider: "openai-codex", planningModelId: {} }, + { planningModelProvider: " ", planningModelId: "gpt-5.6-sol" }, + { planningModelProvider: "openai-codex", planningModelId: " " }, + { planningModelProvider: "openai-codex" }, + { planningModelId: "gpt-5.6-sol" }, + ])("rejects invalid planning model overrides: %j", async (override) => { + const app = setupApp(); + const res = await request(app, "POST", "/api/agents/onboarding/start-streaming", JSON.stringify({ + intent: "Create an agent", + ...override, + context: { existingAgents: [], templates: [] }, + }), { "Content-Type": "application/json" }); + + expect(res.status).toBe(400); + expect(mockStartAgentOnboardingSession).not.toHaveBeenCalled(); + }); + it("rejects invalid mode", async () => { const app = setupApp(); const res = await request(app, "POST", "/api/agents/onboarding/start-streaming", JSON.stringify({ diff --git a/packages/dashboard/src/routes/register-agent-import-export-generation-routes.ts b/packages/dashboard/src/routes/register-agent-import-export-generation-routes.ts index ba4d7a60ef..1694cbc207 100644 --- a/packages/dashboard/src/routes/register-agent-import-export-generation-routes.ts +++ b/packages/dashboard/src/routes/register-agent-import-export-generation-routes.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { Readable } from "node:stream"; import { pipeline as streamPipeline } from "node:stream/promises"; +import { applyTestModeOverrides, resolvePlanningSettingsModel } from "@fusion/core"; import { listEligibleExecutorAgents } from "@fusion/engine"; import { ApiError, badRequest, notFound, rateLimited } from "../api-error.js"; import { createSessionDiagnostics } from "../ai-session-diagnostics.js"; @@ -898,6 +899,26 @@ export function registerAgentGenerationRoutes(ctx: ApiRoutesContext): void { throw badRequest("intent is required and must be a string"); } + const hasPlanningProvider = planningModelProvider !== undefined; + const hasPlanningModelId = planningModelId !== undefined; + if (hasPlanningProvider !== hasPlanningModelId) { + throw badRequest("planningModelProvider and planningModelId must be provided together"); + } + if ( + hasPlanningProvider + && (typeof planningModelProvider !== "string" || planningModelProvider.trim().length === 0) + ) { + throw badRequest("planningModelProvider must be a non-empty string when provided"); + } + if ( + hasPlanningModelId + && (typeof planningModelId !== "string" || planningModelId.trim().length === 0) + ) { + throw badRequest("planningModelId must be a non-empty string when provided"); + } + const explicitPlanningProvider = hasPlanningProvider ? planningModelProvider!.trim() : undefined; + const explicitPlanningModelId = hasPlanningModelId ? planningModelId!.trim() : undefined; + const resolvedMode = mode ?? context?.mode ?? "create"; if (resolvedMode !== "create" && resolvedMode !== "edit") { throw badRequest("mode must be 'create' or 'edit'"); @@ -907,6 +928,14 @@ export function registerAgentGenerationRoutes(ctx: ApiRoutesContext): void { const { store: scopedStore } = await getProjectContext(req); const settings = await scopedStore.getSettings(); + const resolvedPlanningSettings = resolvePlanningSettingsModel(settings); + const hasExplicitPlanningModel = Boolean(explicitPlanningProvider && explicitPlanningModelId); + const resolvedPlanningModel = applyTestModeOverrides( + hasExplicitPlanningModel + ? { provider: explicitPlanningProvider, modelId: explicitPlanningModelId } + : resolvedPlanningSettings, + settings, + ); const ip = req.ip || req.socket.remoteAddress || "unknown"; const { startAgentOnboardingSession } = await import("../agent-onboarding.js"); const sessionId = await startAgentOnboardingSession( @@ -919,8 +948,8 @@ export function registerAgentGenerationRoutes(ctx: ApiRoutesContext): void { existingAgentConfig: resolvedExistingAgentConfig, }, scopedStore.getRootDir(), - planningModelProvider, - planningModelId, + resolvedPlanningModel.provider, + resolvedPlanningModel.modelId, settings.promptOverrides, options?.pluginRunner as Parameters[3], scopedStore,