Fix agent AI interview model routing (#2142)
## Summary - resolve the configured planning model when agent onboarding requests omit an explicit override - align the onboarding prompt with supported runtime/model hint fields, allowing AI-created agents to select runtimes such as Hermes - refresh the generated GitHub issue import limits required by the repository sync gate ## Root cause The agent onboarding route loaded project settings but passed only request-body model fields. The AI Interview UI omits those fields, so `createFnAgent` was called with `provider=undefined, model=undefined`; the session returned no usable assistant JSON. The prompt catalog also prohibited `runtimeHint` despite the parser and form already supporting it. ## Verification - targeted agent onboarding tests: 22 passed - `pnpm --filter @fusion/core typecheck` - `pnpm --filter @fusion/dashboard typecheck` - `pnpm lint` - `pnpm build` - `pnpm smoke:boot` - engine merge-gate subset: 294 passed Full `pnpm test` reached the PostgreSQL gate but this host has no `psql` binary, so 23 PostgreSQL suites could not start; this is an environment prerequisite failure, not a test assertion failure. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Bug Fixes** * Agent onboarding interviews now use the configured planning model when no override is provided. * Runtime suggestions and runtime-hint guidance are preserved during onboarding and reflected in generated configurations. * On onboarding start streaming, planning provider/model resolution now comes from settings with stricter override validation, and test mode continues to take priority. * **Documentation** * Updated onboarding prompt guidance to support additional configuration fields and optional runtime draft hints. * Reduced the maximum GitHub issue import/browse limit from 100 to 50. * **Tests** * Added coverage for runtime-hints prompting and planning-model override behavior. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
7
.changeset/bright-agent-interviews.md
Normal file
7
.changeset/bright-agent-interviews.md
Normal file
@@ -0,0 +1,7 @@
|
||||
---
|
||||
"@runfusion/fusion": patch
|
||||
---
|
||||
|
||||
summary: Fix agent AI interviews to use the configured planning model and preserve runtime suggestions.
|
||||
category: fix
|
||||
dev: Resolves onboarding model settings before session creation and aligns the prompt with supported runtime draft fields.
|
||||
@@ -303,7 +303,7 @@ Import GitHub issues as Fusion tasks. Fetches open issues from a repository and
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|------|----------|-------------|
|
||||
| `ownerRepo` | string | ✓ | Repository in owner/repo format (e.g., 'dustinbyrne/fusion') |
|
||||
| `limit` | number | — | Max issues to import (default: 30, max: 100) |
|
||||
| `limit` | number | — | Max issues to import (default: 30, max: 50) |
|
||||
| `labels` | array | — | Label names to filter by |
|
||||
|
||||
### fn_task_import_github_issue
|
||||
@@ -324,7 +324,7 @@ List open GitHub issues from a repository to browse before importing. Returns is
|
||||
|-----------|------|----------|-------------|
|
||||
| `owner` | string | ✓ | Repository owner (e.g., 'dustinbyrne') |
|
||||
| `repo` | string | ✓ | Repository name (e.g., 'fusion') |
|
||||
| `limit` | number | — | Max issues to show (default: 30, max: 100) |
|
||||
| `limit` | number | — | Max issues to show (default: 30, max: 50) |
|
||||
| `labels` | array | — | Label names to filter by |
|
||||
|
||||
## Mission Tools
|
||||
|
||||
@@ -315,13 +315,17 @@ Ask targeted questions using this JSON format:
|
||||
{"type":"question","data":{"id":"q1","type":"text|single_select|multi_select|confirm","question":"...","description":"...","options":[{"id":"x","label":"X","description":"..."}]}}
|
||||
|
||||
When ready, return a final summary JSON in this exact format:
|
||||
{"type":"complete","data":{"name":"...","role":"executor","instructionsText":"...","thinkingLevel":"medium","maxTurns":25,"title":"...","icon":"🤖","reportsTo":"...","soul":"...","memory":"...","skills":["..."],"templateId":"...","patternAgentId":"...","rationale":"..."}}
|
||||
{"type":"complete","data":{"name":"...","role":"executor","instructionsText":"...","thinkingLevel":"medium","maxTurns":25,"title":"...","icon":"🤖","reportsTo":"...","soul":"...","memory":"...","skills":["..."],"templateId":"...","patternAgentId":"...","rationale":"...","heartbeatProcedurePath":"...","heartbeatIntervalMs":30000,"heartbeatEnabled":true,"modelHint":"...","runtimeHint":"..."}}
|
||||
|
||||
Rules:
|
||||
- role must be one of triage|executor|reviewer|merger|scheduler|engineer|custom
|
||||
- thinkingLevel must be off|minimal|low|medium|high
|
||||
- maxTurns must be a positive integer
|
||||
- Do not include runtimeMode/model/runtimeHint; those are user review-time choices.`,
|
||||
- Use instructionsText for starter operating guidance/playbook content; do not create a separate playbook field
|
||||
- Prefer structuring instructionsText with these markdown sections when drafting: ## Description, ## Expertise, ## Priorities, ## Boundaries, ## Communication, ## Collaboration & Escalation
|
||||
- Freeform instructionsText is still acceptable for compatibility; sectioned structure is preferred for new agents
|
||||
- modelHint and runtimeHint are optional draft suggestions only (not final runtime selection)
|
||||
- heartbeatProcedurePath, heartbeatIntervalMs, and heartbeatEnabled are optional draft hints only.`,
|
||||
},
|
||||
"subtask-breakdown-system": {
|
||||
key: "subtask-breakdown-system",
|
||||
|
||||
@@ -271,6 +271,28 @@ describe("agent-onboarding", () => {
|
||||
expect(prompt).toContain("messageResponseMode: immediate");
|
||||
});
|
||||
|
||||
it("asks the onboarding model for runtime hints that can select Hermes", async () => {
|
||||
mockCreateFnAgent.mockResolvedValueOnce(
|
||||
createMockAgent([
|
||||
JSON.stringify({
|
||||
type: "question",
|
||||
data: { id: "goal", type: "text", question: "What is the primary goal?" },
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
await startAgentOnboardingSession(
|
||||
"127.0.0.1",
|
||||
{ intent: "Use the Hermes runtime for computer-use testing", existingAgents: [], templates: [] },
|
||||
process.cwd(),
|
||||
);
|
||||
|
||||
const options = mockCreateFnAgent.mock.calls.at(-1)?.[0] as { systemPrompt?: string };
|
||||
expect(options.systemPrompt).toContain('"runtimeHint"');
|
||||
expect(options.systemPrompt).toContain("optional draft suggestions");
|
||||
expect(options.systemPrompt).not.toContain("Do not include runtimeMode/model/runtimeHint");
|
||||
});
|
||||
|
||||
it("requests role-fallback and enabled plugin skills for model-only onboarding agents", async () => {
|
||||
mockCreateFnAgent.mockResolvedValueOnce(
|
||||
createMockAgent([
|
||||
|
||||
@@ -63,10 +63,10 @@ function createMockStore(): TaskStore {
|
||||
} as unknown as TaskStore;
|
||||
}
|
||||
|
||||
function setupApp() {
|
||||
function setupApp(store = createMockStore()) {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use("/api", createApiRoutes(createMockStore()));
|
||||
app.use("/api", createApiRoutes(store));
|
||||
return app;
|
||||
}
|
||||
|
||||
@@ -88,6 +88,46 @@ describe("agent onboarding routes", () => {
|
||||
expect(mockStartAgentOnboardingSession.mock.calls[0]?.[1]).toMatchObject({ mode: "create" });
|
||||
});
|
||||
|
||||
it("uses the configured planning model when the request omits an override", async () => {
|
||||
const store = createMockStore();
|
||||
vi.mocked(store.getSettings).mockResolvedValue({
|
||||
planningProvider: "openai-codex",
|
||||
planningModelId: "gpt-5.6-sol",
|
||||
} as Awaited<ReturnType<TaskStore["getSettings"]>>);
|
||||
const app = setupApp(store);
|
||||
|
||||
const res = await request(app, "POST", "/api/agents/onboarding/start-streaming", JSON.stringify({
|
||||
intent: "Create a computer-use tester",
|
||||
context: { existingAgents: [], templates: [] },
|
||||
}), { "Content-Type": "application/json" });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockStartAgentOnboardingSession).toHaveBeenCalledTimes(1);
|
||||
expect(mockStartAgentOnboardingSession.mock.calls[0]?.[3]).toBe("openai-codex");
|
||||
expect(mockStartAgentOnboardingSession.mock.calls[0]?.[4]).toBe("gpt-5.6-sol");
|
||||
});
|
||||
|
||||
it("keeps test mode authoritative over explicit request model overrides", async () => {
|
||||
const store = createMockStore();
|
||||
vi.mocked(store.getSettings).mockResolvedValue({
|
||||
testMode: true,
|
||||
planningProvider: "openai-codex",
|
||||
planningModelId: "gpt-5.6-sol",
|
||||
} as Awaited<ReturnType<TaskStore["getSettings"]>>);
|
||||
const app = setupApp(store);
|
||||
|
||||
const res = await request(app, "POST", "/api/agents/onboarding/start-streaming", JSON.stringify({
|
||||
intent: "Create a test agent",
|
||||
planningModelProvider: "anthropic",
|
||||
planningModelId: "claude-sonnet-4-5",
|
||||
context: { existingAgents: [], templates: [] },
|
||||
}), { "Content-Type": "application/json" });
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(mockStartAgentOnboardingSession.mock.calls[0]?.[3]).toBe("mock");
|
||||
expect(mockStartAgentOnboardingSession.mock.calls[0]?.[4]).toBe("scripted");
|
||||
});
|
||||
|
||||
it("accepts edit mode and forwards existingAgentConfig", async () => {
|
||||
const app = setupApp();
|
||||
const res = await request(app, "POST", "/api/agents/onboarding/start-streaming", JSON.stringify({
|
||||
@@ -113,6 +153,25 @@ describe("agent onboarding routes", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ planningModelProvider: 42, planningModelId: "gpt-5.6-sol" },
|
||||
{ planningModelProvider: "openai-codex", planningModelId: {} },
|
||||
{ planningModelProvider: " ", planningModelId: "gpt-5.6-sol" },
|
||||
{ planningModelProvider: "openai-codex", planningModelId: " " },
|
||||
{ planningModelProvider: "openai-codex" },
|
||||
{ planningModelId: "gpt-5.6-sol" },
|
||||
])("rejects invalid planning model overrides: %j", async (override) => {
|
||||
const app = setupApp();
|
||||
const res = await request(app, "POST", "/api/agents/onboarding/start-streaming", JSON.stringify({
|
||||
intent: "Create an agent",
|
||||
...override,
|
||||
context: { existingAgents: [], templates: [] },
|
||||
}), { "Content-Type": "application/json" });
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
expect(mockStartAgentOnboardingSession).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects invalid mode", async () => {
|
||||
const app = setupApp();
|
||||
const res = await request(app, "POST", "/api/agents/onboarding/start-streaming", JSON.stringify({
|
||||
|
||||
@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { pipeline as streamPipeline } from "node:stream/promises";
|
||||
import { applyTestModeOverrides, resolvePlanningSettingsModel } from "@fusion/core";
|
||||
import { listEligibleExecutorAgents } from "@fusion/engine";
|
||||
import { ApiError, badRequest, notFound, rateLimited } from "../api-error.js";
|
||||
import { createSessionDiagnostics } from "../ai-session-diagnostics.js";
|
||||
@@ -898,6 +899,26 @@ export function registerAgentGenerationRoutes(ctx: ApiRoutesContext): void {
|
||||
throw badRequest("intent is required and must be a string");
|
||||
}
|
||||
|
||||
const hasPlanningProvider = planningModelProvider !== undefined;
|
||||
const hasPlanningModelId = planningModelId !== undefined;
|
||||
if (hasPlanningProvider !== hasPlanningModelId) {
|
||||
throw badRequest("planningModelProvider and planningModelId must be provided together");
|
||||
}
|
||||
if (
|
||||
hasPlanningProvider
|
||||
&& (typeof planningModelProvider !== "string" || planningModelProvider.trim().length === 0)
|
||||
) {
|
||||
throw badRequest("planningModelProvider must be a non-empty string when provided");
|
||||
}
|
||||
if (
|
||||
hasPlanningModelId
|
||||
&& (typeof planningModelId !== "string" || planningModelId.trim().length === 0)
|
||||
) {
|
||||
throw badRequest("planningModelId must be a non-empty string when provided");
|
||||
}
|
||||
const explicitPlanningProvider = hasPlanningProvider ? planningModelProvider!.trim() : undefined;
|
||||
const explicitPlanningModelId = hasPlanningModelId ? planningModelId!.trim() : undefined;
|
||||
|
||||
const resolvedMode = mode ?? context?.mode ?? "create";
|
||||
if (resolvedMode !== "create" && resolvedMode !== "edit") {
|
||||
throw badRequest("mode must be 'create' or 'edit'");
|
||||
@@ -907,6 +928,14 @@ export function registerAgentGenerationRoutes(ctx: ApiRoutesContext): void {
|
||||
|
||||
const { store: scopedStore } = await getProjectContext(req);
|
||||
const settings = await scopedStore.getSettings();
|
||||
const resolvedPlanningSettings = resolvePlanningSettingsModel(settings);
|
||||
const hasExplicitPlanningModel = Boolean(explicitPlanningProvider && explicitPlanningModelId);
|
||||
const resolvedPlanningModel = applyTestModeOverrides(
|
||||
hasExplicitPlanningModel
|
||||
? { provider: explicitPlanningProvider, modelId: explicitPlanningModelId }
|
||||
: resolvedPlanningSettings,
|
||||
settings,
|
||||
);
|
||||
const ip = req.ip || req.socket.remoteAddress || "unknown";
|
||||
const { startAgentOnboardingSession } = await import("../agent-onboarding.js");
|
||||
const sessionId = await startAgentOnboardingSession(
|
||||
@@ -919,8 +948,8 @@ export function registerAgentGenerationRoutes(ctx: ApiRoutesContext): void {
|
||||
existingAgentConfig: resolvedExistingAgentConfig,
|
||||
},
|
||||
scopedStore.getRootDir(),
|
||||
planningModelProvider,
|
||||
planningModelId,
|
||||
resolvedPlanningModel.provider,
|
||||
resolvedPlanningModel.modelId,
|
||||
settings.promptOverrides,
|
||||
options?.pluginRunner as Parameters<typeof import("@fusion/engine").buildSessionSkillContextSync>[3],
|
||||
scopedStore,
|
||||
|
||||
Reference in New Issue
Block a user