From c2d3bf0e8a5e8ef7be8ff36c4ebbaa47baa993b1 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 11 Jul 2026 23:44:14 -0700 Subject: [PATCH] fix: forward chat image attachments into Grok/ACP session/prompt AcpRuntimeAdapter.promptWithFallback ignored options, so dashboard chat images never became ACP ContentBlock image entries. Extract images from prompt options and pass them through buildPromptBlocks for both acp-runtime and the Grok vendored client. --- .../src/__tests__/fixtures/echo-agent.mjs | 8 ++++- .../src/__tests__/prompt-builder.test.ts | 35 ++++++++++++++++++- .../src/__tests__/runtime-adapter.test.ts | 26 ++++++++++++++ .../src/prompt-builder.ts | 33 +++++++++++++++++ .../src/runtime-adapter.ts | 13 +++++-- .../src/acp/prompt-builder.ts | 33 +++++++++++++++++ .../src/acp/runtime-adapter.ts | 13 +++++-- 7 files changed, 153 insertions(+), 8 deletions(-) diff --git a/plugins/fusion-plugin-acp-runtime/src/__tests__/fixtures/echo-agent.mjs b/plugins/fusion-plugin-acp-runtime/src/__tests__/fixtures/echo-agent.mjs index 84fc3c10cd..308c843c34 100644 --- a/plugins/fusion-plugin-acp-runtime/src/__tests__/fixtures/echo-agent.mjs +++ b/plugins/fusion-plugin-acp-runtime/src/__tests__/fixtures/echo-agent.mjs @@ -125,11 +125,17 @@ class EchoAgent { }); return { stopReason: "end_turn" }; } + // FNXC:GrokAcp 2026-07-12-07:15: report image ContentBlock count so + // adapter tests can prove chat image options reach session/prompt. + const imageCount = Array.isArray(params.prompt) + ? params.prompt.filter((block) => block && block.type === "image").length + : 0; + const replyText = imageCount > 0 ? `echo: images=${imageCount}` : "echo: hello"; await this.connection.sessionUpdate({ sessionId: params.sessionId, update: { sessionUpdate: "agent_message_chunk", - content: { type: "text", text: "echo: hello" }, + content: { type: "text", text: replyText }, }, }); // Cancel-mid-prompt test: keep the turn open until cancel() arrives, then diff --git a/plugins/fusion-plugin-acp-runtime/src/__tests__/prompt-builder.test.ts b/plugins/fusion-plugin-acp-runtime/src/__tests__/prompt-builder.test.ts index 8888125e86..70435c0dd7 100644 --- a/plugins/fusion-plugin-acp-runtime/src/__tests__/prompt-builder.test.ts +++ b/plugins/fusion-plugin-acp-runtime/src/__tests__/prompt-builder.test.ts @@ -1,5 +1,38 @@ import { describe, it, expect } from "vitest"; -import { buildPromptBlocks } from "../prompt-builder.js"; +import { buildPromptBlocks, extractPromptImagesFromOptions } from "../prompt-builder.js"; + +describe("extractPromptImagesFromOptions", () => { + /* + FNXC:GrokAcp 2026-07-12-07:15: + Chat attachment options must map into PromptImage for ACP session/prompt. + */ + it("extracts chat-style image contents", () => { + expect( + extractPromptImagesFromOptions({ + images: [{ type: "image", data: "AAAA", mimeType: "image/png" }], + }), + ).toEqual([{ data: "AAAA", mimeType: "image/png" }]); + }); + + it("keeps uri when present and drops malformed entries", () => { + expect( + extractPromptImagesFromOptions({ + images: [ + { data: "AAAA", mimeType: "image/png", uri: "file:///a.png" }, + { data: "", mimeType: "image/png" }, + { mimeType: "image/jpeg" }, + null, + ], + }), + ).toEqual([{ data: "AAAA", mimeType: "image/png", uri: "file:///a.png" }]); + }); + + it("returns undefined for missing or empty images", () => { + expect(extractPromptImagesFromOptions(undefined)).toBeUndefined(); + expect(extractPromptImagesFromOptions({})).toBeUndefined(); + expect(extractPromptImagesFromOptions({ images: [] })).toBeUndefined(); + }); +}); describe("buildPromptBlocks", () => { it("turns a plain string into a single text block", () => { diff --git a/plugins/fusion-plugin-acp-runtime/src/__tests__/runtime-adapter.test.ts b/plugins/fusion-plugin-acp-runtime/src/__tests__/runtime-adapter.test.ts index 94c1b63742..8414a0cb22 100644 --- a/plugins/fusion-plugin-acp-runtime/src/__tests__/runtime-adapter.test.ts +++ b/plugins/fusion-plugin-acp-runtime/src/__tests__/runtime-adapter.test.ts @@ -68,6 +68,32 @@ describe("AcpRuntimeAdapter (U3)", () => { } }); + /* + FNXC:GrokAcp 2026-07-12-07:15: + Chat image attachments must arrive as ACP image ContentBlocks on session/prompt. + */ + it("promptWithFallback forwards chat image options into session/prompt", async () => { + const chunks: string[] = []; + const adapter = makeAdapter(); + const { session } = await adapter.createSession( + makeOptions({ + onText: (t) => { + chunks.push(t); + }, + }), + ); + try { + await expect( + adapter.promptWithFallback(session, "describe this", { + images: [{ type: "image", data: "AAAA", mimeType: "image/png" }], + }), + ).resolves.toEqual({ stopReason: "end_turn" }); + expect(chunks.join("")).toContain("images=1"); + } finally { + await adapter.dispose(session); + } + }); + it("dispose tears down the subprocess and is idempotent", async () => { const adapter = makeAdapter(); const { session } = await adapter.createSession(makeOptions()); diff --git a/plugins/fusion-plugin-acp-runtime/src/prompt-builder.ts b/plugins/fusion-plugin-acp-runtime/src/prompt-builder.ts index 4bbda15e49..6cb827d70a 100644 --- a/plugins/fusion-plugin-acp-runtime/src/prompt-builder.ts +++ b/plugins/fusion-plugin-acp-runtime/src/prompt-builder.ts @@ -21,6 +21,39 @@ export interface BuildPromptOptions { images?: PromptImage[]; } +/* +FNXC:GrokAcp 2026-07-12-07:15: +Dashboard chat forwards attachments as promptWithFallback options +`{ images: ChatImageContent[] }` where each item is +`{ type: "image", data: base64, mimeType }`. ACP session/prompt needs +ContentBlock image variants. Extract defensively so pi-style ImageContent +and PromptImage shapes both work; ignore malformed entries. +*/ +/** + * Pull image attachments from Fusion `promptWithFallback` options. + * Accepts `{ images: Array<{ data, mimeType, uri? }> }` (chat / pi ImageContent). + */ +export function extractPromptImagesFromOptions(options: unknown): PromptImage[] | undefined { + if (!options || typeof options !== "object" || Array.isArray(options)) { + return undefined; + } + const raw = (options as { images?: unknown }).images; + if (!Array.isArray(raw) || raw.length === 0) { + return undefined; + } + const images: PromptImage[] = []; + for (const item of raw) { + if (!item || typeof item !== "object" || Array.isArray(item)) continue; + const rec = item as Record; + const data = typeof rec.data === "string" ? rec.data : undefined; + const mimeType = typeof rec.mimeType === "string" ? rec.mimeType : undefined; + if (!data || !mimeType || data.length === 0 || mimeType.length === 0) continue; + const uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined; + images.push({ data, mimeType, ...(uri ? { uri } : {}) }); + } + return images.length > 0 ? images : undefined; +} + /** * Build the ACP prompt content blocks for a turn. * diff --git a/plugins/fusion-plugin-acp-runtime/src/runtime-adapter.ts b/plugins/fusion-plugin-acp-runtime/src/runtime-adapter.ts index 3070793191..554d54fcd6 100644 --- a/plugins/fusion-plugin-acp-runtime/src/runtime-adapter.ts +++ b/plugins/fusion-plugin-acp-runtime/src/runtime-adapter.ts @@ -17,7 +17,7 @@ import { createBridgingClientHandler, } from "./provider.js"; import { buildSpawnEnv } from "./process-manager.js"; -import { buildPromptBlocks } from "./prompt-builder.js"; +import { buildPromptBlocks, extractPromptImagesFromOptions } from "./prompt-builder.js"; import type { AgentRuntime, AgentRuntimeOptions, @@ -141,7 +141,7 @@ export class AcpRuntimeAdapter implements AgentRuntime { async promptWithFallback( session: AgentSession, prompt: string, - _options?: unknown, + options?: unknown, ): Promise<{ stopReason?: string }> { const acp = session as AcpSession; if (!acp.connection) { @@ -152,7 +152,14 @@ export class AcpRuntimeAdapter implements AgentRuntime { // each turn (FIX 1). Without this, a turn that hit the per-turn output cap // would silently suppress all later turns. acp.resetTurn?.(); - const blocks = buildPromptBlocks(prompt); + /* + FNXC:GrokAcp 2026-07-12-07:15: + Chat/triage pass `{ images: [{ type:"image", data, mimeType }] }` through + promptWithFallback. Previously options were ignored (`_options`) so Grok ACP + and generic ACP sessions never received image ContentBlocks on session/prompt. + */ + const images = extractPromptImagesFromOptions(options); + const blocks = buildPromptBlocks(prompt, images ? { images } : undefined); // Resolve when the SDK prompt promise resolves — it already drains all // session/update notifications for the turn before reporting the stopReason. // The bridging client handler installed at createSession (U4) has already diff --git a/plugins/fusion-plugin-grok-runtime/src/acp/prompt-builder.ts b/plugins/fusion-plugin-grok-runtime/src/acp/prompt-builder.ts index b8ee656b32..fc93facd42 100644 --- a/plugins/fusion-plugin-grok-runtime/src/acp/prompt-builder.ts +++ b/plugins/fusion-plugin-grok-runtime/src/acp/prompt-builder.ts @@ -22,6 +22,39 @@ export interface BuildPromptOptions { images?: PromptImage[]; } +/* +FNXC:GrokAcp 2026-07-12-07:15: +Dashboard chat forwards attachments as promptWithFallback options +`{ images: ChatImageContent[] }` where each item is +`{ type: "image", data: base64, mimeType }`. ACP session/prompt needs +ContentBlock image variants. Extract defensively so pi-style ImageContent +and PromptImage shapes both work; ignore malformed entries. +*/ +/** + * Pull image attachments from Fusion `promptWithFallback` options. + * Accepts `{ images: Array<{ data, mimeType, uri? }> }` (chat / pi ImageContent). + */ +export function extractPromptImagesFromOptions(options: unknown): PromptImage[] | undefined { + if (!options || typeof options !== "object" || Array.isArray(options)) { + return undefined; + } + const raw = (options as { images?: unknown }).images; + if (!Array.isArray(raw) || raw.length === 0) { + return undefined; + } + const images: PromptImage[] = []; + for (const item of raw) { + if (!item || typeof item !== "object" || Array.isArray(item)) continue; + const rec = item as Record; + const data = typeof rec.data === "string" ? rec.data : undefined; + const mimeType = typeof rec.mimeType === "string" ? rec.mimeType : undefined; + if (!data || !mimeType || data.length === 0 || mimeType.length === 0) continue; + const uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined; + images.push({ data, mimeType, ...(uri ? { uri } : {}) }); + } + return images.length > 0 ? images : undefined; +} + /** * Build the ACP prompt content blocks for a turn. * diff --git a/plugins/fusion-plugin-grok-runtime/src/acp/runtime-adapter.ts b/plugins/fusion-plugin-grok-runtime/src/acp/runtime-adapter.ts index a996c8fa8b..ab204e7494 100644 --- a/plugins/fusion-plugin-grok-runtime/src/acp/runtime-adapter.ts +++ b/plugins/fusion-plugin-grok-runtime/src/acp/runtime-adapter.ts @@ -18,7 +18,7 @@ import { createBridgingClientHandler, } from "./provider.js"; import { buildSpawnEnv } from "./process-manager.js"; -import { buildPromptBlocks } from "./prompt-builder.js"; +import { buildPromptBlocks, extractPromptImagesFromOptions } from "./prompt-builder.js"; import type { AgentRuntime, AgentRuntimeOptions, @@ -142,7 +142,7 @@ export class AcpRuntimeAdapter implements AgentRuntime { async promptWithFallback( session: AgentSession, prompt: string, - _options?: unknown, + options?: unknown, ): Promise<{ stopReason?: string }> { const acp = session as AcpSession; if (!acp.connection) { @@ -153,7 +153,14 @@ export class AcpRuntimeAdapter implements AgentRuntime { // each turn (FIX 1). Without this, a turn that hit the per-turn output cap // would silently suppress all later turns. acp.resetTurn?.(); - const blocks = buildPromptBlocks(prompt); + /* + FNXC:GrokAcp 2026-07-12-07:15: + Chat/triage pass `{ images: [{ type:"image", data, mimeType }] }` through + promptWithFallback. Previously options were ignored (`_options`) so Grok ACP + and generic ACP sessions never received image ContentBlocks on session/prompt. + */ + const images = extractPromptImagesFromOptions(options); + const blocks = buildPromptBlocks(prompt, images ? { images } : undefined); // Resolve when the SDK prompt promise resolves — it already drains all // session/update notifications for the turn before reporting the stopReason. // The bridging client handler installed at createSession (U4) has already