From 5f4ad172f4c2cf35ba4b88c60a855e11392220c2 Mon Sep 17 00:00:00 2001 From: gsxdsm Date: Sat, 11 Jul 2026 23:47:31 -0700 Subject: [PATCH] fix: give Grok ACP absolute paths for chat image attachments Grok advertises promptCapabilities.image=false and ignores ACP image ContentBlocks (live probe: NO_IMAGE). Path-based vision works when the agent is given an absolute file path. Include path hints in chat prompts from .fusion/chat-attachments and carry path on ChatImageContent for file:// uris. --- .../__tests__/chat-attachment-content.test.ts | 50 +++++++++++++++++-- .../dashboard/src/chat-attachment-content.ts | 39 +++++++++++++++ packages/dashboard/src/chat.ts | 20 +++++++- .../src/__tests__/prompt-builder.test.ts | 8 +++ .../src/prompt-builder.ts | 6 ++- .../src/acp/prompt-builder.ts | 6 ++- 6 files changed, 120 insertions(+), 9 deletions(-) diff --git a/packages/dashboard/src/__tests__/chat-attachment-content.test.ts b/packages/dashboard/src/__tests__/chat-attachment-content.test.ts index 783e45a623..a9527b2c41 100644 --- a/packages/dashboard/src/__tests__/chat-attachment-content.test.ts +++ b/packages/dashboard/src/__tests__/chat-attachment-content.test.ts @@ -6,6 +6,7 @@ import { tmpdir } from "node:os"; import { CHAT_TEXT_INLINE_LIMIT, formatChatAttachmentContents, + formatChatImageAttachmentHints, readChatAttachmentContents, } from "../chat-attachment-content.js"; @@ -57,6 +58,7 @@ describe("readChatAttachmentContents", () => { const root = await makeRoot(); await mkdir(join(root, ".fusion", "chat-attachments", "session-1"), { recursive: true }); await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "image.png"), PNG_BYTES); + const expectedPath = join(root, ".fusion", "chat-attachments", "session-1", "image.png"); const result = await readChatAttachmentContents(root, { kind: "session", sessionId: "session-1" }, [ attachment({ filename: "image.png", originalName: "image.png", mimeType: "image/png", size: PNG_BYTES.length }), @@ -66,9 +68,17 @@ describe("readChatAttachmentContents", () => { { originalName: "image.png", mimeType: "image/png", text: null }, ]); expect(result.imageContents).toEqual([ - { type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" }, + { + type: "image", + data: PNG_BYTES.toString("base64"), + mimeType: "image/png", + path: expectedPath, + originalName: "image.png", + }, ]); expect(formatChatAttachmentContents(result.attachmentContents)).toBe(""); + expect(formatChatImageAttachmentHints(result.imageContents)).toContain(expectedPath); + expect(formatChatImageAttachmentHints(result.imageContents)).toContain("vision/file tools"); }); it("corrects session webp-labeled PNG image blocks to image/png", async () => { @@ -82,7 +92,15 @@ describe("readChatAttachmentContents", () => { ], diagnostics); expect(result.attachmentContents).toEqual([{ originalName: "mismatch.webp", mimeType: "image/webp", text: null }]); - expect(result.imageContents).toEqual([{ type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" }]); + expect(result.imageContents).toEqual([ + { + type: "image", + data: PNG_BYTES.toString("base64"), + mimeType: "image/png", + path: join(root, ".fusion", "chat-attachments", "session-1", "mismatch.webp"), + originalName: "mismatch.webp", + }, + ]); expect(diagnostics.warn).toHaveBeenCalledWith(expect.stringContaining("from image/webp to image/png")); }); @@ -97,7 +115,15 @@ describe("readChatAttachmentContents", () => { ], diagnostics); expect(result.attachmentContents).toEqual([{ originalName: "mismatch.png", mimeType: "image/png", text: null }]); - expect(result.imageContents).toEqual([{ type: "image", data: WEBP_BYTES.toString("base64"), mimeType: "image/webp" }]); + expect(result.imageContents).toEqual([ + { + type: "image", + data: WEBP_BYTES.toString("base64"), + mimeType: "image/webp", + path: join(root, ".fusion", "chat-room-attachments", "room-1", "mismatch.png"), + originalName: "mismatch.png", + }, + ]); expect(diagnostics.warn).toHaveBeenCalledWith(expect.stringContaining("room room-1 from image/png to image/webp")); }); @@ -111,7 +137,15 @@ describe("readChatAttachmentContents", () => { ]); expect(result.attachmentContents).toEqual([{ originalName: "unknown.webp", mimeType: "image/webp", text: null }]); - expect(result.imageContents).toEqual([{ type: "image", data: UNKNOWN_IMAGE_BYTES.toString("base64"), mimeType: "image/webp" }]); + expect(result.imageContents).toEqual([ + { + type: "image", + data: UNKNOWN_IMAGE_BYTES.toString("base64"), + mimeType: "image/webp", + path: join(root, ".fusion", "chat-attachments", "session-1", "unknown.webp"), + originalName: "unknown.webp", + }, + ]); }); it("returns mixed text and image contents together", async () => { @@ -127,7 +161,13 @@ describe("readChatAttachmentContents", () => { expect(formatChatAttachmentContents(result.attachmentContents)).toContain("```json\n{\"ok\":true}\n```"); expect(result.imageContents).toEqual([ - { type: "image", data: Buffer.from("webp").toString("base64"), mimeType: "image/webp" }, + { + type: "image", + data: Buffer.from("webp").toString("base64"), + mimeType: "image/webp", + path: join(root, ".fusion", "chat-room-attachments", "room-1", "photo.webp"), + originalName: "photo.webp", + }, ]); }); diff --git a/packages/dashboard/src/chat-attachment-content.ts b/packages/dashboard/src/chat-attachment-content.ts index ebbd5aafc5..9504cbbea3 100644 --- a/packages/dashboard/src/chat-attachment-content.ts +++ b/packages/dashboard/src/chat-attachment-content.ts @@ -7,6 +7,16 @@ export interface ChatImageContent { type: "image"; data: string; mimeType: string; + /** + * FNXC:GrokAcp 2026-07-12-07:30: + * Absolute path to the on-disk attachment. Grok ACP advertises + * `promptCapabilities.image: false` and ignores ACP image ContentBlocks; + * agents must open this path with vision/file tools to see pixels. Pi still + * uses `data`/`mimeType` as ImageContent. + */ + path: string; + /** User-facing filename for prompt hints. */ + originalName: string; } export interface ChatAttachmentContent { @@ -81,6 +91,11 @@ function escapeFence(text: string): string { * * FNXC:ChatAttachments 2026-06-16-19:55: * Text attachments are prompt-inlined with the triage-compatible 50KB ceiling while image attachments are forwarded as pi image content blocks through promptWithFallback options. + * + * FNXC:GrokAcp 2026-07-12-07:30: + * Image attachments also carry absolute `path` and prompt path-hints. Grok ACP + * sets promptCapabilities.image=false so ContentBlocks alone are invisible; + * path hints let the agent open pixels via filesystem vision tools. */ export async function readChatAttachmentContents( rootDir: string, @@ -118,6 +133,8 @@ export async function readChatAttachmentContents( type: "image", data: data.toString("base64"), mimeType: imageMimeType, + path: filePath, + originalName: attachment.originalName, }); attachmentContents.push({ originalName: attachment.originalName, @@ -166,3 +183,25 @@ export function formatChatAttachmentContents(attachmentContents: ChatAttachmentC ].join("\n")), ].join("\n\n"); } + +/** + * FNXC:GrokAcp 2026-07-12-07:30: + * Build explicit absolute-path hints for image attachments. Name/size-only + * summaries look like "text placeholders" to CLI agents that cannot ingest ACP + * image ContentBlocks (Grok promptCapabilities.image=false). Paths point at + * files already stored under .fusion/chat-attachments or chat-room-attachments. + */ +export function formatChatImageAttachmentHints(imageContents: ChatImageContent[]): string { + if (imageContents.length === 0) { + return ""; + } + + return [ + "## Image attachments (filesystem paths)", + "The user attached image file(s). Open each absolute path with vision/file tools to inspect the actual pixels — do not invent contents from the filename alone:", + ...imageContents.map( + (image) => + `- ${image.originalName} (${image.mimeType}): ${image.path}`, + ), + ].join("\n"); +} diff --git a/packages/dashboard/src/chat.ts b/packages/dashboard/src/chat.ts index 1c01ae60dc..354916054b 100644 --- a/packages/dashboard/src/chat.ts +++ b/packages/dashboard/src/chat.ts @@ -35,7 +35,11 @@ import { existsSync } from "node:fs"; import { join, resolve, relative } from "node:path"; import { SessionManager } from "@earendil-works/pi-coding-agent"; import { SessionEventBuffer } from "./sse-buffer.js"; -import { formatChatAttachmentContents, readChatAttachmentContents } from "./chat-attachment-content.js"; +import { + formatChatAttachmentContents, + formatChatImageAttachmentHints, + readChatAttachmentContents, +} from "./chat-attachment-content.js"; import { buildTaskPlannerChatContext, TASK_PLANNER_CHAT_CONTEXT_PROMPT_GUIDANCE } from "./task-planner-chat-context.js"; import { formatTaskPlannerChatMetrics } from "./task-planner-chat-metrics.js"; import { emitWorkflowSseEvent, type WorkflowSseEventType } from "./sse.js"; @@ -1707,6 +1711,7 @@ export class ChatManager { diagnostics, ); const attachmentContentBlock = formatChatAttachmentContents(attachmentContents); + const imagePathHints = formatChatImageAttachmentHints(imageContents); const parsedSkillCommands = parseSkillCommands(input.content); const roomPromptParts = [ `You are replying as ${input.responder.name} in room #${input.roomName}.`, @@ -1722,6 +1727,9 @@ export class ChatManager { if (attachmentContentBlock) { roomPromptParts.push(attachmentContentBlock); } + if (imagePathHints) { + roomPromptParts.push(imagePathHints); + } const roomPrompt = roomPromptParts.join("\n\n"); const responderRuntimeModel = extractRuntimeModel(input.responder.runtimeConfig); @@ -2170,12 +2178,20 @@ export class ChatManager { diagnostics, ); const attachmentContentBlock = formatChatAttachmentContents(attachmentContents); + /* + FNXC:GrokAcp 2026-07-12-07:30: + Name/size-only attachmentSummary is not enough for Grok ACP (image ContentBlocks + are unsupported). Include absolute filesystem paths so the agent can open pixels. + */ + const imagePathHints = formatChatImageAttachmentHints(imageContents); // Send only the new user content. Prior turns are reloaded by the // pi/Claude CLI session via SessionManager.open() below — stuffing the // transcript back into the user message would balloon the on-disk // session every turn (and previously did, see chat-store.ts:setCliSessionFile). - const promptContent = [attachmentSummary, attachmentContentBlock, resolvedContent].filter(Boolean).join("\n\n"); + const promptContent = [attachmentSummary, imagePathHints, attachmentContentBlock, resolvedContent] + .filter(Boolean) + .join("\n\n"); // Per-chat session continuity: the pi SessionManager (and, transitively, // the Claude CLI --resume session it owns) is keyed off the chat. On the diff --git a/plugins/fusion-plugin-acp-runtime/src/__tests__/prompt-builder.test.ts b/plugins/fusion-plugin-acp-runtime/src/__tests__/prompt-builder.test.ts index 70435c0dd7..32ee607b42 100644 --- a/plugins/fusion-plugin-acp-runtime/src/__tests__/prompt-builder.test.ts +++ b/plugins/fusion-plugin-acp-runtime/src/__tests__/prompt-builder.test.ts @@ -14,6 +14,14 @@ describe("extractPromptImagesFromOptions", () => { ).toEqual([{ data: "AAAA", mimeType: "image/png" }]); }); + it("maps absolute path to file:// uri", () => { + expect( + extractPromptImagesFromOptions({ + images: [{ type: "image", data: "AAAA", mimeType: "image/png", path: "/tmp/photo.png" }], + }), + ).toEqual([{ data: "AAAA", mimeType: "image/png", uri: "file:///tmp/photo.png" }]); + }); + it("keeps uri when present and drops malformed entries", () => { expect( extractPromptImagesFromOptions({ diff --git a/plugins/fusion-plugin-acp-runtime/src/prompt-builder.ts b/plugins/fusion-plugin-acp-runtime/src/prompt-builder.ts index 6cb827d70a..47a447b11b 100644 --- a/plugins/fusion-plugin-acp-runtime/src/prompt-builder.ts +++ b/plugins/fusion-plugin-acp-runtime/src/prompt-builder.ts @@ -48,7 +48,11 @@ export function extractPromptImagesFromOptions(options: unknown): PromptImage[] const data = typeof rec.data === "string" ? rec.data : undefined; const mimeType = typeof rec.mimeType === "string" ? rec.mimeType : undefined; if (!data || !mimeType || data.length === 0 || mimeType.length === 0) continue; - const uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined; + // Prefer explicit uri; else map absolute filesystem `path` to file:// for agents. + let uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined; + if (!uri && typeof rec.path === "string" && rec.path.length > 0) { + uri = rec.path.startsWith("file:") ? rec.path : `file://${rec.path}`; + } images.push({ data, mimeType, ...(uri ? { uri } : {}) }); } return images.length > 0 ? images : undefined; diff --git a/plugins/fusion-plugin-grok-runtime/src/acp/prompt-builder.ts b/plugins/fusion-plugin-grok-runtime/src/acp/prompt-builder.ts index fc93facd42..e2576156d3 100644 --- a/plugins/fusion-plugin-grok-runtime/src/acp/prompt-builder.ts +++ b/plugins/fusion-plugin-grok-runtime/src/acp/prompt-builder.ts @@ -49,7 +49,11 @@ export function extractPromptImagesFromOptions(options: unknown): PromptImage[] const data = typeof rec.data === "string" ? rec.data : undefined; const mimeType = typeof rec.mimeType === "string" ? rec.mimeType : undefined; if (!data || !mimeType || data.length === 0 || mimeType.length === 0) continue; - const uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined; + // Prefer explicit uri; else map absolute filesystem `path` to file:// for agents. + let uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined; + if (!uri && typeof rec.path === "string" && rec.path.length > 0) { + uri = rec.path.startsWith("file:") ? rec.path : `file://${rec.path}`; + } images.push({ data, mimeType, ...(uri ? { uri } : {}) }); } return images.length > 0 ? images : undefined;