fix: give Grok ACP absolute paths for chat image attachments

Grok advertises promptCapabilities.image=false and ignores ACP image
ContentBlocks (live probe: NO_IMAGE). Path-based vision works when the
agent is given an absolute file path. Include path hints in chat prompts
from .fusion/chat-attachments and carry path on ChatImageContent for
file:// uris.
This commit is contained in:
gsxdsm
2026-07-11 23:47:31 -07:00
parent 3d7eaeabee
commit 5f4ad172f4
6 changed files with 120 additions and 9 deletions

View File

@@ -6,6 +6,7 @@ import { tmpdir } from "node:os";
import {
CHAT_TEXT_INLINE_LIMIT,
formatChatAttachmentContents,
formatChatImageAttachmentHints,
readChatAttachmentContents,
} from "../chat-attachment-content.js";
@@ -57,6 +58,7 @@ describe("readChatAttachmentContents", () => {
const root = await makeRoot();
await mkdir(join(root, ".fusion", "chat-attachments", "session-1"), { recursive: true });
await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "image.png"), PNG_BYTES);
const expectedPath = join(root, ".fusion", "chat-attachments", "session-1", "image.png");
const result = await readChatAttachmentContents(root, { kind: "session", sessionId: "session-1" }, [
attachment({ filename: "image.png", originalName: "image.png", mimeType: "image/png", size: PNG_BYTES.length }),
@@ -66,9 +68,17 @@ describe("readChatAttachmentContents", () => {
{ originalName: "image.png", mimeType: "image/png", text: null },
]);
expect(result.imageContents).toEqual([
{ type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" },
{
type: "image",
data: PNG_BYTES.toString("base64"),
mimeType: "image/png",
path: expectedPath,
originalName: "image.png",
},
]);
expect(formatChatAttachmentContents(result.attachmentContents)).toBe("");
expect(formatChatImageAttachmentHints(result.imageContents)).toContain(expectedPath);
expect(formatChatImageAttachmentHints(result.imageContents)).toContain("vision/file tools");
});
it("corrects session webp-labeled PNG image blocks to image/png", async () => {
@@ -82,7 +92,15 @@ describe("readChatAttachmentContents", () => {
], diagnostics);
expect(result.attachmentContents).toEqual([{ originalName: "mismatch.webp", mimeType: "image/webp", text: null }]);
expect(result.imageContents).toEqual([{ type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" }]);
expect(result.imageContents).toEqual([
{
type: "image",
data: PNG_BYTES.toString("base64"),
mimeType: "image/png",
path: join(root, ".fusion", "chat-attachments", "session-1", "mismatch.webp"),
originalName: "mismatch.webp",
},
]);
expect(diagnostics.warn).toHaveBeenCalledWith(expect.stringContaining("from image/webp to image/png"));
});
@@ -97,7 +115,15 @@ describe("readChatAttachmentContents", () => {
], diagnostics);
expect(result.attachmentContents).toEqual([{ originalName: "mismatch.png", mimeType: "image/png", text: null }]);
expect(result.imageContents).toEqual([{ type: "image", data: WEBP_BYTES.toString("base64"), mimeType: "image/webp" }]);
expect(result.imageContents).toEqual([
{
type: "image",
data: WEBP_BYTES.toString("base64"),
mimeType: "image/webp",
path: join(root, ".fusion", "chat-room-attachments", "room-1", "mismatch.png"),
originalName: "mismatch.png",
},
]);
expect(diagnostics.warn).toHaveBeenCalledWith(expect.stringContaining("room room-1 from image/png to image/webp"));
});
@@ -111,7 +137,15 @@ describe("readChatAttachmentContents", () => {
]);
expect(result.attachmentContents).toEqual([{ originalName: "unknown.webp", mimeType: "image/webp", text: null }]);
expect(result.imageContents).toEqual([{ type: "image", data: UNKNOWN_IMAGE_BYTES.toString("base64"), mimeType: "image/webp" }]);
expect(result.imageContents).toEqual([
{
type: "image",
data: UNKNOWN_IMAGE_BYTES.toString("base64"),
mimeType: "image/webp",
path: join(root, ".fusion", "chat-attachments", "session-1", "unknown.webp"),
originalName: "unknown.webp",
},
]);
});
it("returns mixed text and image contents together", async () => {
@@ -127,7 +161,13 @@ describe("readChatAttachmentContents", () => {
expect(formatChatAttachmentContents(result.attachmentContents)).toContain("```json\n{\"ok\":true}\n```");
expect(result.imageContents).toEqual([
{ type: "image", data: Buffer.from("webp").toString("base64"), mimeType: "image/webp" },
{
type: "image",
data: Buffer.from("webp").toString("base64"),
mimeType: "image/webp",
path: join(root, ".fusion", "chat-room-attachments", "room-1", "photo.webp"),
originalName: "photo.webp",
},
]);
});

View File

@@ -7,6 +7,16 @@ export interface ChatImageContent {
type: "image";
data: string;
mimeType: string;
/**
* FNXC:GrokAcp 2026-07-12-07:30:
* Absolute path to the on-disk attachment. Grok ACP advertises
* `promptCapabilities.image: false` and ignores ACP image ContentBlocks;
* agents must open this path with vision/file tools to see pixels. Pi still
* uses `data`/`mimeType` as ImageContent.
*/
path: string;
/** User-facing filename for prompt hints. */
originalName: string;
}
export interface ChatAttachmentContent {
@@ -81,6 +91,11 @@ function escapeFence(text: string): string {
*
* FNXC:ChatAttachments 2026-06-16-19:55:
* Text attachments are prompt-inlined with the triage-compatible 50KB ceiling while image attachments are forwarded as pi image content blocks through promptWithFallback options.
*
* FNXC:GrokAcp 2026-07-12-07:30:
* Image attachments also carry absolute `path` and prompt path-hints. Grok ACP
* sets promptCapabilities.image=false so ContentBlocks alone are invisible;
* path hints let the agent open pixels via filesystem vision tools.
*/
export async function readChatAttachmentContents(
rootDir: string,
@@ -118,6 +133,8 @@ export async function readChatAttachmentContents(
type: "image",
data: data.toString("base64"),
mimeType: imageMimeType,
path: filePath,
originalName: attachment.originalName,
});
attachmentContents.push({
originalName: attachment.originalName,
@@ -166,3 +183,25 @@ export function formatChatAttachmentContents(attachmentContents: ChatAttachmentC
].join("\n")),
].join("\n\n");
}
/**
* FNXC:GrokAcp 2026-07-12-07:30:
* Build explicit absolute-path hints for image attachments. Name/size-only
* summaries look like "text placeholders" to CLI agents that cannot ingest ACP
* image ContentBlocks (Grok promptCapabilities.image=false). Paths point at
* files already stored under .fusion/chat-attachments or chat-room-attachments.
*/
export function formatChatImageAttachmentHints(imageContents: ChatImageContent[]): string {
if (imageContents.length === 0) {
return "";
}
return [
"## Image attachments (filesystem paths)",
"The user attached image file(s). Open each absolute path with vision/file tools to inspect the actual pixels — do not invent contents from the filename alone:",
...imageContents.map(
(image) =>
`- ${image.originalName} (${image.mimeType}): ${image.path}`,
),
].join("\n");
}

View File

@@ -35,7 +35,11 @@ import { existsSync } from "node:fs";
import { join, resolve, relative } from "node:path";
import { SessionManager } from "@earendil-works/pi-coding-agent";
import { SessionEventBuffer } from "./sse-buffer.js";
import { formatChatAttachmentContents, readChatAttachmentContents } from "./chat-attachment-content.js";
import {
formatChatAttachmentContents,
formatChatImageAttachmentHints,
readChatAttachmentContents,
} from "./chat-attachment-content.js";
import { buildTaskPlannerChatContext, TASK_PLANNER_CHAT_CONTEXT_PROMPT_GUIDANCE } from "./task-planner-chat-context.js";
import { formatTaskPlannerChatMetrics } from "./task-planner-chat-metrics.js";
import { emitWorkflowSseEvent, type WorkflowSseEventType } from "./sse.js";
@@ -1707,6 +1711,7 @@ export class ChatManager {
diagnostics,
);
const attachmentContentBlock = formatChatAttachmentContents(attachmentContents);
const imagePathHints = formatChatImageAttachmentHints(imageContents);
const parsedSkillCommands = parseSkillCommands(input.content);
const roomPromptParts = [
`You are replying as ${input.responder.name} in room #${input.roomName}.`,
@@ -1722,6 +1727,9 @@ export class ChatManager {
if (attachmentContentBlock) {
roomPromptParts.push(attachmentContentBlock);
}
if (imagePathHints) {
roomPromptParts.push(imagePathHints);
}
const roomPrompt = roomPromptParts.join("\n\n");
const responderRuntimeModel = extractRuntimeModel(input.responder.runtimeConfig);
@@ -2170,12 +2178,20 @@ export class ChatManager {
diagnostics,
);
const attachmentContentBlock = formatChatAttachmentContents(attachmentContents);
/*
FNXC:GrokAcp 2026-07-12-07:30:
Name/size-only attachmentSummary is not enough for Grok ACP (image ContentBlocks
are unsupported). Include absolute filesystem paths so the agent can open pixels.
*/
const imagePathHints = formatChatImageAttachmentHints(imageContents);
// Send only the new user content. Prior turns are reloaded by the
// pi/Claude CLI session via SessionManager.open() below — stuffing the
// transcript back into the user message would balloon the on-disk
// session every turn (and previously did, see chat-store.ts:setCliSessionFile).
const promptContent = [attachmentSummary, attachmentContentBlock, resolvedContent].filter(Boolean).join("\n\n");
const promptContent = [attachmentSummary, imagePathHints, attachmentContentBlock, resolvedContent]
.filter(Boolean)
.join("\n\n");
// Per-chat session continuity: the pi SessionManager (and, transitively,
// the Claude CLI --resume session it owns) is keyed off the chat. On the

View File

@@ -14,6 +14,14 @@ describe("extractPromptImagesFromOptions", () => {
).toEqual([{ data: "AAAA", mimeType: "image/png" }]);
});
it("maps absolute path to file:// uri", () => {
expect(
extractPromptImagesFromOptions({
images: [{ type: "image", data: "AAAA", mimeType: "image/png", path: "/tmp/photo.png" }],
}),
).toEqual([{ data: "AAAA", mimeType: "image/png", uri: "file:///tmp/photo.png" }]);
});
it("keeps uri when present and drops malformed entries", () => {
expect(
extractPromptImagesFromOptions({

View File

@@ -48,7 +48,11 @@ export function extractPromptImagesFromOptions(options: unknown): PromptImage[]
const data = typeof rec.data === "string" ? rec.data : undefined;
const mimeType = typeof rec.mimeType === "string" ? rec.mimeType : undefined;
if (!data || !mimeType || data.length === 0 || mimeType.length === 0) continue;
const uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined;
// Prefer explicit uri; else map absolute filesystem `path` to file:// for agents.
let uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined;
if (!uri && typeof rec.path === "string" && rec.path.length > 0) {
uri = rec.path.startsWith("file:") ? rec.path : `file://${rec.path}`;
}
images.push({ data, mimeType, ...(uri ? { uri } : {}) });
}
return images.length > 0 ? images : undefined;

View File

@@ -49,7 +49,11 @@ export function extractPromptImagesFromOptions(options: unknown): PromptImage[]
const data = typeof rec.data === "string" ? rec.data : undefined;
const mimeType = typeof rec.mimeType === "string" ? rec.mimeType : undefined;
if (!data || !mimeType || data.length === 0 || mimeType.length === 0) continue;
const uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined;
// Prefer explicit uri; else map absolute filesystem `path` to file:// for agents.
let uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined;
if (!uri && typeof rec.path === "string" && rec.path.length > 0) {
uri = rec.path.startsWith("file:") ? rec.path : `file://${rec.path}`;
}
images.push({ data, mimeType, ...(uri ? { uri } : {}) });
}
return images.length > 0 ? images : undefined;