fix: give Grok ACP absolute paths for chat image attachments
Grok advertises promptCapabilities.image=false and ignores ACP image ContentBlocks (live probe: NO_IMAGE). Path-based vision works when the agent is given an absolute file path. Include path hints in chat prompts from .fusion/chat-attachments and carry path on ChatImageContent for file:// uris.
This commit is contained in:
@@ -6,6 +6,7 @@ import { tmpdir } from "node:os";
|
||||
import {
|
||||
CHAT_TEXT_INLINE_LIMIT,
|
||||
formatChatAttachmentContents,
|
||||
formatChatImageAttachmentHints,
|
||||
readChatAttachmentContents,
|
||||
} from "../chat-attachment-content.js";
|
||||
|
||||
@@ -57,6 +58,7 @@ describe("readChatAttachmentContents", () => {
|
||||
const root = await makeRoot();
|
||||
await mkdir(join(root, ".fusion", "chat-attachments", "session-1"), { recursive: true });
|
||||
await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "image.png"), PNG_BYTES);
|
||||
const expectedPath = join(root, ".fusion", "chat-attachments", "session-1", "image.png");
|
||||
|
||||
const result = await readChatAttachmentContents(root, { kind: "session", sessionId: "session-1" }, [
|
||||
attachment({ filename: "image.png", originalName: "image.png", mimeType: "image/png", size: PNG_BYTES.length }),
|
||||
@@ -66,9 +68,17 @@ describe("readChatAttachmentContents", () => {
|
||||
{ originalName: "image.png", mimeType: "image/png", text: null },
|
||||
]);
|
||||
expect(result.imageContents).toEqual([
|
||||
{ type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" },
|
||||
{
|
||||
type: "image",
|
||||
data: PNG_BYTES.toString("base64"),
|
||||
mimeType: "image/png",
|
||||
path: expectedPath,
|
||||
originalName: "image.png",
|
||||
},
|
||||
]);
|
||||
expect(formatChatAttachmentContents(result.attachmentContents)).toBe("");
|
||||
expect(formatChatImageAttachmentHints(result.imageContents)).toContain(expectedPath);
|
||||
expect(formatChatImageAttachmentHints(result.imageContents)).toContain("vision/file tools");
|
||||
});
|
||||
|
||||
it("corrects session webp-labeled PNG image blocks to image/png", async () => {
|
||||
@@ -82,7 +92,15 @@ describe("readChatAttachmentContents", () => {
|
||||
], diagnostics);
|
||||
|
||||
expect(result.attachmentContents).toEqual([{ originalName: "mismatch.webp", mimeType: "image/webp", text: null }]);
|
||||
expect(result.imageContents).toEqual([{ type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" }]);
|
||||
expect(result.imageContents).toEqual([
|
||||
{
|
||||
type: "image",
|
||||
data: PNG_BYTES.toString("base64"),
|
||||
mimeType: "image/png",
|
||||
path: join(root, ".fusion", "chat-attachments", "session-1", "mismatch.webp"),
|
||||
originalName: "mismatch.webp",
|
||||
},
|
||||
]);
|
||||
expect(diagnostics.warn).toHaveBeenCalledWith(expect.stringContaining("from image/webp to image/png"));
|
||||
});
|
||||
|
||||
@@ -97,7 +115,15 @@ describe("readChatAttachmentContents", () => {
|
||||
], diagnostics);
|
||||
|
||||
expect(result.attachmentContents).toEqual([{ originalName: "mismatch.png", mimeType: "image/png", text: null }]);
|
||||
expect(result.imageContents).toEqual([{ type: "image", data: WEBP_BYTES.toString("base64"), mimeType: "image/webp" }]);
|
||||
expect(result.imageContents).toEqual([
|
||||
{
|
||||
type: "image",
|
||||
data: WEBP_BYTES.toString("base64"),
|
||||
mimeType: "image/webp",
|
||||
path: join(root, ".fusion", "chat-room-attachments", "room-1", "mismatch.png"),
|
||||
originalName: "mismatch.png",
|
||||
},
|
||||
]);
|
||||
expect(diagnostics.warn).toHaveBeenCalledWith(expect.stringContaining("room room-1 from image/png to image/webp"));
|
||||
});
|
||||
|
||||
@@ -111,7 +137,15 @@ describe("readChatAttachmentContents", () => {
|
||||
]);
|
||||
|
||||
expect(result.attachmentContents).toEqual([{ originalName: "unknown.webp", mimeType: "image/webp", text: null }]);
|
||||
expect(result.imageContents).toEqual([{ type: "image", data: UNKNOWN_IMAGE_BYTES.toString("base64"), mimeType: "image/webp" }]);
|
||||
expect(result.imageContents).toEqual([
|
||||
{
|
||||
type: "image",
|
||||
data: UNKNOWN_IMAGE_BYTES.toString("base64"),
|
||||
mimeType: "image/webp",
|
||||
path: join(root, ".fusion", "chat-attachments", "session-1", "unknown.webp"),
|
||||
originalName: "unknown.webp",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns mixed text and image contents together", async () => {
|
||||
@@ -127,7 +161,13 @@ describe("readChatAttachmentContents", () => {
|
||||
|
||||
expect(formatChatAttachmentContents(result.attachmentContents)).toContain("```json\n{\"ok\":true}\n```");
|
||||
expect(result.imageContents).toEqual([
|
||||
{ type: "image", data: Buffer.from("webp").toString("base64"), mimeType: "image/webp" },
|
||||
{
|
||||
type: "image",
|
||||
data: Buffer.from("webp").toString("base64"),
|
||||
mimeType: "image/webp",
|
||||
path: join(root, ".fusion", "chat-room-attachments", "room-1", "photo.webp"),
|
||||
originalName: "photo.webp",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -7,6 +7,16 @@ export interface ChatImageContent {
|
||||
type: "image";
|
||||
data: string;
|
||||
mimeType: string;
|
||||
/**
|
||||
* FNXC:GrokAcp 2026-07-12-07:30:
|
||||
* Absolute path to the on-disk attachment. Grok ACP advertises
|
||||
* `promptCapabilities.image: false` and ignores ACP image ContentBlocks;
|
||||
* agents must open this path with vision/file tools to see pixels. Pi still
|
||||
* uses `data`/`mimeType` as ImageContent.
|
||||
*/
|
||||
path: string;
|
||||
/** User-facing filename for prompt hints. */
|
||||
originalName: string;
|
||||
}
|
||||
|
||||
export interface ChatAttachmentContent {
|
||||
@@ -81,6 +91,11 @@ function escapeFence(text: string): string {
|
||||
*
|
||||
* FNXC:ChatAttachments 2026-06-16-19:55:
|
||||
* Text attachments are prompt-inlined with the triage-compatible 50KB ceiling while image attachments are forwarded as pi image content blocks through promptWithFallback options.
|
||||
*
|
||||
* FNXC:GrokAcp 2026-07-12-07:30:
|
||||
* Image attachments also carry absolute `path` and prompt path-hints. Grok ACP
|
||||
* sets promptCapabilities.image=false so ContentBlocks alone are invisible;
|
||||
* path hints let the agent open pixels via filesystem vision tools.
|
||||
*/
|
||||
export async function readChatAttachmentContents(
|
||||
rootDir: string,
|
||||
@@ -118,6 +133,8 @@ export async function readChatAttachmentContents(
|
||||
type: "image",
|
||||
data: data.toString("base64"),
|
||||
mimeType: imageMimeType,
|
||||
path: filePath,
|
||||
originalName: attachment.originalName,
|
||||
});
|
||||
attachmentContents.push({
|
||||
originalName: attachment.originalName,
|
||||
@@ -166,3 +183,25 @@ export function formatChatAttachmentContents(attachmentContents: ChatAttachmentC
|
||||
].join("\n")),
|
||||
].join("\n\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* FNXC:GrokAcp 2026-07-12-07:30:
|
||||
* Build explicit absolute-path hints for image attachments. Name/size-only
|
||||
* summaries look like "text placeholders" to CLI agents that cannot ingest ACP
|
||||
* image ContentBlocks (Grok promptCapabilities.image=false). Paths point at
|
||||
* files already stored under .fusion/chat-attachments or chat-room-attachments.
|
||||
*/
|
||||
export function formatChatImageAttachmentHints(imageContents: ChatImageContent[]): string {
|
||||
if (imageContents.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return [
|
||||
"## Image attachments (filesystem paths)",
|
||||
"The user attached image file(s). Open each absolute path with vision/file tools to inspect the actual pixels — do not invent contents from the filename alone:",
|
||||
...imageContents.map(
|
||||
(image) =>
|
||||
`- ${image.originalName} (${image.mimeType}): ${image.path}`,
|
||||
),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -35,7 +35,11 @@ import { existsSync } from "node:fs";
|
||||
import { join, resolve, relative } from "node:path";
|
||||
import { SessionManager } from "@earendil-works/pi-coding-agent";
|
||||
import { SessionEventBuffer } from "./sse-buffer.js";
|
||||
import { formatChatAttachmentContents, readChatAttachmentContents } from "./chat-attachment-content.js";
|
||||
import {
|
||||
formatChatAttachmentContents,
|
||||
formatChatImageAttachmentHints,
|
||||
readChatAttachmentContents,
|
||||
} from "./chat-attachment-content.js";
|
||||
import { buildTaskPlannerChatContext, TASK_PLANNER_CHAT_CONTEXT_PROMPT_GUIDANCE } from "./task-planner-chat-context.js";
|
||||
import { formatTaskPlannerChatMetrics } from "./task-planner-chat-metrics.js";
|
||||
import { emitWorkflowSseEvent, type WorkflowSseEventType } from "./sse.js";
|
||||
@@ -1707,6 +1711,7 @@ export class ChatManager {
|
||||
diagnostics,
|
||||
);
|
||||
const attachmentContentBlock = formatChatAttachmentContents(attachmentContents);
|
||||
const imagePathHints = formatChatImageAttachmentHints(imageContents);
|
||||
const parsedSkillCommands = parseSkillCommands(input.content);
|
||||
const roomPromptParts = [
|
||||
`You are replying as ${input.responder.name} in room #${input.roomName}.`,
|
||||
@@ -1722,6 +1727,9 @@ export class ChatManager {
|
||||
if (attachmentContentBlock) {
|
||||
roomPromptParts.push(attachmentContentBlock);
|
||||
}
|
||||
if (imagePathHints) {
|
||||
roomPromptParts.push(imagePathHints);
|
||||
}
|
||||
const roomPrompt = roomPromptParts.join("\n\n");
|
||||
|
||||
const responderRuntimeModel = extractRuntimeModel(input.responder.runtimeConfig);
|
||||
@@ -2170,12 +2178,20 @@ export class ChatManager {
|
||||
diagnostics,
|
||||
);
|
||||
const attachmentContentBlock = formatChatAttachmentContents(attachmentContents);
|
||||
/*
|
||||
FNXC:GrokAcp 2026-07-12-07:30:
|
||||
Name/size-only attachmentSummary is not enough for Grok ACP (image ContentBlocks
|
||||
are unsupported). Include absolute filesystem paths so the agent can open pixels.
|
||||
*/
|
||||
const imagePathHints = formatChatImageAttachmentHints(imageContents);
|
||||
|
||||
// Send only the new user content. Prior turns are reloaded by the
|
||||
// pi/Claude CLI session via SessionManager.open() below — stuffing the
|
||||
// transcript back into the user message would balloon the on-disk
|
||||
// session every turn (and previously did, see chat-store.ts:setCliSessionFile).
|
||||
const promptContent = [attachmentSummary, attachmentContentBlock, resolvedContent].filter(Boolean).join("\n\n");
|
||||
const promptContent = [attachmentSummary, imagePathHints, attachmentContentBlock, resolvedContent]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
|
||||
// Per-chat session continuity: the pi SessionManager (and, transitively,
|
||||
// the Claude CLI --resume session it owns) is keyed off the chat. On the
|
||||
|
||||
@@ -14,6 +14,14 @@ describe("extractPromptImagesFromOptions", () => {
|
||||
).toEqual([{ data: "AAAA", mimeType: "image/png" }]);
|
||||
});
|
||||
|
||||
it("maps absolute path to file:// uri", () => {
|
||||
expect(
|
||||
extractPromptImagesFromOptions({
|
||||
images: [{ type: "image", data: "AAAA", mimeType: "image/png", path: "/tmp/photo.png" }],
|
||||
}),
|
||||
).toEqual([{ data: "AAAA", mimeType: "image/png", uri: "file:///tmp/photo.png" }]);
|
||||
});
|
||||
|
||||
it("keeps uri when present and drops malformed entries", () => {
|
||||
expect(
|
||||
extractPromptImagesFromOptions({
|
||||
|
||||
@@ -48,7 +48,11 @@ export function extractPromptImagesFromOptions(options: unknown): PromptImage[]
|
||||
const data = typeof rec.data === "string" ? rec.data : undefined;
|
||||
const mimeType = typeof rec.mimeType === "string" ? rec.mimeType : undefined;
|
||||
if (!data || !mimeType || data.length === 0 || mimeType.length === 0) continue;
|
||||
const uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined;
|
||||
// Prefer explicit uri; else map absolute filesystem `path` to file:// for agents.
|
||||
let uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined;
|
||||
if (!uri && typeof rec.path === "string" && rec.path.length > 0) {
|
||||
uri = rec.path.startsWith("file:") ? rec.path : `file://${rec.path}`;
|
||||
}
|
||||
images.push({ data, mimeType, ...(uri ? { uri } : {}) });
|
||||
}
|
||||
return images.length > 0 ? images : undefined;
|
||||
|
||||
@@ -49,7 +49,11 @@ export function extractPromptImagesFromOptions(options: unknown): PromptImage[]
|
||||
const data = typeof rec.data === "string" ? rec.data : undefined;
|
||||
const mimeType = typeof rec.mimeType === "string" ? rec.mimeType : undefined;
|
||||
if (!data || !mimeType || data.length === 0 || mimeType.length === 0) continue;
|
||||
const uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined;
|
||||
// Prefer explicit uri; else map absolute filesystem `path` to file:// for agents.
|
||||
let uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined;
|
||||
if (!uri && typeof rec.path === "string" && rec.path.length > 0) {
|
||||
uri = rec.path.startsWith("file:") ? rec.path : `file://${rec.path}`;
|
||||
}
|
||||
images.push({ data, mimeType, ...(uri ? { uri } : {}) });
|
||||
}
|
||||
return images.length > 0 ? images : undefined;
|
||||
|
||||
Reference in New Issue
Block a user