root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: forward chat image attachments into Grok/ACP session/prompt

Browse Source 
AcpRuntimeAdapter.promptWithFallback ignored options, so dashboard chat
images never became ACP ContentBlock image entries. Extract images from
prompt options and pass them through buildPromptBlocks for both acp-runtime
and the Grok vendored client.
This commit is contained in:
gsxdsm
2026-07-11 23:44:14 -07:00
parent 70cca2f96b
commit c2d3bf0e8a
7 changed files with 153 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
plugins/fusion-plugin-acp-runtime/src/__tests__/fixtures/echo-agent.mjs
View File

@@ -125,11 +125,17 @@ class EchoAgent {
});
return { stopReason: "end_turn" };
}
// FNXC:GrokAcp 2026-07-12-07:15: report image ContentBlock count so
// adapter tests can prove chat image options reach session/prompt.
const imageCount = Array.isArray(params.prompt)
? params.prompt.filter((block) => block && block.type === "image").length
: 0;
const replyText = imageCount > 0 ? `echo: images=${imageCount}` : "echo: hello";
await this.connection.sessionUpdate({
sessionId: params.sessionId,
update: {
sessionUpdate: "agent_message_chunk",
content: { type: "text", text: "echo: hello" },
content: { type: "text", text: replyText },
},
});
// Cancel-mid-prompt test: keep the turn open until cancel() arrives, then

35
plugins/fusion-plugin-acp-runtime/src/__tests__/prompt-builder.test.ts
View File

@@ -1,5 +1,38 @@
import { describe, it, expect } from "vitest";
import { buildPromptBlocks } from "../prompt-builder.js";
import { buildPromptBlocks, extractPromptImagesFromOptions } from "../prompt-builder.js";
describe("extractPromptImagesFromOptions", () => {
/*
FNXC:GrokAcp 2026-07-12-07:15:
Chat attachment options must map into PromptImage for ACP session/prompt.
*/
it("extracts chat-style image contents", () => {
expect(
extractPromptImagesFromOptions({
images: [{ type: "image", data: "AAAA", mimeType: "image/png" }],
}),
).toEqual([{ data: "AAAA", mimeType: "image/png" }]);
});
it("keeps uri when present and drops malformed entries", () => {
expect(
extractPromptImagesFromOptions({
images: [
{ data: "AAAA", mimeType: "image/png", uri: "file:///a.png" },
{ data: "", mimeType: "image/png" },
{ mimeType: "image/jpeg" },
null,
],
}),
).toEqual([{ data: "AAAA", mimeType: "image/png", uri: "file:///a.png" }]);
});
it("returns undefined for missing or empty images", () => {
expect(extractPromptImagesFromOptions(undefined)).toBeUndefined();
expect(extractPromptImagesFromOptions({})).toBeUndefined();
expect(extractPromptImagesFromOptions({ images: [] })).toBeUndefined();
});
});
describe("buildPromptBlocks", () => {
it("turns a plain string into a single text block", () => {

26
plugins/fusion-plugin-acp-runtime/src/__tests__/runtime-adapter.test.ts
View File

@@ -68,6 +68,32 @@ describe("AcpRuntimeAdapter (U3)", () => {
}
});
/*
FNXC:GrokAcp 2026-07-12-07:15:
Chat image attachments must arrive as ACP image ContentBlocks on session/prompt.
*/
it("promptWithFallback forwards chat image options into session/prompt", async () => {
const chunks: string[] = [];
const adapter = makeAdapter();
const { session } = await adapter.createSession(
makeOptions({
onText: (t) => {
chunks.push(t);
},
}),
);
try {
await expect(
adapter.promptWithFallback(session, "describe this", {
images: [{ type: "image", data: "AAAA", mimeType: "image/png" }],
}),
).resolves.toEqual({ stopReason: "end_turn" });
expect(chunks.join("")).toContain("images=1");
} finally {
await adapter.dispose(session);
}
});
it("dispose tears down the subprocess and is idempotent", async () => {
const adapter = makeAdapter();
const { session } = await adapter.createSession(makeOptions());

33
plugins/fusion-plugin-acp-runtime/src/prompt-builder.ts
View File

@@ -21,6 +21,39 @@ export interface BuildPromptOptions {
images?: PromptImage[];
}
/*
FNXC:GrokAcp 2026-07-12-07:15:
Dashboard chat forwards attachments as promptWithFallback options
`{ images: ChatImageContent[] }` where each item is
`{ type: "image", data: base64, mimeType }`. ACP session/prompt needs
ContentBlock image variants. Extract defensively so pi-style ImageContent
and PromptImage shapes both work; ignore malformed entries.
*/
/**
* Pull image attachments from Fusion `promptWithFallback` options.
* Accepts `{ images: Array<{ data, mimeType, uri? }> }` (chat / pi ImageContent).
*/
export function extractPromptImagesFromOptions(options: unknown): PromptImage[] | undefined {
if (!options || typeof options !== "object" || Array.isArray(options)) {
return undefined;
}
const raw = (options as { images?: unknown }).images;
if (!Array.isArray(raw) || raw.length === 0) {
return undefined;
}
const images: PromptImage[] = [];
for (const item of raw) {
if (!item || typeof item !== "object" || Array.isArray(item)) continue;
const rec = item as Record<string, unknown>;
const data = typeof rec.data === "string" ? rec.data : undefined;
const mimeType = typeof rec.mimeType === "string" ? rec.mimeType : undefined;
if (!data || !mimeType || data.length === 0 || mimeType.length === 0) continue;
const uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined;
images.push({ data, mimeType, ...(uri ? { uri } : {}) });
}
return images.length > 0 ? images : undefined;
}
/**
* Build the ACP prompt content blocks for a turn.
*

13
plugins/fusion-plugin-acp-runtime/src/runtime-adapter.ts
View File

@@ -17,7 +17,7 @@ import {
createBridgingClientHandler,
} from "./provider.js";
import { buildSpawnEnv } from "./process-manager.js";
import { buildPromptBlocks } from "./prompt-builder.js";
import { buildPromptBlocks, extractPromptImagesFromOptions } from "./prompt-builder.js";
import type {
AgentRuntime,
AgentRuntimeOptions,
@@ -141,7 +141,7 @@ export class AcpRuntimeAdapter implements AgentRuntime {
async promptWithFallback(
session: AgentSession,
prompt: string,
_options?: unknown,
options?: unknown,
): Promise<{ stopReason?: string }> {
const acp = session as AcpSession;
if (!acp.connection) {
@@ -152,7 +152,14 @@ export class AcpRuntimeAdapter implements AgentRuntime {
// each turn (FIX 1). Without this, a turn that hit the per-turn output cap
// would silently suppress all later turns.
acp.resetTurn?.();
const blocks = buildPromptBlocks(prompt);
/*
FNXC:GrokAcp 2026-07-12-07:15:
Chat/triage pass `{ images: [{ type:"image", data, mimeType }] }` through
promptWithFallback. Previously options were ignored (`_options`) so Grok ACP
and generic ACP sessions never received image ContentBlocks on session/prompt.
*/
const images = extractPromptImagesFromOptions(options);
const blocks = buildPromptBlocks(prompt, images ? { images } : undefined);
// Resolve when the SDK prompt promise resolves — it already drains all
// session/update notifications for the turn before reporting the stopReason.
// The bridging client handler installed at createSession (U4) has already

33
plugins/fusion-plugin-grok-runtime/src/acp/prompt-builder.ts
View File

@@ -22,6 +22,39 @@ export interface BuildPromptOptions {
images?: PromptImage[];
}
/*
FNXC:GrokAcp 2026-07-12-07:15:
Dashboard chat forwards attachments as promptWithFallback options
`{ images: ChatImageContent[] }` where each item is
`{ type: "image", data: base64, mimeType }`. ACP session/prompt needs
ContentBlock image variants. Extract defensively so pi-style ImageContent
and PromptImage shapes both work; ignore malformed entries.
*/
/**
* Pull image attachments from Fusion `promptWithFallback` options.
* Accepts `{ images: Array<{ data, mimeType, uri? }> }` (chat / pi ImageContent).
*/
export function extractPromptImagesFromOptions(options: unknown): PromptImage[] | undefined {
if (!options || typeof options !== "object" || Array.isArray(options)) {
return undefined;
}
const raw = (options as { images?: unknown }).images;
if (!Array.isArray(raw) || raw.length === 0) {
return undefined;
}
const images: PromptImage[] = [];
for (const item of raw) {
if (!item || typeof item !== "object" || Array.isArray(item)) continue;
const rec = item as Record<string, unknown>;
const data = typeof rec.data === "string" ? rec.data : undefined;
const mimeType = typeof rec.mimeType === "string" ? rec.mimeType : undefined;
if (!data || !mimeType || data.length === 0 || mimeType.length === 0) continue;
const uri = typeof rec.uri === "string" && rec.uri.length > 0 ? rec.uri : undefined;
images.push({ data, mimeType, ...(uri ? { uri } : {}) });
}
return images.length > 0 ? images : undefined;
}
/**
* Build the ACP prompt content blocks for a turn.
*

13
plugins/fusion-plugin-grok-runtime/src/acp/runtime-adapter.ts
View File

@@ -18,7 +18,7 @@ import {
createBridgingClientHandler,
} from "./provider.js";
import { buildSpawnEnv } from "./process-manager.js";
import { buildPromptBlocks } from "./prompt-builder.js";
import { buildPromptBlocks, extractPromptImagesFromOptions } from "./prompt-builder.js";
import type {
AgentRuntime,
AgentRuntimeOptions,
@@ -142,7 +142,7 @@ export class AcpRuntimeAdapter implements AgentRuntime {
async promptWithFallback(
session: AgentSession,
prompt: string,
_options?: unknown,
options?: unknown,
): Promise<{ stopReason?: string }> {
const acp = session as AcpSession;
if (!acp.connection) {
@@ -153,7 +153,14 @@ export class AcpRuntimeAdapter implements AgentRuntime {
// each turn (FIX 1). Without this, a turn that hit the per-turn output cap
// would silently suppress all later turns.
acp.resetTurn?.();
const blocks = buildPromptBlocks(prompt);
/*
FNXC:GrokAcp 2026-07-12-07:15:
Chat/triage pass `{ images: [{ type:"image", data, mimeType }] }` through
promptWithFallback. Previously options were ignored (`_options`) so Grok ACP
and generic ACP sessions never received image ContentBlocks on session/prompt.
*/
const images = extractPromptImagesFromOptions(options);
const blocks = buildPromptBlocks(prompt, images ? { images } : undefined);
// Resolve when the SDK prompt promise resolves — it already drains all
// session/update notifications for the turn before reporting the stopReason.
// The bridging client handler installed at createSession (U4) has already