root/fusion
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

FN-7211: fix image attachment MIME sniffing

Browse Source 
Correct image attachment media types from bytes before sending image blocks to models.

- Add core image MIME sniffing for PNG, JPEG, GIF, and WEBP signatures.
- Use detected image bytes in task triage and dashboard chat attachment image blocks while preserving stored attachment metadata.
- Cover mismatched and unknown image bytes with core, triage, and chat attachment tests.
- Add a patch changeset for the published CLI package.

Files changed:
 .changeset/fn-7211-webp-image-mime-sniff.md        |  7 +++
 packages/core/src/__tests__/image-mime.test.ts     | 29 ++++++++++
 packages/core/src/image-mime.ts                    | 50 +++++++++++++++++
 packages/core/src/index.ts                         |  2 +
 .../src/__tests__/chat-attachment-content.test.ts  | 54 +++++++++++++++++--
 packages/dashboard/src/chat-attachment-content.ts  |  9 +++-
 packages/engine/src/__tests__/triage.test.ts       | 63 +++++++++++++++++-----
 packages/engine/src/triage.ts                      |  8 ++-
 8 files changed, 201 insertions(+), 21 deletions(-)

Fusion-Task-Id: FN-7211

Fusion-Task-Lineage: 22bfb812-15c4-4ff7-a281-87b25427561e

Co-authored-by: Fusion (runfusion.ai) <noreply@runfusion.ai>
This commit is contained in:
gsxdsm
2026-06-28 16:51:49 -07:00
parent 969d03b474
commit 8eed09c126
8 changed files with 201 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.changeset/fn-7211-webp-image-mime-sniff.md Normal file
View File

@@ -0,0 +1,7 @@
---
"@runfusion/fusion": patch
---
summary: Fix planning/chat failures when image attachment bytes do not match the file extension.
category: fix
dev: Adds detectImageMimeFromBytes in core and applies it in triage and dashboard chat attachment read paths.

29
packages/core/src/__tests__/image-mime.test.ts Normal file
View File

@@ -0,0 +1,29 @@
import { describe, expect, it } from "vitest";
import { detectImageMimeFromBytes } from "../image-mime.js";
describe("detectImageMimeFromBytes", () => {
it("detects PNG magic bytes", () => {
expect(detectImageMimeFromBytes(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]))).toBe("image/png");
});
it("detects JPEG magic bytes", () => {
expect(detectImageMimeFromBytes(Buffer.from([0xff, 0xd8, 0xff, 0xe0]))).toBe("image/jpeg");
});
it("detects GIF87a and GIF89a magic bytes", () => {
expect(detectImageMimeFromBytes(Buffer.from("GIF87a"))).toBe("image/gif");
expect(detectImageMimeFromBytes(Buffer.from("GIF89a"))).toBe("image/gif");
});
it("detects WEBP only when RIFF and WEBP segments are present", () => {
expect(detectImageMimeFromBytes(Buffer.from([0x52, 0x49, 0x46, 0x46, 0x01, 0x02, 0x03, 0x04, 0x57, 0x45, 0x42, 0x50]))).toBe("image/webp");
expect(detectImageMimeFromBytes(Buffer.from([0x52, 0x49, 0x46, 0x46, 0x01, 0x02, 0x03, 0x04, 0x4e, 0x4f, 0x50, 0x45]))).toBeNull();
expect(detectImageMimeFromBytes(Buffer.from([0x4e, 0x4f, 0x50, 0x45, 0x01, 0x02, 0x03, 0x04, 0x57, 0x45, 0x42, 0x50]))).toBeNull();
});
it("returns null for short or unknown bytes", () => {
expect(detectImageMimeFromBytes(Buffer.from([0x89, 0x50, 0x4e, 0x47]))).toBeNull();
expect(detectImageMimeFromBytes(Buffer.from([0x01, 0x02, 0x03, 0x04]))).toBeNull();
expect(detectImageMimeFromBytes(new Uint8Array())).toBeNull();
});
});

50
packages/core/src/image-mime.ts Normal file
View File

@@ -0,0 +1,50 @@
export type DetectedImageMime = "image/png" | "image/jpeg" | "image/gif" | "image/webp";
/*
FNXC:ImageAttachments 2026-06-28-00:00:
FN-7211: AI image-block media_type must match the real image bytes. Attachment storage keeps the extension-derived mimeType for display, but Anthropic rejects mismatched pairings such as stored image/webp over PNG bytes, so model-bound image blocks sniff bytes at build time.
*/
export function detectImageMimeFromBytes(bytes: Buffer | Uint8Array): DetectedImageMime | null {
if (bytes.length >= 8
&& bytes[0] === 0x89
&& bytes[1] === 0x50
&& bytes[2] === 0x4e
&& bytes[3] === 0x47
&& bytes[4] === 0x0d
&& bytes[5] === 0x0a
&& bytes[6] === 0x1a
&& bytes[7] === 0x0a) {
return "image/png";
}
if (bytes.length >= 3
&& bytes[0] === 0xff
&& bytes[1] === 0xd8
&& bytes[2] === 0xff) {
return "image/jpeg";
}
if (bytes.length >= 6
&& bytes[0] === 0x47
&& bytes[1] === 0x49
&& bytes[2] === 0x46
&& bytes[3] === 0x38
&& (bytes[4] === 0x37 || bytes[4] === 0x39)
&& bytes[5] === 0x61) {
return "image/gif";
}
if (bytes.length >= 12
&& bytes[0] === 0x52
&& bytes[1] === 0x49
&& bytes[2] === 0x46
&& bytes[3] === 0x46
&& bytes[8] === 0x57
&& bytes[9] === 0x45
&& bytes[10] === 0x42
&& bytes[11] === 0x50) {
return "image/webp";
}
return null;
}

2
packages/core/src/index.ts
View File

@@ -16,6 +16,8 @@ export type {
EntryPointBranchAssignment,
} from "./branch-assignment.js";
export { customProviderRegistryKey } from "./custom-provider-key.js";
export { detectImageMimeFromBytes } from "./image-mime.js";
export type { DetectedImageMime } from "./image-mime.js";
export { redactSecrets } from "./redact-secrets.js";
export { resolvePlanApprovalRequired } from "./plan-approval.js";
export type { PlanApprovalMode } from "./plan-approval.js";

54
packages/dashboard/src/__tests__/chat-attachment-content.test.ts
View File

@@ -10,6 +10,9 @@ import {
} from "../chat-attachment-content.js";
const roots: string[] = [];
const PNG_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]);
const WEBP_BYTES = Buffer.from([0x52, 0x49, 0x46, 0x46, 0x01, 0x02, 0x03, 0x04, 0x57, 0x45, 0x42, 0x50]);
const UNKNOWN_IMAGE_BYTES = Buffer.from([0x01, 0x02, 0x03, 0x04]);
function attachment(overrides: Partial<ChatAttachment>): ChatAttachment {
return {
@@ -50,24 +53,67 @@ describe("readChatAttachmentContents", () => {
expect(formatChatAttachmentContents(result.attachmentContents)).toContain("hello from attachment");
});
it("converts image attachments to base64 content blocks", async () => {
it("converts matching image attachments to base64 content blocks", async () => {
const root = await makeRoot();
await mkdir(join(root, ".fusion", "chat-attachments", "session-1"), { recursive: true });
await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "image.png"), Buffer.from([1, 2, 3, 4]));
await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "image.png"), PNG_BYTES);
const result = await readChatAttachmentContents(root, { kind: "session", sessionId: "session-1" }, [
attachment({ filename: "image.png", originalName: "image.png", mimeType: "image/png", size: 4 }),
attachment({ filename: "image.png", originalName: "image.png", mimeType: "image/png", size: PNG_BYTES.length }),
]);
expect(result.attachmentContents).toEqual([
{ originalName: "image.png", mimeType: "image/png", text: null },
]);
expect(result.imageContents).toEqual([
{ type: "image", data: Buffer.from([1, 2, 3, 4]).toString("base64"), mimeType: "image/png" },
{ type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" },
]);
expect(formatChatAttachmentContents(result.attachmentContents)).toBe("");
});
it("corrects session webp-labeled PNG image blocks to image/png", async () => {
const root = await makeRoot();
const diagnostics = { warn: vi.fn() };
await mkdir(join(root, ".fusion", "chat-attachments", "session-1"), { recursive: true });
await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "mismatch.webp"), PNG_BYTES);
const result = await readChatAttachmentContents(root, { kind: "session", sessionId: "session-1" }, [
attachment({ filename: "mismatch.webp", originalName: "mismatch.webp", mimeType: "image/webp", size: PNG_BYTES.length }),
], diagnostics);
expect(result.attachmentContents).toEqual([{ originalName: "mismatch.webp", mimeType: "image/webp", text: null }]);
expect(result.imageContents).toEqual([{ type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" }]);
expect(diagnostics.warn).toHaveBeenCalledWith(expect.stringContaining("from image/webp to image/png"));
});
it("corrects room png-labeled WEBP image blocks to image/webp", async () => {
const root = await makeRoot();
const diagnostics = { warn: vi.fn() };
await mkdir(join(root, ".fusion", "chat-room-attachments", "room-1"), { recursive: true });
await writeFile(join(root, ".fusion", "chat-room-attachments", "room-1", "mismatch.png"), WEBP_BYTES);
const result = await readChatAttachmentContents(root, { kind: "room", roomId: "room-1" }, [
attachment({ filename: "mismatch.png", originalName: "mismatch.png", mimeType: "image/png", size: WEBP_BYTES.length }),
], diagnostics);
expect(result.attachmentContents).toEqual([{ originalName: "mismatch.png", mimeType: "image/png", text: null }]);
expect(result.imageContents).toEqual([{ type: "image", data: WEBP_BYTES.toString("base64"), mimeType: "image/webp" }]);
expect(diagnostics.warn).toHaveBeenCalledWith(expect.stringContaining("room room-1 from image/png to image/webp"));
});
it("falls back to stored image mime type for unrecognized session bytes", async () => {
const root = await makeRoot();
await mkdir(join(root, ".fusion", "chat-attachments", "session-1"), { recursive: true });
await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "unknown.webp"), UNKNOWN_IMAGE_BYTES);
const result = await readChatAttachmentContents(root, { kind: "session", sessionId: "session-1" }, [
attachment({ filename: "unknown.webp", originalName: "unknown.webp", mimeType: "image/webp", size: UNKNOWN_IMAGE_BYTES.length }),
]);
expect(result.attachmentContents).toEqual([{ originalName: "unknown.webp", mimeType: "image/webp", text: null }]);
expect(result.imageContents).toEqual([{ type: "image", data: UNKNOWN_IMAGE_BYTES.toString("base64"), mimeType: "image/webp" }]);
});
it("returns mixed text and image contents together", async () => {
const root = await makeRoot();
await mkdir(join(root, ".fusion", "chat-room-attachments", "room-1"), { recursive: true });

9
packages/dashboard/src/chat-attachment-content.ts
View File

@@ -1,4 +1,4 @@
import type { ChatAttachment } from "@fusion/core";
import { detectImageMimeFromBytes, type ChatAttachment } from "@fusion/core";
import { readFile } from "node:fs/promises";
import { basename, resolve } from "node:path";
import { CHAT_ALLOWED_MIME_TYPES } from "./routes/chat-attachment-config.js";
@@ -109,10 +109,15 @@ export async function readChatAttachmentContents(
try {
if (IMAGE_MIME_TYPES.has(attachment.mimeType)) {
const data = await readFile(filePath);
const detectedMimeType = detectImageMimeFromBytes(data);
const imageMimeType = detectedMimeType ?? attachment.mimeType;
if (detectedMimeType && detectedMimeType !== attachment.mimeType) {
diagnostics?.warn(`Corrected chat image attachment media type for '${attachment.filename}' in ${getScopeLabel(scope)} from ${attachment.mimeType} to ${detectedMimeType}`);
}
imageContents.push({
type: "image",
data: data.toString("base64"),
mimeType: attachment.mimeType,
mimeType: imageMimeType,
});
attachmentContents.push({
originalName: attachment.originalName,

63
packages/engine/src/__tests__/triage.test.ts
View File

@@ -1206,6 +1206,20 @@ describe("fast-mode triage", () => {
});
});
const PNG_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]);
const WEBP_BYTES = Buffer.from([0x52, 0x49, 0x46, 0x46, 0x01, 0x02, 0x03, 0x04, 0x57, 0x45, 0x42, 0x50]);
function attachmentFixture(overrides: Partial<TaskDetail["attachments"][number]>): TaskDetail["attachments"][number] {
return {
filename: "1234567890-image.png",
originalName: "image.png",
mimeType: "image/png",
size: 100,
createdAt: "2026-01-01T00:00:00.000Z",
...overrides,
};
}
describe("readAttachmentContents", () => {
let testDir = "";
const taskId = "FN-TEST";
@@ -1284,31 +1298,52 @@ describe("readAttachmentContents", () => {
});
it("reads image as base64 content", async () => {
const attachments = [
{
filename: "1234567890-image.png",
originalName: "image.png",
mimeType: "image/png" as const,
size: 100,
createdAt: "2026-01-01T00:00:00.000Z",
},
];
const attachments = [attachmentFixture({ filename: "1234567890-image.png", originalName: "image.png", mimeType: "image/png" })];
// Write fake PNG data (just some bytes)
const imageData = Buffer.from([0x89, 0x50, 0x4e, 0x47]); // PNG magic bytes
await writeFile(
join(testDir, ".fusion", "tasks", taskId, "attachments", "1234567890-image.png"),
imageData,
PNG_BYTES,
);
const result = await readAttachmentContents(testDir, taskId, attachments);
expect(result.attachmentContents).toHaveLength(1);
expect(result.attachmentContents[0].text).toBeNull();
expect(result.attachmentContents[0]).toMatchObject({ originalName: "image.png", mimeType: "image/png", text: null });
expect(result.imageContents).toHaveLength(1);
expect(result.imageContents[0].type).toBe("image");
expect(result.imageContents[0].mimeType).toBe("image/png");
expect(result.imageContents[0].data).toBe(imageData.toString("base64"));
expect(result.imageContents[0].data).toBe(PNG_BYTES.toString("base64"));
});
it("corrects a webp-labeled PNG image block to image/png", async () => {
const attachments = [attachmentFixture({ filename: "mismatch.webp", originalName: "mismatch.webp", mimeType: "image/webp" })];
await writeFile(join(testDir, ".fusion", "tasks", taskId, "attachments", "mismatch.webp"), PNG_BYTES);
const result = await readAttachmentContents(testDir, taskId, attachments);
expect(result.attachmentContents).toEqual([{ originalName: "mismatch.webp", mimeType: "image/webp", text: null }]);
expect(result.imageContents).toEqual([{ type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" }]);
});
it("corrects a png-labeled WEBP image block to image/webp", async () => {
const attachments = [attachmentFixture({ filename: "mismatch.png", originalName: "mismatch.png", mimeType: "image/png" })];
await writeFile(join(testDir, ".fusion", "tasks", taskId, "attachments", "mismatch.png"), WEBP_BYTES);
const result = await readAttachmentContents(testDir, taskId, attachments);
expect(result.attachmentContents).toEqual([{ originalName: "mismatch.png", mimeType: "image/png", text: null }]);
expect(result.imageContents).toEqual([{ type: "image", data: WEBP_BYTES.toString("base64"), mimeType: "image/webp" }]);
});
it("falls back to stored image mime type for unrecognized bytes", async () => {
const unknownBytes = Buffer.from([0x01, 0x02, 0x03, 0x04]);
const attachments = [attachmentFixture({ filename: "unknown.webp", originalName: "unknown.webp", mimeType: "image/webp" })];
await writeFile(join(testDir, ".fusion", "tasks", taskId, "attachments", "unknown.webp"), unknownBytes);
const result = await readAttachmentContents(testDir, taskId, attachments);
expect(result.attachmentContents).toEqual([{ originalName: "unknown.webp", mimeType: "image/webp", text: null }]);
expect(result.imageContents).toEqual([{ type: "image", data: unknownBytes.toString("base64"), mimeType: "image/webp" }]);
});
it("skips unreadable attachments", async () => {

8
packages/engine/src/triage.ts
View File

@@ -27,6 +27,7 @@ import {
extractIntentSignature,
findNearDuplicates,
isNearDuplicateCanonicalInactive,
detectImageMimeFromBytes,
applyFrontendUxCriteria,
extractEffectiveWriteScopeFromPrompt,
MAX_TASK_LIST_TEXT_CHARS,
@@ -2754,10 +2755,15 @@ export async function readAttachmentContents(
try {
if (IMAGE_MIME_TYPES.has(att.mimeType)) {
const data = await readFile(filePath);
const detectedMimeType = detectImageMimeFromBytes(data);
const imageMimeType = detectedMimeType ?? att.mimeType;
if (detectedMimeType && detectedMimeType !== att.mimeType) {
planLog.warn(`${taskId}: corrected image attachment media type for '${att.filename}' (${att.originalName}) from ${att.mimeType} to ${detectedMimeType}`);
}
imageContents.push({
type: "image",
data: data.toString("base64"),
mimeType: att.mimeType,
mimeType: imageMimeType,
});
attachmentContents.push({
originalName: att.originalName,