diff --git a/.changeset/fn-7211-webp-image-mime-sniff.md b/.changeset/fn-7211-webp-image-mime-sniff.md new file mode 100644 index 0000000000..1769381b9f --- /dev/null +++ b/.changeset/fn-7211-webp-image-mime-sniff.md @@ -0,0 +1,7 @@ +--- +"@runfusion/fusion": patch +--- + +summary: Fix planning/chat failures when image attachment bytes do not match the file extension. +category: fix +dev: Adds detectImageMimeFromBytes in core and applies it in triage and dashboard chat attachment read paths. diff --git a/packages/core/src/__tests__/image-mime.test.ts b/packages/core/src/__tests__/image-mime.test.ts new file mode 100644 index 0000000000..5fbac3a9c3 --- /dev/null +++ b/packages/core/src/__tests__/image-mime.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { detectImageMimeFromBytes } from "../image-mime.js"; + +describe("detectImageMimeFromBytes", () => { + it("detects PNG magic bytes", () => { + expect(detectImageMimeFromBytes(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]))).toBe("image/png"); + }); + + it("detects JPEG magic bytes", () => { + expect(detectImageMimeFromBytes(Buffer.from([0xff, 0xd8, 0xff, 0xe0]))).toBe("image/jpeg"); + }); + + it("detects GIF87a and GIF89a magic bytes", () => { + expect(detectImageMimeFromBytes(Buffer.from("GIF87a"))).toBe("image/gif"); + expect(detectImageMimeFromBytes(Buffer.from("GIF89a"))).toBe("image/gif"); + }); + + it("detects WEBP only when RIFF and WEBP segments are present", () => { + expect(detectImageMimeFromBytes(Buffer.from([0x52, 0x49, 0x46, 0x46, 0x01, 0x02, 0x03, 0x04, 0x57, 0x45, 0x42, 0x50]))).toBe("image/webp"); + expect(detectImageMimeFromBytes(Buffer.from([0x52, 0x49, 0x46, 0x46, 0x01, 0x02, 0x03, 0x04, 0x4e, 0x4f, 0x50, 0x45]))).toBeNull(); + expect(detectImageMimeFromBytes(Buffer.from([0x4e, 0x4f, 0x50, 0x45, 0x01, 0x02, 0x03, 0x04, 0x57, 0x45, 0x42, 0x50]))).toBeNull(); + }); + + it("returns null for short or unknown bytes", () => { + expect(detectImageMimeFromBytes(Buffer.from([0x89, 0x50, 0x4e, 0x47]))).toBeNull(); + expect(detectImageMimeFromBytes(Buffer.from([0x01, 0x02, 0x03, 0x04]))).toBeNull(); + expect(detectImageMimeFromBytes(new Uint8Array())).toBeNull(); + }); +}); diff --git a/packages/core/src/image-mime.ts b/packages/core/src/image-mime.ts new file mode 100644 index 0000000000..5566f65b94 --- /dev/null +++ b/packages/core/src/image-mime.ts @@ -0,0 +1,50 @@ +export type DetectedImageMime = "image/png" | "image/jpeg" | "image/gif" | "image/webp"; + +/* +FNXC:ImageAttachments 2026-06-28-00:00: +FN-7211: AI image-block media_type must match the real image bytes. Attachment storage keeps the extension-derived mimeType for display, but Anthropic rejects mismatched pairings such as stored image/webp over PNG bytes, so model-bound image blocks sniff bytes at build time. +*/ +export function detectImageMimeFromBytes(bytes: Buffer | Uint8Array): DetectedImageMime | null { + if (bytes.length >= 8 + && bytes[0] === 0x89 + && bytes[1] === 0x50 + && bytes[2] === 0x4e + && bytes[3] === 0x47 + && bytes[4] === 0x0d + && bytes[5] === 0x0a + && bytes[6] === 0x1a + && bytes[7] === 0x0a) { + return "image/png"; + } + + if (bytes.length >= 3 + && bytes[0] === 0xff + && bytes[1] === 0xd8 + && bytes[2] === 0xff) { + return "image/jpeg"; + } + + if (bytes.length >= 6 + && bytes[0] === 0x47 + && bytes[1] === 0x49 + && bytes[2] === 0x46 + && bytes[3] === 0x38 + && (bytes[4] === 0x37 || bytes[4] === 0x39) + && bytes[5] === 0x61) { + return "image/gif"; + } + + if (bytes.length >= 12 + && bytes[0] === 0x52 + && bytes[1] === 0x49 + && bytes[2] === 0x46 + && bytes[3] === 0x46 + && bytes[8] === 0x57 + && bytes[9] === 0x45 + && bytes[10] === 0x42 + && bytes[11] === 0x50) { + return "image/webp"; + } + + return null; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d008f525b4..6f730111ff 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -16,6 +16,8 @@ export type { EntryPointBranchAssignment, } from "./branch-assignment.js"; export { customProviderRegistryKey } from "./custom-provider-key.js"; +export { detectImageMimeFromBytes } from "./image-mime.js"; +export type { DetectedImageMime } from "./image-mime.js"; export { redactSecrets } from "./redact-secrets.js"; export { resolvePlanApprovalRequired } from "./plan-approval.js"; export type { PlanApprovalMode } from "./plan-approval.js"; diff --git a/packages/dashboard/src/__tests__/chat-attachment-content.test.ts b/packages/dashboard/src/__tests__/chat-attachment-content.test.ts index 6b889117c4..783e45a623 100644 --- a/packages/dashboard/src/__tests__/chat-attachment-content.test.ts +++ b/packages/dashboard/src/__tests__/chat-attachment-content.test.ts @@ -10,6 +10,9 @@ import { } from "../chat-attachment-content.js"; const roots: string[] = []; +const PNG_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]); +const WEBP_BYTES = Buffer.from([0x52, 0x49, 0x46, 0x46, 0x01, 0x02, 0x03, 0x04, 0x57, 0x45, 0x42, 0x50]); +const UNKNOWN_IMAGE_BYTES = Buffer.from([0x01, 0x02, 0x03, 0x04]); function attachment(overrides: Partial): ChatAttachment { return { @@ -50,24 +53,67 @@ describe("readChatAttachmentContents", () => { expect(formatChatAttachmentContents(result.attachmentContents)).toContain("hello from attachment"); }); - it("converts image attachments to base64 content blocks", async () => { + it("converts matching image attachments to base64 content blocks", async () => { const root = await makeRoot(); await mkdir(join(root, ".fusion", "chat-attachments", "session-1"), { recursive: true }); - await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "image.png"), Buffer.from([1, 2, 3, 4])); + await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "image.png"), PNG_BYTES); const result = await readChatAttachmentContents(root, { kind: "session", sessionId: "session-1" }, [ - attachment({ filename: "image.png", originalName: "image.png", mimeType: "image/png", size: 4 }), + attachment({ filename: "image.png", originalName: "image.png", mimeType: "image/png", size: PNG_BYTES.length }), ]); expect(result.attachmentContents).toEqual([ { originalName: "image.png", mimeType: "image/png", text: null }, ]); expect(result.imageContents).toEqual([ - { type: "image", data: Buffer.from([1, 2, 3, 4]).toString("base64"), mimeType: "image/png" }, + { type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" }, ]); expect(formatChatAttachmentContents(result.attachmentContents)).toBe(""); }); + it("corrects session webp-labeled PNG image blocks to image/png", async () => { + const root = await makeRoot(); + const diagnostics = { warn: vi.fn() }; + await mkdir(join(root, ".fusion", "chat-attachments", "session-1"), { recursive: true }); + await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "mismatch.webp"), PNG_BYTES); + + const result = await readChatAttachmentContents(root, { kind: "session", sessionId: "session-1" }, [ + attachment({ filename: "mismatch.webp", originalName: "mismatch.webp", mimeType: "image/webp", size: PNG_BYTES.length }), + ], diagnostics); + + expect(result.attachmentContents).toEqual([{ originalName: "mismatch.webp", mimeType: "image/webp", text: null }]); + expect(result.imageContents).toEqual([{ type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" }]); + expect(diagnostics.warn).toHaveBeenCalledWith(expect.stringContaining("from image/webp to image/png")); + }); + + it("corrects room png-labeled WEBP image blocks to image/webp", async () => { + const root = await makeRoot(); + const diagnostics = { warn: vi.fn() }; + await mkdir(join(root, ".fusion", "chat-room-attachments", "room-1"), { recursive: true }); + await writeFile(join(root, ".fusion", "chat-room-attachments", "room-1", "mismatch.png"), WEBP_BYTES); + + const result = await readChatAttachmentContents(root, { kind: "room", roomId: "room-1" }, [ + attachment({ filename: "mismatch.png", originalName: "mismatch.png", mimeType: "image/png", size: WEBP_BYTES.length }), + ], diagnostics); + + expect(result.attachmentContents).toEqual([{ originalName: "mismatch.png", mimeType: "image/png", text: null }]); + expect(result.imageContents).toEqual([{ type: "image", data: WEBP_BYTES.toString("base64"), mimeType: "image/webp" }]); + expect(diagnostics.warn).toHaveBeenCalledWith(expect.stringContaining("room room-1 from image/png to image/webp")); + }); + + it("falls back to stored image mime type for unrecognized session bytes", async () => { + const root = await makeRoot(); + await mkdir(join(root, ".fusion", "chat-attachments", "session-1"), { recursive: true }); + await writeFile(join(root, ".fusion", "chat-attachments", "session-1", "unknown.webp"), UNKNOWN_IMAGE_BYTES); + + const result = await readChatAttachmentContents(root, { kind: "session", sessionId: "session-1" }, [ + attachment({ filename: "unknown.webp", originalName: "unknown.webp", mimeType: "image/webp", size: UNKNOWN_IMAGE_BYTES.length }), + ]); + + expect(result.attachmentContents).toEqual([{ originalName: "unknown.webp", mimeType: "image/webp", text: null }]); + expect(result.imageContents).toEqual([{ type: "image", data: UNKNOWN_IMAGE_BYTES.toString("base64"), mimeType: "image/webp" }]); + }); + it("returns mixed text and image contents together", async () => { const root = await makeRoot(); await mkdir(join(root, ".fusion", "chat-room-attachments", "room-1"), { recursive: true }); diff --git a/packages/dashboard/src/chat-attachment-content.ts b/packages/dashboard/src/chat-attachment-content.ts index 027c0e0da2..ebbd5aafc5 100644 --- a/packages/dashboard/src/chat-attachment-content.ts +++ b/packages/dashboard/src/chat-attachment-content.ts @@ -1,4 +1,4 @@ -import type { ChatAttachment } from "@fusion/core"; +import { detectImageMimeFromBytes, type ChatAttachment } from "@fusion/core"; import { readFile } from "node:fs/promises"; import { basename, resolve } from "node:path"; import { CHAT_ALLOWED_MIME_TYPES } from "./routes/chat-attachment-config.js"; @@ -109,10 +109,15 @@ export async function readChatAttachmentContents( try { if (IMAGE_MIME_TYPES.has(attachment.mimeType)) { const data = await readFile(filePath); + const detectedMimeType = detectImageMimeFromBytes(data); + const imageMimeType = detectedMimeType ?? attachment.mimeType; + if (detectedMimeType && detectedMimeType !== attachment.mimeType) { + diagnostics?.warn(`Corrected chat image attachment media type for '${attachment.filename}' in ${getScopeLabel(scope)} from ${attachment.mimeType} to ${detectedMimeType}`); + } imageContents.push({ type: "image", data: data.toString("base64"), - mimeType: attachment.mimeType, + mimeType: imageMimeType, }); attachmentContents.push({ originalName: attachment.originalName, diff --git a/packages/engine/src/__tests__/triage.test.ts b/packages/engine/src/__tests__/triage.test.ts index 5c216713d7..6e7b7b5be4 100644 --- a/packages/engine/src/__tests__/triage.test.ts +++ b/packages/engine/src/__tests__/triage.test.ts @@ -1206,6 +1206,20 @@ describe("fast-mode triage", () => { }); }); +const PNG_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]); +const WEBP_BYTES = Buffer.from([0x52, 0x49, 0x46, 0x46, 0x01, 0x02, 0x03, 0x04, 0x57, 0x45, 0x42, 0x50]); + +function attachmentFixture(overrides: Partial): TaskDetail["attachments"][number] { + return { + filename: "1234567890-image.png", + originalName: "image.png", + mimeType: "image/png", + size: 100, + createdAt: "2026-01-01T00:00:00.000Z", + ...overrides, + }; +} + describe("readAttachmentContents", () => { let testDir = ""; const taskId = "FN-TEST"; @@ -1284,31 +1298,52 @@ describe("readAttachmentContents", () => { }); it("reads image as base64 content", async () => { - const attachments = [ - { - filename: "1234567890-image.png", - originalName: "image.png", - mimeType: "image/png" as const, - size: 100, - createdAt: "2026-01-01T00:00:00.000Z", - }, - ]; + const attachments = [attachmentFixture({ filename: "1234567890-image.png", originalName: "image.png", mimeType: "image/png" })]; - // Write fake PNG data (just some bytes) - const imageData = Buffer.from([0x89, 0x50, 0x4e, 0x47]); // PNG magic bytes await writeFile( join(testDir, ".fusion", "tasks", taskId, "attachments", "1234567890-image.png"), - imageData, + PNG_BYTES, ); const result = await readAttachmentContents(testDir, taskId, attachments); expect(result.attachmentContents).toHaveLength(1); - expect(result.attachmentContents[0].text).toBeNull(); + expect(result.attachmentContents[0]).toMatchObject({ originalName: "image.png", mimeType: "image/png", text: null }); expect(result.imageContents).toHaveLength(1); expect(result.imageContents[0].type).toBe("image"); expect(result.imageContents[0].mimeType).toBe("image/png"); - expect(result.imageContents[0].data).toBe(imageData.toString("base64")); + expect(result.imageContents[0].data).toBe(PNG_BYTES.toString("base64")); + }); + + it("corrects a webp-labeled PNG image block to image/png", async () => { + const attachments = [attachmentFixture({ filename: "mismatch.webp", originalName: "mismatch.webp", mimeType: "image/webp" })]; + await writeFile(join(testDir, ".fusion", "tasks", taskId, "attachments", "mismatch.webp"), PNG_BYTES); + + const result = await readAttachmentContents(testDir, taskId, attachments); + + expect(result.attachmentContents).toEqual([{ originalName: "mismatch.webp", mimeType: "image/webp", text: null }]); + expect(result.imageContents).toEqual([{ type: "image", data: PNG_BYTES.toString("base64"), mimeType: "image/png" }]); + }); + + it("corrects a png-labeled WEBP image block to image/webp", async () => { + const attachments = [attachmentFixture({ filename: "mismatch.png", originalName: "mismatch.png", mimeType: "image/png" })]; + await writeFile(join(testDir, ".fusion", "tasks", taskId, "attachments", "mismatch.png"), WEBP_BYTES); + + const result = await readAttachmentContents(testDir, taskId, attachments); + + expect(result.attachmentContents).toEqual([{ originalName: "mismatch.png", mimeType: "image/png", text: null }]); + expect(result.imageContents).toEqual([{ type: "image", data: WEBP_BYTES.toString("base64"), mimeType: "image/webp" }]); + }); + + it("falls back to stored image mime type for unrecognized bytes", async () => { + const unknownBytes = Buffer.from([0x01, 0x02, 0x03, 0x04]); + const attachments = [attachmentFixture({ filename: "unknown.webp", originalName: "unknown.webp", mimeType: "image/webp" })]; + await writeFile(join(testDir, ".fusion", "tasks", taskId, "attachments", "unknown.webp"), unknownBytes); + + const result = await readAttachmentContents(testDir, taskId, attachments); + + expect(result.attachmentContents).toEqual([{ originalName: "unknown.webp", mimeType: "image/webp", text: null }]); + expect(result.imageContents).toEqual([{ type: "image", data: unknownBytes.toString("base64"), mimeType: "image/webp" }]); }); it("skips unreadable attachments", async () => { diff --git a/packages/engine/src/triage.ts b/packages/engine/src/triage.ts index 074f51e742..d7b581ac5a 100644 --- a/packages/engine/src/triage.ts +++ b/packages/engine/src/triage.ts @@ -27,6 +27,7 @@ import { extractIntentSignature, findNearDuplicates, isNearDuplicateCanonicalInactive, + detectImageMimeFromBytes, applyFrontendUxCriteria, extractEffectiveWriteScopeFromPrompt, MAX_TASK_LIST_TEXT_CHARS, @@ -2754,10 +2755,15 @@ export async function readAttachmentContents( try { if (IMAGE_MIME_TYPES.has(att.mimeType)) { const data = await readFile(filePath); + const detectedMimeType = detectImageMimeFromBytes(data); + const imageMimeType = detectedMimeType ?? att.mimeType; + if (detectedMimeType && detectedMimeType !== att.mimeType) { + planLog.warn(`${taskId}: corrected image attachment media type for '${att.filename}' (${att.originalName}) from ${att.mimeType} to ${detectedMimeType}`); + } imageContents.push({ type: "image", data: data.toString("base64"), - mimeType: att.mimeType, + mimeType: imageMimeType, }); attachmentContents.push({ originalName: att.originalName,