From 194f853f0bbf2bda37f4d034db8a6d99d159d8ac Mon Sep 17 00:00:00 2001
From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com>
Date: Fri, 23 Oct 2026 10:19:29 -0500
Subject: [PATCH] fix(ai): detect image type from bytes when OpenAI-compatible
 hosts omit output_format (#58475)

---
 packages/ai/src/protocols/openai-images.ts | 22 ++++++++--------------
 packages/ai/test/image.test.ts             | 14 +++++++++-----
 2 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/packages/ai/src/protocols/openai-images.ts b/packages/ai/src/protocols/openai-images.ts
index ff938dd35445..f778cdb5786b 100644
--- a/packages/ai/src/protocols/openai-images.ts
+++ b/packages/ai/src/protocols/openai-images.ts
@@ -86,8 +86,8 @@ const StreamEvent = Schema.Union([
 const decodeEvent = route.decodeFrame(StreamEvent)
 const decodeDocument = Schema.decodeUnknownEffect(Schema.fromJsonString(OpenAIImageResponse))
 
-/** `generate` reads the whole JSON response as one frame, with the requested format for responses that omit it. */
-type Frame = string | { readonly document: string; readonly requested: string | undefined }
+/** `generate` reads the whole JSON response as one frame. */
+type Frame = string | { readonly document: string }
 
 // ---------------------------------------------------------------------------
 // 4. Parser state
@@ -180,12 +180,6 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Me
 // 6. Stream parsing
 // ---------------------------------------------------------------------------
 
-const requestedFormat = (body: MediaProtocol.Body) => {
-  if (body.type === "binary") return undefined
-  const value = body.type === "json" ? body.value.output_format : body.value.get("output_format")
-  return typeof value === "string" ? value : undefined
-}
-
 const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage | undefined =>
   value === undefined
     ? undefined
@@ -198,7 +192,7 @@ const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage
       }
 
 /** `size` echoes the rendered `WIDTHxHEIGHT`; `auto` or any other value leaves the dimensions unknown. */
-const info = (format: string, size: string | undefined): Media.Info => {
+const info = (format: string | undefined, size: string | undefined): Media.Info => {
   const match = size?.match(/^(\d+)x(\d+)$/)
   return match ? { format, width: Number(match[1]), height: Number(match[2]) } : { format }
 }
@@ -234,9 +228,11 @@ const onDocument = Effect.fnUntraced(function* (frame: Exclude<Frame, string>) {
   const decoded = yield* decodeDocument(frame.document).pipe(
     Effect.mapError((cause) => invalid(`${route.name} returned an invalid response`, cause)),
   )
-  const format = decoded.output_format ?? frame.requested ?? "png"
+  // Compatible hosts may omit `output_format` and ignore the requested one, so the bytes decide unreported types.
+  const format = decoded.output_format
+  const mediaType = format === undefined ? undefined : `image/${format}`
   const images = yield* Effect.forEach(decoded.data, (item, index) =>
-    MediaInput.decodedAsset(invalid, `${route.name} result ${index}`, item.b64_json, `image/${format}`, {
+    MediaInput.decodedAsset(invalid, `${route.name} result ${index}`, item.b64_json, mediaType, {
       info: info(format, decoded.size),
     }),
   )
@@ -281,9 +277,7 @@ export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>(
   frames: (bytes, context) =>
     context.request.mode === "stream"
       ? Framing.sse.frame(bytes)
-      : Framing.document
-          .frame(bytes)
-          .pipe(Stream.map((document) => ({ document, requested: requestedFormat(context.body) }))),
+      : Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
   initial: () => ({ completed: 0 }),
   step,
   finish,
diff --git a/packages/ai/test/image.test.ts b/packages/ai/test/image.test.ts
index 7a6435e88d1a..8dfce1c7bd02 100644
--- a/packages/ai/test/image.test.ts
+++ b/packages/ai/test/image.test.ts
@@ -127,7 +127,7 @@ describe("Image", () => {
     ),
   )
 
-  it.effect("sends only model and prompt by default and decodes OpenAI bytes as png", () =>
+  it.effect("sends only model and prompt by default and sniffs bytes when output_format is missing", () =>
     Effect.gen(function* () {
       const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" })
       expect(openai.image("gpt-image-2").route.id).toBe("openai-images")
@@ -140,7 +140,7 @@ describe("Image", () => {
                   const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
                   expect(web.url).toBe("https://openai.test/v1/images/generations")
                   expect(JSON.parse(input.text)).toEqual({ model: "gpt-image-2", prompt: "A lighthouse" })
-                  return input.respond(JSON.stringify({ data: [{ b64_json: "AQID" }] }), {
+                  return input.respond(JSON.stringify({ data: [{ b64_json: "/9j/4AAQ" }] }), {
                     headers: { "content-type": "application/json" },
                   })
                 }),
@@ -149,8 +149,12 @@ describe("Image", () => {
           ),
         ),
       )
-      expect(response.image.source).toEqual({ type: "bytes", data: Uint8Array.from([1, 2, 3]), mediaType: "image/png" })
-      expect(response.image.info).toEqual({ format: "png" })
+      expect(response.image.source).toEqual({
+        type: "bytes",
+        data: Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]),
+        mediaType: "image/jpeg",
+      })
+      expect(response.providerMetadata).toEqual({ openai: {} })
     }),
   )
 
@@ -186,7 +190,7 @@ describe("Image", () => {
                 provider_future_option: { enabled: true },
               })
               return Effect.succeed(
-                input.respond(JSON.stringify({ data: [{ b64_json: "AQID" }] }), {
+                input.respond(JSON.stringify({ data: [{ b64_json: "AQID" }], output_format: "avif" }), {
                   headers: { "content-type": "application/json" },
                 }),
               )