vrr/opencode
2
0
Fork 0
mirror of https://github.com/anomalyco/opencode.git synced 2026-10-31 22:20:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

fix(ai): detect image type from bytes when OpenAI-compatible hosts omit output_format (#58475)

Browse source
This commit is contained in:
Aiden Cline 2026-10-23 10:19:29 -05:00 • committed by GitHub
parent 1e89b18186
commit 194f853f0b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 17 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

22
packages/ai/src/protocols/openai-images.ts
View file

@ -86,8 +86,8 @@ const StreamEvent = Schema.Union([
const decodeEvent = route.decodeFrame(StreamEvent)
const decodeDocument = Schema.decodeUnknownEffect(Schema.fromJsonString(OpenAIImageResponse))
/** `generate` reads the whole JSON response as one frame, with the requested format for responses that omit it. */
type Frame = string | { readonly document: string; readonly requested: string | undefined }
/** `generate` reads the whole JSON response as one frame. */
type Frame = string | { readonly document: string }
// ---------------------------------------------------------------------------
// 4. Parser state
@ -180,12 +180,6 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Me
// 6. Stream parsing
// ---------------------------------------------------------------------------
const requestedFormat = (body: MediaProtocol.Body) => {
if (body.type === "binary") return undefined
const value = body.type === "json" ? body.value.output_format : body.value.get("output_format")
return typeof value === "string" ? value : undefined
}
const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage | undefined =>
value === undefined
? undefined
@ -198,7 +192,7 @@ const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage
}
/** `size` echoes the rendered `WIDTHxHEIGHT`; `auto` or any other value leaves the dimensions unknown. */
const info = (format: string, size: string | undefined): Media.Info => {
const info = (format: string | undefined, size: string | undefined): Media.Info => {
const match = size?.match(/^(\d+)x(\d+)$/)
return match ? { format, width: Number(match[1]), height: Number(match[2]) } : { format }
}
@ -234,9 +228,11 @@ const onDocument = Effect.fnUntraced(function* (frame: Exclude<Frame, string>) {
const decoded = yield* decodeDocument(frame.document).pipe(
Effect.mapError((cause) => invalid(`${route.name} returned an invalid response`, cause)),
)
const format = decoded.output_format ?? frame.requested ?? "png"
// Compatible hosts may omit `output_format` and ignore the requested one, so the bytes decide unreported types.
const format = decoded.output_format
const mediaType = format === undefined ? undefined : `image/${format}`
const images = yield* Effect.forEach(decoded.data, (item, index) =>
MediaInput.decodedAsset(invalid, `${route.name} result ${index}`, item.b64_json, `image/${format}`, {
MediaInput.decodedAsset(invalid, `${route.name} result ${index}`, item.b64_json, mediaType, {
info: info(format, decoded.size),
}),
)
@ -281,9 +277,7 @@ export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>(
frames: (bytes, context) =>
context.request.mode === "stream"
? Framing.sse.frame(bytes)
: Framing.document
.frame(bytes)
.pipe(Stream.map((document) => ({ document, requested: requestedFormat(context.body) }))),
: Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
initial: () => ({ completed: 0 }),
step,
finish,

14
packages/ai/test/image.test.ts
View file

@ -127,7 +127,7 @@ describe("Image", () => {
),
)
it.effect("sends only model and prompt by default and decodes OpenAI bytes as png", () =>
it.effect("sends only model and prompt by default and sniffs bytes when output_format is missing", () =>
Effect.gen(function* () {
const openai = OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" })
expect(openai.image("gpt-image-2").route.id).toBe("openai-images")
@ -140,7 +140,7 @@ describe("Image", () => {
const web = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
expect(web.url).toBe("https://openai.test/v1/images/generations")
expect(JSON.parse(input.text)).toEqual({ model: "gpt-image-2", prompt: "A lighthouse" })
return input.respond(JSON.stringify({ data: [{ b64_json: "AQID" }] }), {
return input.respond(JSON.stringify({ data: [{ b64_json: "/9j/4AAQ" }] }), {
headers: { "content-type": "application/json" },
})
}),
@ -149,8 +149,12 @@ describe("Image", () => {
),
),
)
expect(response.image.source).toEqual({ type: "bytes", data: Uint8Array.from([1, 2, 3]), mediaType: "image/png" })
expect(response.image.info).toEqual({ format: "png" })
expect(response.image.source).toEqual({
type: "bytes",
data: Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]),
mediaType: "image/jpeg",
})
expect(response.providerMetadata).toEqual({ openai: {} })
}),
)
@ -186,7 +190,7 @@ describe("Image", () => {
provider_future_option: { enabled: true },
})
return Effect.succeed(
input.respond(JSON.stringify({ data: [{ b64_json: "AQID" }] }), {
input.respond(JSON.stringify({ data: [{ b64_json: "AQID" }], output_format: "avif" }), {
headers: { "content-type": "application/json" },
}),
)