vrr/opencode
2
0
Fork 0
mirror of https://github.com/anomalyco/opencode.git synced 2026-11-01 07:20:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

fix(ai): encode video in Chat routes by default (#58613)

Browse source
This commit is contained in:
Aiden Cline 2026-10-24 12:26:47 -05:00 • committed by GitHub
parent cab780b970
commit 53b044d313
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 47 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
packages/ai/src/protocols/openai-chat.ts
View file

@ -128,6 +128,10 @@ const OpenAIChatUserContent = Schema.Union([
type: Schema.Literal("image_url"),
image_url: Schema.Struct({ url: Schema.String }),
}),
Schema.Struct({
type: Schema.Literal("video_url"),
video_url: Schema.Struct({ url: Schema.String }),
}),
Schema.Struct({
type: Schema.Literal("file"),
file: Schema.Struct({ filename: Schema.String, file_data: Schema.String }),
@ -373,10 +377,11 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart) {
file_data: (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl,
},
}
if (part.media.kind !== "image")
if (part.media.kind !== "image" && part.media.kind !== "video")
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.media.mediaType}`)
const url =
ProviderShared.mediaUrl(part.media) ?? (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl
if (part.media.kind === "video") return { type: "video_url" as const, video_url: { url } }
return { type: "image_url" as const, image_url: { url } }
})

42
packages/ai/test/provider/openai-compatible-chat.test.ts
View file

@ -1,7 +1,7 @@
import { describe, expect } from "bun:test"
import { Effect, Schema } from "effect"
import { HttpClientRequest } from "effect/http"
import { LLM, LLMRequest, Message, ToolCallPart, ToolChoice, ToolDefinition } from "../../src/index.js"
import { LLM, LLMRequest, Media, Message, ToolCallPart, ToolChoice, ToolDefinition } from "../../src/index.js"
import { Auth, LLMClient } from "../../src/route.js"
import { compileRequest } from "../../src/route/client.js"
import * as OpenAICompatibleChat from "../../src/protocols/openai-compatible-chat.js"
@ -41,6 +41,46 @@ const usageChunk = (usage: object) => ({
})
describe("OpenAI-compatible Chat route", () => {
it.effect("serializes user and tool video for unknown compatible providers", () =>
Effect.gen(function* () {
const inline = "data:video/mp4;base64,AAEC"
const url = "https://example.com/clip.mp4"
const prepared = yield* compileRequest(
LLM.request({
model: OpenAICompatibleChat.route
.with({ provider: "custom", endpoint: { baseURL: "https://api.custom.test/v1" } })
.model({ id: "video-model" }),
messages: [
Message.user([
Message.media(Media.fromDataUrl(inline)),
Message.media(Media.url(url, { mediaType: "video/mp4" })),
]),
Message.assistant(ToolCallPart.make({ id: "call_video", name: "read", input: {} })),
Message.tool({
id: "call_video",
name: "read",
result: { type: "content", value: [{ type: "file", uri: url, mime: "video/mp4" }] },
}),
],
}),
)
expect(prepared.body.messages[0]).toEqual({
role: "user",
content: [
{ type: "video_url", video_url: { url: inline } },
{ type: "video_url", video_url: { url } },
],
})
expect(prepared.body.messages.slice(-2)).toEqual([
{ role: "tool", tool_call_id: "call_video", content: "" },
{
role: "user",
content: [{ type: "video_url", video_url: { url } }],
},
])
}),
)
it.effect("prepares generic Chat target", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(