From 53b044d3139ddb28a4504c934cd4fa1e7e9829d7 Mon Sep 17 00:00:00 2001
From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com>
Date: Sat, 24 Oct 2026 12:26:47 -0500
Subject: [PATCH] fix(ai): encode video in Chat routes by default (#58613)

---
 packages/ai/src/protocols/openai-chat.ts      |  7 +++-
 .../provider/openai-compatible-chat.test.ts   | 42 ++++++++++++++++++-
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts
index f8d533600f69..ffef495485a5 100644
--- a/packages/ai/src/protocols/openai-chat.ts
+++ b/packages/ai/src/protocols/openai-chat.ts
@@ -128,6 +128,10 @@ const OpenAIChatUserContent = Schema.Union([
     type: Schema.Literal("image_url"),
     image_url: Schema.Struct({ url: Schema.String }),
   }),
+  Schema.Struct({
+    type: Schema.Literal("video_url"),
+    video_url: Schema.Struct({ url: Schema.String }),
+  }),
   Schema.Struct({
     type: Schema.Literal("file"),
     file: Schema.Struct({ filename: Schema.String, file_data: Schema.String }),
@@ -373,10 +377,11 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart) {
         file_data: (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl,
       },
     }
-  if (part.media.kind !== "image")
+  if (part.media.kind !== "image" && part.media.kind !== "video")
     return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.media.mediaType}`)
   const url =
     ProviderShared.mediaUrl(part.media) ?? (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl
+  if (part.media.kind === "video") return { type: "video_url" as const, video_url: { url } }
   return { type: "image_url" as const, image_url: { url } }
 })
 
diff --git a/packages/ai/test/provider/openai-compatible-chat.test.ts b/packages/ai/test/provider/openai-compatible-chat.test.ts
index 368a8d5ed7f4..c18a372cd53b 100644
--- a/packages/ai/test/provider/openai-compatible-chat.test.ts
+++ b/packages/ai/test/provider/openai-compatible-chat.test.ts
@@ -1,7 +1,7 @@
 import { describe, expect } from "bun:test"
 import { Effect, Schema } from "effect"
 import { HttpClientRequest } from "effect/http"
-import { LLM, LLMRequest, Message, ToolCallPart, ToolChoice, ToolDefinition } from "../../src/index.js"
+import { LLM, LLMRequest, Media, Message, ToolCallPart, ToolChoice, ToolDefinition } from "../../src/index.js"
 import { Auth, LLMClient } from "../../src/route.js"
 import { compileRequest } from "../../src/route/client.js"
 import * as OpenAICompatibleChat from "../../src/protocols/openai-compatible-chat.js"
@@ -41,6 +41,46 @@ const usageChunk = (usage: object) => ({
 })
 
 describe("OpenAI-compatible Chat route", () => {
+  it.effect("serializes user and tool video for unknown compatible providers", () =>
+    Effect.gen(function* () {
+      const inline = "data:video/mp4;base64,AAEC"
+      const url = "https://example.com/clip.mp4"
+      const prepared = yield* compileRequest(
+        LLM.request({
+          model: OpenAICompatibleChat.route
+            .with({ provider: "custom", endpoint: { baseURL: "https://api.custom.test/v1" } })
+            .model({ id: "video-model" }),
+          messages: [
+            Message.user([
+              Message.media(Media.fromDataUrl(inline)),
+              Message.media(Media.url(url, { mediaType: "video/mp4" })),
+            ]),
+            Message.assistant(ToolCallPart.make({ id: "call_video", name: "read", input: {} })),
+            Message.tool({
+              id: "call_video",
+              name: "read",
+              result: { type: "content", value: [{ type: "file", uri: url, mime: "video/mp4" }] },
+            }),
+          ],
+        }),
+      )
+      expect(prepared.body.messages[0]).toEqual({
+        role: "user",
+        content: [
+          { type: "video_url", video_url: { url: inline } },
+          { type: "video_url", video_url: { url } },
+        ],
+      })
+      expect(prepared.body.messages.slice(-2)).toEqual([
+        { role: "tool", tool_call_id: "call_video", content: "" },
+        {
+          role: "user",
+          content: [{ type: "video_url", video_url: { url } }],
+        },
+      ])
+    }),
+  )
+
   it.effect("prepares generic Chat target", () =>
     Effect.gen(function* () {
       const prepared = yield* compileRequest(