fix(ai): skip non-cacheable content parts when placing cache breakpoints (#57610)

packages/ai/src/cache-policy.ts

@@ -107,8 +107,13 @@ const markSystemBoundaries = (system: LLMRequest["system"], hint: CacheHint, bud
 const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]): number =>
   messages.findLastIndex((m) => m.role === role)
 
+type CacheablePart = Exclude<ContentPart, { readonly type: "compaction" | "effort" }>
+
+const isCacheablePart = (part: ContentPart): part is CacheablePart =>
+  part.type !== "compaction" && part.type !== "effort"
+
 // Mark the last text part of `messages[index]`. If no text part exists, mark
-// the last content part regardless of type — that's the breakpoint position
+// the last cacheable content part — that's the breakpoint position
 // in tool-result-only messages too.
 const markMessageAt = (
   messages: ReadonlyArray<Message>,
@@ -120,11 +125,11 @@ const markMessageAt = (
   const target = messages[index]!
   if (target.content.length === 0) return messages
   const lastTextIndex = target.content.findLastIndex((part) => part.type === "text")
-  const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1
-  const existing = target.content[markAt]!
-  if (("cache" in existing && existing.cache) || budget.remaining === 0) return messages
+  const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.findLastIndex(isCacheablePart)
+  const existing = target.content[markAt]
+  if (!existing || !isCacheablePart(existing) || existing.cache || budget.remaining === 0) return messages
   budget.remaining -= 1
-  const nextContent = target.content.map((part, i) => (i === markAt ? ({ ...part, cache: hint } as ContentPart) : part))
+  const nextContent = target.content.map((part, i) => (i === markAt ? { ...existing, cache: hint } : part))
   const next = new Message({ ...target, content: nextContent })
   // Single pass over `messages`, substituting the one updated entry. Long
   // conversations call this on every request, so avoid `.map()` here — its
@@ -164,7 +169,7 @@ const countHints = (request: LLMRequest) =>
     (count, message) =>
       count +
       message.content.reduce(
-        (contentCount, part) => contentCount + ("cache" in part && part.cache !== undefined ? 1 : 0),
+        (contentCount, part) => contentCount + (isCacheablePart(part) && part.cache !== undefined ? 1 : 0),
         0,
       ),
     0,

packages/ai/src/image.ts

@@ -3,7 +3,14 @@ import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./gen
 import { Media } from "./media.js"
 import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
 import { MediaRoute } from "./route/media.js"
-import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js"
+import {
+  AIError,
+  HttpOptions,
+  MediaUsage,
+  ProviderMetadata,
+  ProviderOptions,
+  type OpenString,
+} from "./schema/index.js"
 import { ImageClient, Service } from "./image-client.js"
 
 // ---------------------------------------------------------------------------
@@ -74,7 +81,7 @@ export class ImageRequest extends Schema.Class<ImageRequest>("Image.Request")({
   aspectRatio: Schema.optional(ImageAspectRatio),
   seed: Schema.optional(Schema.Number),
   format: Schema.optional(Schema.String),
-  providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
+  providerOptions: Schema.optional(ProviderOptions),
   http: Schema.optional(HttpOptions),
 }) {
   declare protected readonly _ImageRequest: void

packages/ai/src/media-model.ts

@@ -2,7 +2,14 @@ import { Effect } from "effect"
 import { Endpoint } from "./route/endpoint.js"
 import { MediaRoute } from "./route/media.js"
 import type { MediaProtocol } from "./route/media-protocol.js"
-import { AIError, HttpOptions, InvalidRequestError, ModelID, ProviderID } from "./schema/index.js"
+import {
+  AIError,
+  HttpOptions,
+  InvalidRequestError,
+  ModelID,
+  ProviderID,
+  type ProviderOptions,
+} from "./schema/index.js"
 
 /**
  * What every media model carries: ids, the configured route, and deployment `http` overlays. Modality classes
@@ -27,7 +34,7 @@ export class MediaModel<Route, Options> {
 }
 
 export namespace MediaModel {
-  export type Options = Record<string, unknown>
+  export type Options = ProviderOptions
 
   export interface Input<Route> {
     readonly id: string | ModelID

packages/ai/src/speech.ts

@@ -3,7 +3,14 @@ import { ProgressEvent, QueuedEvent } from "./generation.js"
 import { Media } from "./media.js"
 import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
 import { MediaRoute } from "./route/media.js"
-import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js"
+import {
+  AIError,
+  HttpOptions,
+  MediaUsage,
+  ProviderMetadata,
+  ProviderOptions,
+  type OpenString,
+} from "./schema/index.js"
 import { SpeechClient, Service } from "./speech-client.js"
 
 // ---------------------------------------------------------------------------
@@ -76,7 +83,7 @@ export class SpeechRequest extends Schema.Class<SpeechRequest>("Speech.Request")
   language: Schema.optional(Schema.String),
   instructions: Schema.optional(Schema.String),
   timestamps: Schema.optional(Schema.Boolean),
-  providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
+  providerOptions: Schema.optional(ProviderOptions),
   http: Schema.optional(HttpOptions),
 }) {
   declare protected readonly _SpeechRequest: void

packages/ai/src/transcription.ts

@@ -3,7 +3,7 @@ import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./gen
 import { Media } from "./media.js"
 import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
 import { MediaRoute } from "./route/media.js"
-import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js"
+import { AIError, HttpOptions, MediaUsage, ProviderMetadata, ProviderOptions } from "./schema/index.js"
 import { TranscriptionClient, Service } from "./transcription-client.js"
 
 // ---------------------------------------------------------------------------
@@ -74,7 +74,7 @@ export class TranscriptionRequest extends Schema.Class<TranscriptionRequest>("Tr
   timestamps: Schema.optional(TranscriptionTimestamps),
   diarize: Schema.optional(Schema.Boolean),
   speakers: Schema.optional(Schema.Int),
-  providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
+  providerOptions: Schema.optional(ProviderOptions),
   http: Schema.optional(HttpOptions),
 }) {
   declare protected readonly _TranscriptionRequest: void

packages/ai/src/video.ts

@@ -3,7 +3,14 @@ import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./gen
 import { Media } from "./media.js"
 import { MediaModel, composeRoute, tryRequest } from "./media-model.js"
 import { MediaRoute } from "./route/media.js"
-import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js"
+import {
+  AIError,
+  HttpOptions,
+  MediaUsage,
+  ProviderMetadata,
+  ProviderOptions,
+  type OpenString,
+} from "./schema/index.js"
 import { VideoClient, Service } from "./video-client.js"
 
 // ---------------------------------------------------------------------------
@@ -79,7 +86,7 @@ export class VideoRequest extends Schema.Class<VideoRequest>("Video.Request")({
   n: Schema.optional(Schema.Int),
   seed: Schema.optional(Schema.Number),
   negativePrompt: Schema.optional(Schema.String),
-  providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
+  providerOptions: Schema.optional(ProviderOptions),
   http: Schema.optional(HttpOptions),
 }) {
   declare protected readonly _VideoRequest: void

packages/ai/test/cache-policy.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test"
 import { Effect } from "effect"
-import { CacheHint, LLM, Message } from "../src/index.js"
+import { CacheHint, CompactionPart, LLM, Message, ProviderID, ToolCallPart } from "../src/index.js"
 import { Auth } from "../src/route.js"
 import { compileRequest } from "../src/route/client.js"
 import {
@@ -525,6 +525,31 @@ describe("applyCachePolicy", () => {
     }),
   )
 
+  test("skips non-cacheable parts when placing message breakpoints", () => {
+    const compaction = CompactionPart.make({ provider: ProviderID.make("anthropic"), text: "summary" })
+    const toolCall = ToolCallPart.make({ id: "tc_1", name: "lookup", input: {} })
+
+    const mixed = LLM.request({
+      model: anthropicModel,
+      messages: [Message.assistant([toolCall, compaction])],
+      cache: { messages: "latest-assistant" },
+    })
+    const appliedMixed = applyCachePolicy(mixed)
+    expect(appliedMixed.messages[0]?.content[0]).toMatchObject({
+      type: "tool-call",
+      id: "tc_1",
+      cache: { type: "ephemeral" },
+    })
+    expect(appliedMixed.messages[0]?.content[1]).toEqual(compaction)
+
+    const compactionOnly = LLM.request({
+      model: anthropicModel,
+      messages: [Message.assistant([compaction])],
+      cache: { messages: "latest-assistant" },
+    })
+    expect(applyCachePolicy(compactionOnly)).toBe(compactionOnly)
+  })
+
   test("returns the same request reference when policy is a no-op (pure function)", () => {
     const request = LLM.request({
       model: anthropicModel,