diff --git a/packages/ai/src/cache-policy.ts b/packages/ai/src/cache-policy.ts index 81ae0f4ed5c6..b00d8cc24807 100644 --- a/packages/ai/src/cache-policy.ts +++ b/packages/ai/src/cache-policy.ts @@ -107,8 +107,13 @@ const markSystemBoundaries = (system: LLMRequest["system"], hint: CacheHint, bud const lastIndexOfRole = (messages: ReadonlyArray, role: Message["role"]): number => messages.findLastIndex((m) => m.role === role) +type CacheablePart = Exclude + +const isCacheablePart = (part: ContentPart): part is CacheablePart => + part.type !== "compaction" && part.type !== "effort" + // Mark the last text part of `messages[index]`. If no text part exists, mark -// the last content part regardless of type — that's the breakpoint position +// the last cacheable content part — that's the breakpoint position // in tool-result-only messages too. const markMessageAt = ( messages: ReadonlyArray, @@ -120,11 +125,11 @@ const markMessageAt = ( const target = messages[index]! if (target.content.length === 0) return messages const lastTextIndex = target.content.findLastIndex((part) => part.type === "text") - const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1 - const existing = target.content[markAt]! - if (("cache" in existing && existing.cache) || budget.remaining === 0) return messages + const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.findLastIndex(isCacheablePart) + const existing = target.content[markAt] + if (!existing || !isCacheablePart(existing) || existing.cache || budget.remaining === 0) return messages budget.remaining -= 1 - const nextContent = target.content.map((part, i) => (i === markAt ? ({ ...part, cache: hint } as ContentPart) : part)) + const nextContent = target.content.map((part, i) => (i === markAt ? { ...existing, cache: hint } : part)) const next = new Message({ ...target, content: nextContent }) // Single pass over `messages`, substituting the one updated entry. Long // conversations call this on every request, so avoid `.map()` here — its @@ -164,7 +169,7 @@ const countHints = (request: LLMRequest) => (count, message) => count + message.content.reduce( - (contentCount, part) => contentCount + ("cache" in part && part.cache !== undefined ? 1 : 0), + (contentCount, part) => contentCount + (isCacheablePart(part) && part.cache !== undefined ? 1 : 0), 0, ), 0, diff --git a/packages/ai/src/image.ts b/packages/ai/src/image.ts index b25efee811fc..be96446bf8fb 100644 --- a/packages/ai/src/image.ts +++ b/packages/ai/src/image.ts @@ -3,7 +3,14 @@ import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./gen import { Media } from "./media.js" import { MediaModel, composeRoute, tryRequest } from "./media-model.js" import { MediaRoute } from "./route/media.js" -import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js" +import { + AIError, + HttpOptions, + MediaUsage, + ProviderMetadata, + ProviderOptions, + type OpenString, +} from "./schema/index.js" import { ImageClient, Service } from "./image-client.js" // --------------------------------------------------------------------------- @@ -74,7 +81,7 @@ export class ImageRequest extends Schema.Class("Image.Request")({ aspectRatio: Schema.optional(ImageAspectRatio), seed: Schema.optional(Schema.Number), format: Schema.optional(Schema.String), - providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + providerOptions: Schema.optional(ProviderOptions), http: Schema.optional(HttpOptions), }) { declare protected readonly _ImageRequest: void diff --git a/packages/ai/src/media-model.ts b/packages/ai/src/media-model.ts index 6e453ab7ad33..8d611564d740 100644 --- a/packages/ai/src/media-model.ts +++ b/packages/ai/src/media-model.ts @@ -2,7 +2,14 @@ import { Effect } from "effect" import { Endpoint } from "./route/endpoint.js" import { MediaRoute } from "./route/media.js" import type { MediaProtocol } from "./route/media-protocol.js" -import { AIError, HttpOptions, InvalidRequestError, ModelID, ProviderID } from "./schema/index.js" +import { + AIError, + HttpOptions, + InvalidRequestError, + ModelID, + ProviderID, + type ProviderOptions, +} from "./schema/index.js" /** * What every media model carries: ids, the configured route, and deployment `http` overlays. Modality classes @@ -27,7 +34,7 @@ export class MediaModel { } export namespace MediaModel { - export type Options = Record + export type Options = ProviderOptions export interface Input { readonly id: string | ModelID diff --git a/packages/ai/src/speech.ts b/packages/ai/src/speech.ts index 9c07febe72c0..38fc50d07c44 100644 --- a/packages/ai/src/speech.ts +++ b/packages/ai/src/speech.ts @@ -3,7 +3,14 @@ import { ProgressEvent, QueuedEvent } from "./generation.js" import { Media } from "./media.js" import { MediaModel, composeRoute, tryRequest } from "./media-model.js" import { MediaRoute } from "./route/media.js" -import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js" +import { + AIError, + HttpOptions, + MediaUsage, + ProviderMetadata, + ProviderOptions, + type OpenString, +} from "./schema/index.js" import { SpeechClient, Service } from "./speech-client.js" // --------------------------------------------------------------------------- @@ -76,7 +83,7 @@ export class SpeechRequest extends Schema.Class("Speech.Request") language: Schema.optional(Schema.String), instructions: Schema.optional(Schema.String), timestamps: Schema.optional(Schema.Boolean), - providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + providerOptions: Schema.optional(ProviderOptions), http: Schema.optional(HttpOptions), }) { declare protected readonly _SpeechRequest: void diff --git a/packages/ai/src/transcription.ts b/packages/ai/src/transcription.ts index 1302d72fccd6..33f0a8f55af5 100644 --- a/packages/ai/src/transcription.ts +++ b/packages/ai/src/transcription.ts @@ -3,7 +3,7 @@ import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./gen import { Media } from "./media.js" import { MediaModel, composeRoute, tryRequest } from "./media-model.js" import { MediaRoute } from "./route/media.js" -import { AIError, HttpOptions, MediaUsage, ProviderMetadata } from "./schema/index.js" +import { AIError, HttpOptions, MediaUsage, ProviderMetadata, ProviderOptions } from "./schema/index.js" import { TranscriptionClient, Service } from "./transcription-client.js" // --------------------------------------------------------------------------- @@ -74,7 +74,7 @@ export class TranscriptionRequest extends Schema.Class("Tr timestamps: Schema.optional(TranscriptionTimestamps), diarize: Schema.optional(Schema.Boolean), speakers: Schema.optional(Schema.Int), - providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + providerOptions: Schema.optional(ProviderOptions), http: Schema.optional(HttpOptions), }) { declare protected readonly _TranscriptionRequest: void diff --git a/packages/ai/src/video.ts b/packages/ai/src/video.ts index c97ac08f43f6..65a2109a4da1 100644 --- a/packages/ai/src/video.ts +++ b/packages/ai/src/video.ts @@ -3,7 +3,14 @@ import { Generation, ProgressEvent, QueuedEvent, type AwaitOptions } from "./gen import { Media } from "./media.js" import { MediaModel, composeRoute, tryRequest } from "./media-model.js" import { MediaRoute } from "./route/media.js" -import { AIError, HttpOptions, MediaUsage, ProviderMetadata, type OpenString } from "./schema/index.js" +import { + AIError, + HttpOptions, + MediaUsage, + ProviderMetadata, + ProviderOptions, + type OpenString, +} from "./schema/index.js" import { VideoClient, Service } from "./video-client.js" // --------------------------------------------------------------------------- @@ -79,7 +86,7 @@ export class VideoRequest extends Schema.Class("Video.Request")({ n: Schema.optional(Schema.Int), seed: Schema.optional(Schema.Number), negativePrompt: Schema.optional(Schema.String), - providerOptions: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + providerOptions: Schema.optional(ProviderOptions), http: Schema.optional(HttpOptions), }) { declare protected readonly _VideoRequest: void diff --git a/packages/ai/test/cache-policy.test.ts b/packages/ai/test/cache-policy.test.ts index 97664896bfc2..b8513c810ae7 100644 --- a/packages/ai/test/cache-policy.test.ts +++ b/packages/ai/test/cache-policy.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test" import { Effect } from "effect" -import { CacheHint, LLM, Message } from "../src/index.js" +import { CacheHint, CompactionPart, LLM, Message, ProviderID, ToolCallPart } from "../src/index.js" import { Auth } from "../src/route.js" import { compileRequest } from "../src/route/client.js" import { @@ -525,6 +525,31 @@ describe("applyCachePolicy", () => { }), ) + test("skips non-cacheable parts when placing message breakpoints", () => { + const compaction = CompactionPart.make({ provider: ProviderID.make("anthropic"), text: "summary" }) + const toolCall = ToolCallPart.make({ id: "tc_1", name: "lookup", input: {} }) + + const mixed = LLM.request({ + model: anthropicModel, + messages: [Message.assistant([toolCall, compaction])], + cache: { messages: "latest-assistant" }, + }) + const appliedMixed = applyCachePolicy(mixed) + expect(appliedMixed.messages[0]?.content[0]).toMatchObject({ + type: "tool-call", + id: "tc_1", + cache: { type: "ephemeral" }, + }) + expect(appliedMixed.messages[0]?.content[1]).toEqual(compaction) + + const compactionOnly = LLM.request({ + model: anthropicModel, + messages: [Message.assistant([compaction])], + cache: { messages: "latest-assistant" }, + }) + expect(applyCachePolicy(compactionOnly)).toBe(compactionOnly) + }) + test("returns the same request reference when policy is a no-op (pure function)", () => { const request = LLM.request({ model: anthropicModel,