From 69a71426ce74bf8362fa6183937b008fa24a5dfe Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:23:14 -0500 Subject: [PATCH] feat(core): configure Anthropic cache breakpoints with compatibility.cacheFormat (#54945) --- packages/ai/src/cache-policy.ts | 26 +++++++------ packages/ai/src/protocols/openai-chat.ts | 10 ++++- packages/ai/src/schema/options.ts | 5 +++ .../provider/openai-compatible-chat.test.ts | 37 +++++++++++++++++++ packages/ai/test/provider/openrouter.test.ts | 22 ++++++++++- .../client/src/promise/generated/types.ts | 4 ++ .../core/src/plugin/provider/databricks.ts | 9 +++++ .../src/plugin/provider/snowflake-cortex.ts | 3 ++ packages/core/test/config/provider.test.ts | 25 +++++++++++++ .../plugin/provider-snowflake-cortex.test.ts | 2 + packages/protocol/openapi.json | 10 +++++ packages/schema/src/model-compatibility.ts | 5 +++ services/www/openapi.json | 10 +++++ services/www/public/openapi.json | 10 +++++ services/www/src/docs/content/models.mdx | 32 ++++++++++++++++ 15 files changed, 196 insertions(+), 14 deletions(-) diff --git a/packages/ai/src/cache-policy.ts b/packages/ai/src/cache-policy.ts index 8ddde6206933..81ae0f4ed5c6 100644 --- a/packages/ai/src/cache-policy.ts +++ b/packages/ai/src/cache-policy.ts @@ -171,17 +171,7 @@ const countHints = (request: LLMRequest) => ) export const applyCachePolicy = (request: LLMRequest): LLMRequest => { - const route = request.model.route.id - if (!RESPECTS_INLINE_HINTS.has(route)) return request - const policy = - (route === "openrouter" || route === "vercel-ai-gateway-messages") && - (request.cache === undefined || request.cache === "auto") - ? gatewayPolicy(request.model.id) - : route === "alibaba-chat" && (request.cache === undefined || request.cache === "auto") - ? request.model.id.toLowerCase().startsWith("qwen") - ? QWEN - : NONE - : resolve(request.cache) + const policy = selectPolicy(request) if (!policy.tools && !policy.system && !policy.messages) return request const hint = makeHint(policy.ttlSeconds) @@ -193,3 +183,17 @@ export const applyCachePolicy = (request: LLMRequest): LLMRequest => { if (tools === request.tools && system === request.system && messages === request.messages) return request return LLMRequest.update(request, { tools, system, messages }) } + +// A declared cache format overrides the route allowlist and its model-ID heuristics. +const selectPolicy = (request: LLMRequest): CachePolicyObject => { + const format = request.model.compatibility?.cacheFormat + const route = request.model.route.id + const auto = request.cache === undefined || request.cache === "auto" + if (format === "alibaba") return auto ? QWEN : { ...resolve(request.cache), tools: false } + if (format === "anthropic") return resolve(request.cache) + if (!RESPECTS_INLINE_HINTS.has(route)) return NONE + if (!auto) return resolve(request.cache) + if (route === "openrouter" || route === "vercel-ai-gateway-messages") return gatewayPolicy(request.model.id) + if (route === "alibaba-chat") return request.model.id.toLowerCase().startsWith("qwen") ? QWEN : NONE + return AUTO +} diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index f2d3f6cd30a5..27e9843d628c 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -24,6 +24,7 @@ import { } from "../schema/index.js" import { classifyProviderFailure } from "../provider-error.js" import { isRecord, JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js" +import { cacheControl } from "./utils/cache.js" import { OpenAIOptions } from "./utils/openai-options.js" import { Lifecycle } from "./utils/lifecycle.js" import { ToolStream } from "./utils/tool-stream.js" @@ -827,15 +828,16 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* ( request.model.compatibility?.zaiToolStream ?? detectZaiToolStream(provider, baseURL, request.model.id) const hasHistory = hasToolHistory(request.messages) const hasActiveTools = flattened.tools.length > 0 + const lowering = { ...options, cacheControl: options.cacheControl ?? declaredCacheControl(request) } return { model: request.model.id, - messages: yield* lowerMessages(flattened.request, options), + messages: yield* lowerMessages(flattened.request, lowering), tools: flattened.tools.length === 0 ? hasHistory ? [] : undefined - : flattened.tools.map((tool) => lowerTool(tool, options, supportsStrictMode)), + : flattened.tools.map((tool) => lowerTool(tool, lowering, supportsStrictMode)), tool_choice: hasActiveTools && request.toolChoice ? yield* lowerToolChoice(request.toolChoice) : undefined, stream: true as const, ...(supportsUsageInStreaming ? { stream_options: { include_usage: true } } : {}), @@ -853,6 +855,10 @@ export const fromRequest = Effect.fn("OpenAIChat.fromRequest")(function* ( } }) +// Provider wrappers that pass their own marker writer keep it; otherwise a declared cache format adds one. +const declaredCacheControl = (request: LLMRequest) => + request.model.compatibility?.cacheFormat === undefined ? undefined : cacheControl() + // ============================================================================= // Stream Parsing // ============================================================================= diff --git a/packages/ai/src/schema/options.ts b/packages/ai/src/schema/options.ts index e1ca60494132..4898b5d76a1d 100644 --- a/packages/ai/src/schema/options.ts +++ b/packages/ai/src/schema/options.ts @@ -201,6 +201,9 @@ export type LanguageModelMaxTokensFieldCompatibility = Schema.Schema.Type< typeof LanguageModelMaxTokensFieldCompatibility > +export const LanguageModelCacheFormatCompatibility = Schema.Literals(["anthropic", "alibaba"]) +export type LanguageModelCacheFormatCompatibility = Schema.Schema.Type + export class LanguageModelCompatibility extends Schema.Class( "LLM.LanguageModelCompatibility", )({ @@ -218,6 +221,8 @@ export class LanguageModelCompatibility extends Schema.Class { }), ) + it.effect("places cache breakpoints only when compatibility declares the format", () => + Effect.gen(function* () { + const prepare = (cacheFormat: "anthropic" | "alibaba" | undefined) => + compileRequest( + LLM.request({ + model: OpenAICompatibleChat.route + .with({ provider: "custom", endpoint: { baseURL: "https://api.custom.test/v1" } }) + .model({ id: "claude-sonnet-5", compatibility: cacheFormat === undefined ? undefined : { cacheFormat } }), + system: "You are concise.", + prompt: "Say hello.", + tools: [ToolDefinition.make({ name: "lookup", description: "Lookup data", inputSchema: {} })], + }), + ) + + const declared = yield* prepare("anthropic") + expect(declared.body).toMatchObject({ + messages: [ + { role: "system", content: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }] }, + { role: "user", content: [{ type: "text", text: "Say hello.", cache_control: { type: "ephemeral" } }] }, + ], + tools: [{ function: { name: "lookup" }, cache_control: { type: "ephemeral" } }], + }) + expect(JSON.stringify(declared.body)).not.toContain('"ttl"') + + const alibaba = yield* prepare("alibaba") + expect(alibaba.body).toMatchObject({ + messages: [ + { role: "system", content: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }] }, + { role: "user", content: [{ type: "text", text: "Say hello.", cache_control: { type: "ephemeral" } }] }, + ], + }) + expect(alibaba.body.tools?.[0]?.cache_control).toBeUndefined() + + expect(JSON.stringify((yield* prepare(undefined)).body)).not.toContain("cache_control") + }), + ) + it.effect("configures the max tokens request field", () => Effect.gen(function* () { const compatible = OpenAICompatibleChat.route diff --git a/packages/ai/test/provider/openrouter.test.ts b/packages/ai/test/provider/openrouter.test.ts index a4b446b17e7f..1c374d268aaa 100644 --- a/packages/ai/test/provider/openrouter.test.ts +++ b/packages/ai/test/provider/openrouter.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" -import { CacheHint, LLM, Message } from "../../src/index.js" +import { CacheHint, LanguageModel, LLM, Message } from "../../src/index.js" import { LLMClient } from "../../src/route.js" import { compileRequest } from "../../src/route/client.js" import * as OpenRouter from "../../src/providers/openrouter.js" @@ -127,6 +127,26 @@ describe("OpenRouter", () => { }), ) + it.effect("places breakpoints for any upstream when compatibility declares the cache format", () => + Effect.gen(function* () { + const model = OpenRouter.configure({ apiKey: "test-key" }).model("vendor/claude-proxy") + const prepared = yield* compileRequest( + LLM.request({ + model: LanguageModel.update(model, { compatibility: { cacheFormat: "anthropic" } }), + system: "Base agent", + tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object", properties: {} } }], + prompt: "Hello", + }), + ) + + expect(prepared.body.tools?.[0]?.cache_control).toEqual({ type: "ephemeral" }) + expect(prepared.body.messages).toMatchObject([ + { role: "system", content: [{ text: "Base agent", cache_control: { type: "ephemeral" } }] }, + { role: "user", content: [{ text: "Hello", cache_control: { type: "ephemeral" } }] }, + ]) + }), + ) + it.effect("sends no default breakpoints to upstreams that cache without them", () => Effect.gen(function* () { const openrouter = OpenRouter.configure({ apiKey: "test-key" }) diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index d98a03f48569..67cfbf5ea05c 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -166,6 +166,8 @@ export type ModelReasoningField = "reasoning" | "reasoning_content" | "reasoning export type ModelMaxTokensField = "max_completion_tokens" | "max_tokens" +export type ModelCacheFormat = "anthropic" | "alibaba" + export type ModelCapabilities = { tools: boolean; input: Array; output: Array } export type MoneyUSDPerMillionTokens = number @@ -632,6 +634,7 @@ export type ModelCompatibility = { requireFinishReason?: boolean requireAssistantAfterTool?: boolean supportsPromptCacheKey?: boolean + cacheFormat?: ModelCacheFormat supportsThinkingBlockBinding?: boolean supportsEffortUpdates?: boolean } @@ -643,6 +646,7 @@ export type ProviderCompatibility = { requireFinishReason?: boolean requireAssistantAfterTool?: boolean supportsPromptCacheKey?: boolean + cacheFormat?: ModelCacheFormat } export type ModelCost = { diff --git a/packages/core/src/plugin/provider/databricks.ts b/packages/core/src/plugin/provider/databricks.ts index 36c6be717ad1..74e739eb661f 100644 --- a/packages/core/src/plugin/provider/databricks.ts +++ b/packages/core/src/plugin/provider/databricks.ts @@ -8,5 +8,14 @@ export const DatabricksPlugin = define({ // models.dev also lists DATABRICKS_HOST, which only fills the base URL template. editor.method.update({ integrationID: "databricks", method: { type: "env", names: ["DATABRICKS_TOKEN"] } }) }) + yield* ctx.model.transform((models) => { + for (const item of models.list("databricks")) { + // Databricks caches Claude only through explicit `cache_control` breakpoints. + if (!(item.modelID ?? item.id).includes("claude")) continue + models.update(item.providerID, item.id, (model) => { + model.compatibility = { cacheFormat: "anthropic", ...model.compatibility } + }) + } + }) }), }) diff --git a/packages/core/src/plugin/provider/snowflake-cortex.ts b/packages/core/src/plugin/provider/snowflake-cortex.ts index 741f51a08c63..9f41a03cd6e6 100644 --- a/packages/core/src/plugin/provider/snowflake-cortex.ts +++ b/packages/core/src/plugin/provider/snowflake-cortex.ts @@ -197,6 +197,9 @@ export const SnowflakeCortexPlugin = define({ for (const item of models.list(providerID)) { models.update(providerID, item.id, (model) => { model.compatibility = { maxTokensField: "max_completion_tokens", ...model.compatibility } + // Cortex Chat Completions caches Claude only through explicit `cache_control` breakpoints. + if ((model.modelID ?? model.id).includes("claude")) + model.compatibility = { cacheFormat: "anthropic", ...model.compatibility } if (model.settings?.baseURL !== undefined) model.settings.baseURL = endpoint(model.settings.baseURL, account.value) }) diff --git a/packages/core/test/config/provider.test.ts b/packages/core/test/config/provider.test.ts index 51ce7b262e6e..debe30cc101d 100644 --- a/packages/core/test/config/provider.test.ts +++ b/packages/core/test/config/provider.test.ts @@ -170,6 +170,31 @@ describe("ConfigProviderPlugin.Plugin", () => { }), ) + it.effect("carries a configured provider cache format to the runtime model", () => + Effect.gen(function* () { + const models = yield* Model.Service + yield* addPlugin([ + new Document({ + type: "document", + info: decode({ + providers: { + litellm: { + package: "@opencode/ai/providers/openai-compatible", + settings: { baseURL: "https://litellm.test/v1" }, + compatibility: { cacheFormat: "anthropic" }, + models: { "claude-sonnet-5": {} }, + }, + }, + }), + }), + ]) + + const model = required(yield* models.get(Provider.ID.make("litellm"), Model.ID.make("claude-sonnet-5"))) + expect(model.compatibility).toEqual({ cacheFormat: "anthropic" }) + expect((yield* ModelResolver.fromCatalogModel(model)).compatibility?.cacheFormat).toBe("anthropic") + }), + ) + it.effect("applies provider compatibility to every provider model with model overrides", () => Effect.gen(function* () { const providers = yield* Provider.Service diff --git a/packages/core/test/plugin/provider-snowflake-cortex.test.ts b/packages/core/test/plugin/provider-snowflake-cortex.test.ts index 36959af196f0..76b14c308b42 100644 --- a/packages/core/test/plugin/provider-snowflake-cortex.test.ts +++ b/packages/core/test/plugin/provider-snowflake-cortex.test.ts @@ -192,6 +192,8 @@ it.live("manual PAT uses its account and native compatibility rather than an SDK const body = yield* Effect.promise(() => request.json()) expect(body).toMatchObject({ model: modelID, max_completion_tokens: 1024 }) expect(body).not.toHaveProperty("max_tokens") + // Cortex caches Claude only through explicit Anthropic-style breakpoints. + expect(JSON.stringify(body)).toContain('"cache_control":{"type":"ephemeral"}') }), ), ) diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index 51b36845afaa..8c9bda790fb0 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -16600,6 +16600,10 @@ "required": ["_tag", "sessionID", "messageID", "message"], "additionalProperties": false }, + "Model.CacheFormat": { + "type": "string", + "enum": ["anthropic", "alibaba"] + }, "Model.Capabilities": { "type": "object", "properties": { @@ -16643,6 +16647,9 @@ "supportsPromptCacheKey": { "type": "boolean" }, + "cacheFormat": { + "$ref": "#/components/schemas/Model.CacheFormat" + }, "supportsThinkingBlockBinding": { "type": "boolean" }, @@ -17741,6 +17748,9 @@ }, "supportsPromptCacheKey": { "type": "boolean" + }, + "cacheFormat": { + "$ref": "#/components/schemas/Model.CacheFormat" } }, "additionalProperties": false diff --git a/packages/schema/src/model-compatibility.ts b/packages/schema/src/model-compatibility.ts index 1caca7b3d53e..92a09006d9e6 100644 --- a/packages/schema/src/model-compatibility.ts +++ b/packages/schema/src/model-compatibility.ts @@ -13,6 +13,9 @@ export const MaxTokensField = Schema.Literals(["max_completion_tokens", "max_tok }) export type MaxTokensField = typeof MaxTokensField.Type +export const CacheFormat = Schema.Literals(["anthropic", "alibaba"]).annotate({ identifier: "Model.CacheFormat" }) +export type CacheFormat = typeof CacheFormat.Type + export interface Compatibility extends Schema.Schema.Type {} export const Compatibility = Schema.Struct({ reasoningField: ReasoningField.pipe(optional), @@ -22,6 +25,8 @@ export const Compatibility = Schema.Struct({ requireFinishReason: Schema.Boolean.pipe(optional), requireAssistantAfterTool: Schema.Boolean.pipe(optional), supportsPromptCacheKey: Schema.Boolean.pipe(optional), + /** Prompt-cache marker syntax the endpoint accepts. Overrides the route default; both send `cache_control`, and `alibaba` skips tool definitions. */ + cacheFormat: CacheFormat.pipe(optional), /** Supports Anthropic's thinking-prefix mismatch controls. Overrides model-ID detection. */ supportsThinkingBlockBinding: Schema.Boolean.pipe(optional), /** Supports per-message effort updates. Overrides model-ID detection. */ diff --git a/services/www/openapi.json b/services/www/openapi.json index 51b36845afaa..8c9bda790fb0 100644 --- a/services/www/openapi.json +++ b/services/www/openapi.json @@ -16600,6 +16600,10 @@ "required": ["_tag", "sessionID", "messageID", "message"], "additionalProperties": false }, + "Model.CacheFormat": { + "type": "string", + "enum": ["anthropic", "alibaba"] + }, "Model.Capabilities": { "type": "object", "properties": { @@ -16643,6 +16647,9 @@ "supportsPromptCacheKey": { "type": "boolean" }, + "cacheFormat": { + "$ref": "#/components/schemas/Model.CacheFormat" + }, "supportsThinkingBlockBinding": { "type": "boolean" }, @@ -17741,6 +17748,9 @@ }, "supportsPromptCacheKey": { "type": "boolean" + }, + "cacheFormat": { + "$ref": "#/components/schemas/Model.CacheFormat" } }, "additionalProperties": false diff --git a/services/www/public/openapi.json b/services/www/public/openapi.json index 51b36845afaa..8c9bda790fb0 100644 --- a/services/www/public/openapi.json +++ b/services/www/public/openapi.json @@ -16600,6 +16600,10 @@ "required": ["_tag", "sessionID", "messageID", "message"], "additionalProperties": false }, + "Model.CacheFormat": { + "type": "string", + "enum": ["anthropic", "alibaba"] + }, "Model.Capabilities": { "type": "object", "properties": { @@ -16643,6 +16647,9 @@ "supportsPromptCacheKey": { "type": "boolean" }, + "cacheFormat": { + "$ref": "#/components/schemas/Model.CacheFormat" + }, "supportsThinkingBlockBinding": { "type": "boolean" }, @@ -17741,6 +17748,9 @@ }, "supportsPromptCacheKey": { "type": "boolean" + }, + "cacheFormat": { + "$ref": "#/components/schemas/Model.CacheFormat" } }, "additionalProperties": false diff --git a/services/www/src/docs/content/models.mdx b/services/www/src/docs/content/models.mdx index 003ef6df1b25..d244c5e87018 100644 --- a/services/www/src/docs/content/models.mdx +++ b/services/www/src/docs/content/models.mdx @@ -226,6 +226,38 @@ OpenCode recognizes `reasoning`, `reasoning_content`, and `reasoning_text`, and string. It reads streamed reasoning from that field and restores the field when sending prior assistant messages back to the model. +## Prompt caching + +OpenCode places prompt-cache breakpoints automatically on routes that are known to need them, such as Anthropic. When +Claude models are served through an OpenAI-compatible proxy such as LiteLLM, OpenCode cannot tell that they need +Anthropic-style `cache_control` breakpoints. Set `compatibility.cacheFormat` on those Claude models: + +```jsonc title="opencode.jsonc" +{ + "providers": { + "litellm": { + "package": "@opencode/ai/providers/openai-compatible", + "settings": { "baseURL": "https://litellm.example.com/v1" }, + "models": { + "claude-sonnet-5": { + "compatibility": { "cacheFormat": "anthropic" }, + }, + "claude-opus-5-5": { + "compatibility": { "cacheFormat": "anthropic" }, + }, + // Other models on the same proxy cache automatically and need no setting. + "gpt-6.1": {}, + }, + }, + }, +} +``` + +OpenCode then marks the last tool definition, the system prompt, and the latest message with +`cache_control: { "type": "ephemeral" }`, which uses the endpoint's default five-minute cache lifetime. Use `"alibaba"` +for Alibaba Model Studio models, which accept the same markers but ignore them on tool definitions. Set +`compatibility.cacheFormat` on the provider instead when every model it serves accepts the same format. + ## Local OpenCode automatically discovers models from Ollama, LM Studio, and vLLM at their default local addresses. You can also