From 5a4b4130b25bd609c954c47eba8c1fac797f322e Mon Sep 17 00:00:00 2001 From: Alessandro <155005371+3clyp50@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:30:09 +0200 Subject: [PATCH] Use native LiteLLM routes for core providers Route Nebius, Venice chat, Z.AI/Coding and OpenRouter embeddings through LiteLLM's native integrations. Retain the Token Factory and Coding endpoint overrides, OpenRouter attribution, and Venice/proxy embedding fallbacks. Keep Cerebras and Z.AI model-list URLs in listing metadata while allowing LiteLLM to supply inference defaults and Copilot request headers. Merge YAML-declared extra_body defaults, legacy top-level values and explicit nested overrides in the shared provider-default loader. This preserves saved Venice settings without provider-specific branches in the model-config plugin. Validation: 114 focused SDK, API-key, onboarding, model-search and preset tests passed in the port-32081 framework runtime with LiteLLM 1.104.0. Native routes were checked over loopback HTTP without paid provider calls; the restarted service returned HTTP 200 and loaded the updated provider definitions. --- AGENTS.md | 1 + conf/AGENTS.md | 2 + conf/model_providers.yaml | 26 +++---- models.py | 10 ++- tests/test_litellm_sdk.py | 115 +++++++++++++++++++++++++++- tests/test_model_config_api_keys.py | 23 +++--- tests/test_onboarding_static.py | 4 +- 7 files changed, 147 insertions(+), 34 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f0987dae5..5f66af9f0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,7 @@ - Native function calls use canonical call content for history and combine public response text with calls for repeat comparison. The canonical history content passes through the normal history template/masking hook; provider output metadata remains intact. - `initialize.py` owns framework initialization. - `models.py` owns model-provider configuration and LiteLLM integration. `get_api_key_raw()` reads the stored key or key list for settings/editing; `get_api_key()` selects a runtime key and runs provider extension hooks. +- `models._merge_provider_defaults` merges YAML-declared `extra_body` defaults with legacy top-level values for those declared fields and explicit nested overrides, in that order; unrelated kwargs and original-provider API-key selection remain intact. - `models.apply_rate_limiter` skips token counting and returns no limiter when all request, input, and output limits are unset; any configured limit retains normal accounting. - `requirements.txt` includes HTTPX's SOCKS extra so framework clients support SOCKS proxy environment variables before plugin setup. - LiteLLM and OpenAI pins must remain compatible. Retain shared dependency security floors and verify upgrades with installed-SDK HTTP tests in `tests/test_litellm_sdk.py` using the framework runtime. diff --git a/conf/AGENTS.md b/conf/AGENTS.md index e658a9876..285b67d63 100644 --- a/conf/AGENTS.md +++ b/conf/AGENTS.md @@ -18,6 +18,8 @@ - Defaults must work in a clean checkout and in Docker. - Chat Completions is the transport default; providers intentionally using Responses must set `a0_api_mode: responses` explicitly. - Local provider defaults omit `api_key`: LiteLLM handles unauthenticated endpoints, while saved or explicit keys remain available for authenticated servers. Static dummy keys would override saved keys. OAuth compatibility keys belong to the OAuth plugin's connected-account hook. +- Prefer native LiteLLM routes for shipped providers; keep endpoint overrides for Nebius Token Factory and Z.AI Coding. Model-list `default_base` is independent of LiteLLM's inference defaults. +- Venice chat uses `veniceai` with `venice_parameters` in `extra_body`; Venice embeddings and the Agent Zero proxy retain their OpenAI-compatible routes. OpenRouter embeddings use the native route and retain Agent Zero attribution headers. - Templates must avoid accidentally unignoring private runtime content. ## Work Guidance diff --git a/conf/model_providers.yaml b/conf/model_providers.yaml index d3c6d5d64..56203c418 100644 --- a/conf/model_providers.yaml +++ b/conf/model_providers.yaml @@ -50,9 +50,9 @@ chat: litellm_provider: cerebras models_list: endpoint_url: "/models" + default_base: https://api.cerebras.ai/v1 kwargs: a0_api_mode: chat - api_base: https://api.cerebras.ai/v1 cometapi: name: CometAPI litellm_provider: cometapi @@ -72,10 +72,6 @@ chat: litellm_provider: github_copilot kwargs: a0_api_mode: chat - extra_headers: - "Editor-Version": "vscode/1.85.1" - "Copilot-Integration-Id": "vscode-chat" - "Copilot-Vision-Request": "true" google: name: Google litellm_provider: gemini @@ -133,7 +129,7 @@ chat: a0_api_mode: chat nebius: name: Nebius Token Factory - litellm_provider: openai + litellm_provider: nebius models_list: endpoint_url: "/models" kwargs: @@ -214,14 +210,14 @@ chat: a0_api_mode: chat venice: name: Venice.ai - litellm_provider: openai + litellm_provider: veniceai models_list: endpoint_url: "https://api.venice.ai/api/v1/models" kwargs: a0_api_mode: chat - api_base: https://api.venice.ai/api/v1 - venice_parameters: - include_venice_system_prompt: false + extra_body: + venice_parameters: + include_venice_system_prompt: false vllm: name: vLLM litellm_provider: hosted_vllm @@ -240,15 +236,15 @@ chat: a0_api_mode: chat zai: name: Z.AI - litellm_provider: openai + litellm_provider: zai models_list: endpoint_url: "/models" + default_base: https://api.z.ai/api/paas/v4 kwargs: a0_api_mode: chat - api_base: https://api.z.ai/api/paas/v4 zai_coding: name: Z.AI Coding - litellm_provider: openai + litellm_provider: zai models_list: endpoint_url: "/models" kwargs: @@ -304,12 +300,10 @@ embedding: bedrock: name: AWS Bedrock litellm_provider: bedrock - # TODO: OpenRouter not yet supported by LiteLLM, replace with native litellm_provider openrouter and remove api_base when ready openrouter: name: OpenRouter - litellm_provider: openai + litellm_provider: openrouter kwargs: - api_base: https://openrouter.ai/api/v1 extra_headers: "HTTP-Referer": "https://agent-zero.ai/" "X-Title": "Agent Zero" diff --git a/models.py b/models.py index b43a02f7c..9b3cf3ecd 100644 --- a/models.py +++ b/models.py @@ -975,7 +975,15 @@ def _merge_provider_defaults( extra_kwargs = cfg.get("kwargs") if isinstance(cfg, dict) else None # type: ignore[arg-type] if isinstance(extra_kwargs, dict): for k, v in extra_kwargs.items(): - kwargs.setdefault(k, v) + if k == "extra_body" and isinstance(v, dict): + body = dict(v) + for key in v: + if key in kwargs: + body[key] = kwargs.pop(key) + body.update(kwargs.get(k) or {}) + kwargs[k] = body + else: + kwargs.setdefault(k, v) # Inject API key based on the *original* provider id if still missing if "api_key" not in kwargs: diff --git a/tests/test_litellm_sdk.py b/tests/test_litellm_sdk.py index 0ea5c4937..3336d9b97 100644 --- a/tests/test_litellm_sdk.py +++ b/tests/test_litellm_sdk.py @@ -27,12 +27,13 @@ USAGE = {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} @pytest.fixture def provider(): - state = SimpleNamespace(replies=deque(), requests=[]) + state = SimpleNamespace(replies=deque(), requests=[], headers=[]) class Handler(BaseHTTPRequestHandler): def do_POST(self): body = self.rfile.read(int(self.headers.get("Content-Length", 0))) state.requests.append((self.command, self.path, json.loads(body) if body else {})) + state.headers.append(dict(self.headers)) status, reply = state.replies.popleft() if state.replies else (500, {"error": "Unexpected request"}) streaming = isinstance(reply, str) payload = (reply if streaming else json.dumps(reply)).encode() @@ -200,3 +201,115 @@ def test_sdk_anthropic_cache_markers_over_http(provider): body = provider.requests[0][2] assert body["system"][-1]["cache_control"] == {"type": "ephemeral"} assert body["messages"][1]["content"][-1]["cache_control"] == {"type": "ephemeral"} + + +@pytest.mark.parametrize("provider_id,name,route,base", [ + ("cerebras", "gpt-oss-120b", "cerebras", "https://api.cerebras.ai/v1"), + ("nebius", "meta-llama/Llama-3.3-70B-Instruct", "nebius", "https://api.tokenfactory.nebius.com/v1"), + ("zai", "glm-4.6", "zai", "https://api.z.ai/api/paas/v4"), + ("zai_coding", "glm-4.6", "zai", "https://api.z.ai/api/coding/paas/v4"), + ("venice", "llama-3.3-70b", "veniceai", "https://api.venice.ai/api/v1"), +]) +def test_native_provider_defaults_over_http(provider, monkeypatch, provider_id, name, route, base): + from pathlib import Path + import yaml + import litellm + from plugins._model_config.api.model_search import ModelSearch + from plugins._model_config.helpers import model_config + + definitions = yaml.safe_load((Path(__file__).resolve().parents[1] / "conf/model_providers.yaml").read_text()) + lookup = lambda kind, pid: definitions[kind].get(pid) + monkeypatch.setattr(models, "get_provider_config", lookup) + monkeypatch.setattr(models, "get_api_key", lambda pid: f"test-{pid}") + mc = model_config.build_model_config({"provider": provider_id, "name": name}, models.ModelType.CHAT) + wrapper = models.get_chat_model(provider_id, name, model_config=mc, **mc.build_kwargs()) + assert wrapper.model_name == f"{route}/{name}" + assert wrapper.kwargs["api_key"] == f"test-{provider_id}" + assert litellm.get_llm_provider(model=wrapper.model_name, api_key=wrapper.kwargs["api_key"], + api_base=wrapper.kwargs.get("api_base"))[3] == base + cfg = definitions["chat"][provider_id] + assert ModelSearch._resolve_url(cfg["models_list"], wrapper.kwargs.get("api_base", ""))[0] == f"{base}/models" + + provider.replies.append((200, chat_reply(False))) + transport = LiteLLMTransport(model=wrapper.model_name, messages=[{"role": "user", "content": "Hello"}], + kwargs={**wrapper.kwargs, **provider.kwargs}) + transport.complete() + _, path, body = provider.requests[0] + assert path == "/v1/chat/completions" and body["model"] == name + if provider_id == "venice": + assert body["venice_parameters"] == {"include_venice_system_prompt": False} + + +@pytest.mark.parametrize("kwargs,expected", [ + ({"extra_body": {"custom": 1}}, False), + ({"venice_parameters": {"include_venice_system_prompt": True}, "extra_body": {"custom": 1}}, True), + ({"venice_parameters": {"include_venice_system_prompt": False}, + "extra_body": {"custom": 1, "venice_parameters": {"include_venice_system_prompt": True}}}, True), +]) +def test_saved_venice_parameters_over_http(provider, monkeypatch, kwargs, expected): + from copy import deepcopy + from plugins._model_config.helpers import model_config + + monkeypatch.setattr(models, "get_api_key", lambda pid: "test-key") + original = deepcopy(kwargs) + cfg = {"provider": "venice", "name": "llama-3.3-70b", "kwargs": kwargs} + mc = model_config.build_model_config(cfg, models.ModelType.CHAT) + wrapper = models.get_chat_model(mc.provider, mc.name, **mc.build_kwargs()) + provider.replies.append((200, chat_reply(False))) + LiteLLMTransport(model=wrapper.model_name, messages=[{"role": "user", "content": "Hello"}], + kwargs={**wrapper.kwargs, **provider.kwargs}).complete() + body = provider.requests[0][2] + assert body["venice_parameters"] == {"include_venice_system_prompt": expected} + assert body["custom"] == 1 + assert kwargs == original + for pid, kind in [("a0_venice", models.ModelType.CHAT), ("venice", models.ModelType.EMBEDDING)]: + assert model_config.build_model_config({**cfg, "provider": pid}, kind).kwargs == original + + +@pytest.mark.parametrize("name", ["google/gemini-embedding-001", "openai/text-embedding-3-small", + "nvidia/llama-nemotron-embed-vl-1b-v2:free"]) +def test_native_openrouter_embeddings_over_http(provider, monkeypatch, name): + monkeypatch.setattr(models, "get_api_key", lambda pid: f"test-{pid}") + wrapper = models.get_embedding_model("openrouter", name, **provider.kwargs) + assert wrapper.model_name == f"openrouter/{name}" + provider.replies.append((200, {"object": "list", "model": name, + "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}], + "usage": {"prompt_tokens": 1, "total_tokens": 1}})) + assert wrapper.embed(["hello"]) == [[1.0, 0.0]] + assert provider.requests[0][1] == "/v1/embeddings" + assert provider.requests[0][2]["model"] == name + headers = {key.lower(): value for key, value in provider.headers[0].items()} + assert headers["http-referer"] == "https://agent-zero.ai/" + assert headers["x-title"] == "Agent Zero" + assert headers["x-openrouter-categories"] == "personal-agent,cloud-agent" + + +def test_native_copilot_supplies_headers_and_detects_vision(monkeypatch): + from litellm.llms.github_copilot.chat.transformation import GithubCopilotConfig + from litellm.llms.github_copilot.authenticator import Authenticator + + monkeypatch.setattr(Authenticator, "get_api_key", lambda self: "test-key") + config = GithubCopilotConfig() + for content, vision in [("Hello", False), ([{"type": "image_url", "image_url": {"url": "https://example.invalid/image.png"}}], True)]: + headers = config.validate_environment(headers={}, model="gpt-4o", + messages=[{"role": "user", "content": content}], optional_params={}, litellm_params={}, api_key="test-key") + headers = {key.lower(): value for key, value in headers.items()} + assert headers["copilot-integration-id"] == "vscode-chat" + assert headers["editor-version"].startswith("vscode/") + assert (headers.get("copilot-vision-request") == "true") == vision + + +def test_extra_body_defaults_merge_without_provider_specific_rules(monkeypatch): + defaults = {"litellm_provider": "openai", "kwargs": { + "extra_body": {"vendor_options": {"enabled": False}, "default_only": 1}}} + monkeypatch.setattr(models, "get_provider_config", lambda kind, pid: defaults) + monkeypatch.setattr(models, "get_api_key", lambda pid: "None") + route, kwargs = models._merge_provider_defaults("chat", "test-provider", { + "vendor_options": {"enabled": True}, "temperature": 0.2, + "extra_body": {"vendor_options": {"enabled": False}, "custom": 2}}) + assert route == "openai" + assert kwargs == {"temperature": 0.2, "extra_body": { + "vendor_options": {"enabled": False}, "default_only": 1, "custom": 2}} + assert defaults["kwargs"]["extra_body"] == {"vendor_options": {"enabled": False}, "default_only": 1} + _, legacy = models._merge_provider_defaults("chat", "test-provider", {"vendor_options": {"enabled": True}}) + assert legacy["extra_body"]["vendor_options"] == {"enabled": True} diff --git a/tests/test_model_config_api_keys.py b/tests/test_model_config_api_keys.py index b432ed445..2af5959f4 100644 --- a/tests/test_model_config_api_keys.py +++ b/tests/test_model_config_api_keys.py @@ -372,16 +372,14 @@ def test_cerebras_provider_uses_chat_completions_and_live_model_catalog(monkeypa assert cerebras["name"] == "Cerebras" assert cerebras["litellm_provider"] == "cerebras" assert cerebras["models_list"]["endpoint_url"] == "/models" - assert cerebras["kwargs"] == { - "a0_api_mode": "chat", - "api_base": "https://api.cerebras.ai/v1", - } + assert cerebras["kwargs"] == {"a0_api_mode": "chat"} + assert cerebras["models_list"]["default_base"] == "https://api.cerebras.ai/v1" assert model_config.provider_requires_api_key("cerebras") is True model = models.get_chat_model("cerebras", "gpt-oss-120b") assert model.model_name == "cerebras/gpt-oss-120b" assert model.kwargs["a0_api_mode"] == "chat" - assert model.kwargs["api_base"] == "https://api.cerebras.ai/v1" + assert "api_base" not in model.kwargs assert model.kwargs["api_key"] == "test-key" @@ -395,8 +393,9 @@ def test_direct_venice_chat_provider_defaults_to_chat_completions(monkeypatch): venice = provider_config["chat"]["venice"] assert venice["kwargs"]["a0_api_mode"] == "chat" - assert venice["kwargs"]["api_base"] == "https://api.venice.ai/api/v1" - assert venice["kwargs"]["venice_parameters"] == { + assert venice["litellm_provider"] == "veniceai" + assert "api_base" not in venice["kwargs"] + assert venice["kwargs"]["extra_body"]["venice_parameters"] == { "include_venice_system_prompt": False } assert provider_config["chat"]["a0_venice"]["kwargs"]["a0_api_mode"] == "chat" @@ -812,18 +811,14 @@ def test_openai_compatible_embedding_keeps_gateway_model_string(monkeypatch): assert embedding.model_name == f"openai/{model}" assert embedding.kwargs["api_base"] == gateway - # The bundled OpenRouter embedding provider is affected the same way: it - # resolves to litellm_provider `openai` against OpenRouter's api_base, so an - # OpenRouter-style id has to survive intact. Previously `openai/` was - # handed to LiteLLM bare, which consumed the `openai/` segment and forwarded - # only `` to OpenRouter. + # The native OpenRouter route must also preserve the upstream provider/model ID. for model in ( "openai/text-embedding-3-small", "nvidia/llama-nemotron-embed-vl-1b-v2:free", ): openrouter_embedding = models.get_embedding_model("openrouter", model) - assert openrouter_embedding.model_name == f"openai/{model}" - assert openrouter_embedding.kwargs["api_base"] == "https://openrouter.ai/api/v1" + assert openrouter_embedding.model_name == f"openrouter/{model}" + assert "api_base" not in openrouter_embedding.kwargs # Plain OpenAI behaviour is unchanged: LiteLLM strips the `openai/` prefix # and forwards the bare model id, exactly as it did without a prefix. diff --git a/tests/test_onboarding_static.py b/tests/test_onboarding_static.py index 226062987..b748df463 100644 --- a/tests/test_onboarding_static.py +++ b/tests/test_onboarding_static.py @@ -195,13 +195,13 @@ def test_onboarding_provider_grid_names_are_present_in_metadata(): assert (PROJECT_ROOT / "plugins/_onboarding/webui/assets/provider-logos/vllm.svg").exists() -def test_nebius_provider_config_uses_openai_compatible_token_factory_endpoint(): +def test_nebius_provider_config_uses_native_route_with_token_factory_endpoint(): provider_path = PROJECT_ROOT / "conf/model_providers.yaml" provider_config = yaml.safe_load(provider_path.read_text(encoding="utf-8")) nebius = provider_config["chat"]["nebius"] assert nebius["name"] == "Nebius Token Factory" - assert nebius["litellm_provider"] == "openai" + assert nebius["litellm_provider"] == "nebius" assert nebius["kwargs"]["api_base"] == "https://api.tokenfactory.nebius.com/v1" assert nebius["models_list"]["endpoint_url"] == "/models" assert "api_key_mode" not in nebius