mirror of
https://github.com/agent0ai/agent-zero.git
synced 2026-10-14 22:07:00 +00:00
Skip rate-limiter token counting without limits
apply_rate_limiter tokenized the full serialized prompt on every call even when no request, input or output limit was configured. Return early in that case; callers already skip output accounting when no limiter is returned. Add a regression for missing config, disabled limits, and each enabled limit; document the accounting contract. Validation: 115 rate-limiter, streaming, Responses and embedding tests passed with this change alone on the original base in the Docker framework runtime. The disabled-limits regression fails on the old code. Earlier live Codex measurements with a 760 KB prompt showed about 70 ms dropping to 0 ms in apply_rate_limiter.
This commit is contained in:
parent
ade204c7e3
commit
827f9176d2
3 changed files with 37 additions and 1 deletions
|
|
@ -21,6 +21,7 @@
|
||||||
- Native function calls use canonical call content for history and repeat comparison, even when accompanied by commentary. The canonical content passes through the normal history template/masking hook; provider output metadata remains intact.
|
- Native function calls use canonical call content for history and repeat comparison, even when accompanied by commentary. The canonical content passes through the normal history template/masking hook; provider output metadata remains intact.
|
||||||
- `initialize.py` owns framework initialization.
|
- `initialize.py` owns framework initialization.
|
||||||
- `models.py` owns model-provider configuration and LiteLLM integration. `get_api_key_raw()` reads the stored key or key list for settings/editing; `get_api_key()` selects a runtime key and runs provider extension hooks.
|
- `models.py` owns model-provider configuration and LiteLLM integration. `get_api_key_raw()` reads the stored key or key list for settings/editing; `get_api_key()` selects a runtime key and runs provider extension hooks.
|
||||||
|
- `models.apply_rate_limiter` skips token counting and returns no limiter when all request, input, and output limits are unset; any configured limit retains normal accounting.
|
||||||
- `requirements.txt` includes HTTPX's SOCKS extra so framework clients support SOCKS proxy environment variables before plugin setup.
|
- `requirements.txt` includes HTTPX's SOCKS extra so framework clients support SOCKS proxy environment variables before plugin setup.
|
||||||
- Native agent-turn response callbacks may receive a display-only commentary preview as their full text; legacy callbacks, accumulated model output, usage and native dispatch retain the original text and result metadata.
|
- Native agent-turn response callbacks may receive a display-only commentary preview as their full text; legacy callbacks, accumulated model output, usage and native dispatch retain the original text and result metadata.
|
||||||
- `run_ui.py` is the WebUI entry point.
|
- `run_ui.py` is the WebUI entry point.
|
||||||
|
|
|
||||||
|
|
@ -336,7 +336,10 @@ async def apply_rate_limiter(
|
||||||
Callable[[str, str, int, int], Awaitable[bool]] | None
|
Callable[[str, str, int, int], Awaitable[bool]] | None
|
||||||
) = None,
|
) = None,
|
||||||
):
|
):
|
||||||
if not model_config:
|
# skip token counting of the whole prompt when no limit is set
|
||||||
|
if not model_config or not (
|
||||||
|
model_config.limit_requests or model_config.limit_input or model_config.limit_output
|
||||||
|
):
|
||||||
return
|
return
|
||||||
limiter = get_rate_limiter(
|
limiter = get_rate_limiter(
|
||||||
model_config.provider,
|
model_config.provider,
|
||||||
|
|
|
||||||
32
tests/test_model_rate_limiter.py
Normal file
32
tests/test_model_rate_limiter.py
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||||
|
import models
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("limit", [None, "none", "requests", "input", "output"])
|
||||||
|
async def test_rate_limiter_counts_tokens_only_when_limits_are_configured(monkeypatch, limit):
|
||||||
|
monkeypatch.setattr(models, "rate_limiters", {})
|
||||||
|
counted = []
|
||||||
|
monkeypatch.setattr(models, "approximate_tokens", lambda text: counted.append(text) or 3)
|
||||||
|
config = models.ModelConfig(
|
||||||
|
type=models.ModelType.CHAT, provider="openai", name="test",
|
||||||
|
) if limit is not None else None
|
||||||
|
if limit in ("requests", "input", "output"):
|
||||||
|
setattr(config, f"limit_{limit}", 10)
|
||||||
|
|
||||||
|
limiter = await models.apply_rate_limiter(config, "prompt")
|
||||||
|
|
||||||
|
if limit in (None, "none"):
|
||||||
|
assert limiter is None
|
||||||
|
assert counted == []
|
||||||
|
assert models.rate_limiters == {}
|
||||||
|
else:
|
||||||
|
assert counted == ["prompt"]
|
||||||
|
assert limiter.limits[limit] == 10
|
||||||
|
assert await limiter.get_total("input") == 3
|
||||||
|
assert await limiter.get_total("requests") == 1
|
||||||
Loading…
Add table
Add a link
Reference in a new issue