mirror of
https://github.com/Alishahryar1/free-claude-code.git
synced 2026-10-19 07:23:36 +00:00
4e99ddaa5c
Some checks are pending
## Why
Fallback rows could choose a model and reasoning effort but always used
the provider's first API key. Users need to choose which numbered key
each fallback uses, including different keys for the same model.
## How
Add a Key N selector and persist a validated `api_key_number`,
defaulting existing entries to Key 1. Reject duplicate model/key pairs
and key removals that leave a saved fallback's selection unavailable.
Carry the number through routing and the request's retained runtime
generation. Cache and construct clients per provider/key pair so
concurrent attempts keep their selected credentials.
Share rate and concurrency capacity across a provider's keys, while
recovery belongs to the actual credential. A healthy selected fallback
key receives an attempt after another key exhausts rate-limit retries.
Reordering a key preserves its recovery state, and replacing it uses a
separate state. Keep the existing retry classification, attempt count,
and backoff.
Refresh key choices after provider edits while preserving unsaved model
drafts. Display removed numbers as unavailable and reset the choice to
Key 1 when a row changes provider. Numbers follow the current order in
the provider's settings. Providers with another authentication method
show a disabled default selection.
The Admin editor targets desktop viewports, as requested. Its existing
rows and add row keep the three selections aligned horizontally.
Local CI passes, including 9,049 Python tests and 277 browser tests.
Added regressions capture actual authentication headers after 403 and
exhausted 429 failures across both public APIs and all three custom
endpoint formats, and check shared capacity, client coalescing, retained
generations, credential reorder/replacement, and Admin edits.
<!-- greptile_comment -->
<!-- greptile_summary -->
<h2><a
href="https://app.greptile.com/api/retrigger?id=79105240"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://greptile-static-assets.s3.amazonaws.com/badges/RetriggerDark.svg?v=2"><source
media="(prefers-color-scheme: light)"
srcset="https://greptile-static-assets.s3.amazonaws.com/badges/Retrigger.svg?v=2"><img
alt="Retrigger"
src="https://greptile-static-assets.s3.amazonaws.com/badges/Retrigger.svg?v=2"
align="right"></picture></a>Confidence Score: 5/5</h2>
<!-- greptile-impact-verdict --> No outstanding blocking findings were
identified. The reviewed changes are safe to merge.
<details open><summary><h3>Summary</h3></summary>
Adds a numbered API-key choice to each fallback model and carries that
choice through routing and provider construction.
- Fallback rows can choose a numbered API key.
- Fallback attempts send requests with their selected API keys.
- Provider clients keep separate keys while sharing provider limits.
</details>
<!-- greptile_confidence_score:5 -->
<sub>Reviews (3) · Last reviewed commit: ["Keep credential recovery
separate from
p..."](c7c33ac61b)
· [Reviewed by
Greptile](https://www.greptile.com/?utm_source=greptile_expert&utm_medium=github&utm_campaign=code_reviews&utm_content=alishahryar1/free-claude-code)</sub>
<!-- /greptile_comment -->
1331 lines
44 KiB
Python
1331 lines
44 KiB
Python
"""Application-owned provider execution contracts."""
|
|
|
|
import asyncio
|
|
from collections.abc import AsyncIterator, Mapping
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from free_claude_code.application.errors import InvalidRequestError
|
|
from free_claude_code.application.execution import ProviderExecutor
|
|
from free_claude_code.application.model_metadata import ProviderModelInfo
|
|
from free_claude_code.application.routing import (
|
|
ProviderModelCandidate,
|
|
ProviderModelTarget,
|
|
ResolvedModelRoute,
|
|
RoutedMessagesRequest,
|
|
RoutedResponsesRequest,
|
|
)
|
|
from free_claude_code.config.reasoning import ReasoningPreference
|
|
from free_claude_code.core.anthropic.models import Message, MessagesRequest
|
|
from free_claude_code.core.anthropic.passthrough import NativeMessagesRequest
|
|
from free_claude_code.core.async_iterators import AsyncCloseable
|
|
from free_claude_code.core.failures import ExecutionFailure, FailureKind
|
|
from free_claude_code.core.openai_responses import OpenAIResponsesRequest
|
|
from free_claude_code.core.reasoning import ReasoningCapability, ReasoningPolicy
|
|
from free_claude_code.core.stream_recovery import ContinuationSeed
|
|
|
|
|
|
class FakeProvider:
|
|
def __init__(self) -> None:
|
|
self.stream_calls: list[dict[str, object]] = []
|
|
self.stream_close_calls = 0
|
|
|
|
def stream_native_messages(
|
|
self,
|
|
request: NativeMessagesRequest,
|
|
*,
|
|
request_id: str,
|
|
response_model: str,
|
|
request_headers: Mapping[str, str] | None = None,
|
|
reasoning_override: ReasoningPolicy | None = None,
|
|
continuation: ContinuationSeed | None = None,
|
|
) -> AsyncIterator[str]:
|
|
raise AssertionError("Compatibility test provider received a native request")
|
|
|
|
async def stream_messages(
|
|
self,
|
|
request: MessagesRequest,
|
|
input_tokens: int = 0,
|
|
*,
|
|
request_id: str | None = None,
|
|
response_model: str | None = None,
|
|
reasoning: ReasoningPolicy,
|
|
request_headers: Mapping[str, str] | None = None,
|
|
model_info: ProviderModelInfo | None = None,
|
|
continuation: ContinuationSeed | None = None,
|
|
) -> AsyncIterator[str]:
|
|
self._record_stream_call(
|
|
request,
|
|
input_tokens=input_tokens,
|
|
request_id=request_id,
|
|
response_model=response_model,
|
|
reasoning=reasoning,
|
|
)
|
|
try:
|
|
yield "event: message_stop\ndata: {}\n\n"
|
|
finally:
|
|
self.stream_close_calls += 1
|
|
|
|
async def stream_responses(
|
|
self,
|
|
request: OpenAIResponsesRequest,
|
|
*,
|
|
input_tokens: int,
|
|
request_id: str,
|
|
response_model: str,
|
|
reasoning: ReasoningPolicy,
|
|
request_headers: Mapping[str, str] | None = None,
|
|
model_info: ProviderModelInfo | None = None,
|
|
continuation: ContinuationSeed | None = None,
|
|
) -> AsyncIterator[str]:
|
|
raise AssertionError("Messages test provider received a Responses request")
|
|
yield ""
|
|
|
|
def _record_stream_call(
|
|
self,
|
|
request: MessagesRequest,
|
|
*,
|
|
input_tokens: int,
|
|
request_id: str | None,
|
|
response_model: str | None,
|
|
reasoning: ReasoningPolicy,
|
|
) -> None:
|
|
self.stream_calls.append(
|
|
{
|
|
"request": request,
|
|
"input_tokens": input_tokens,
|
|
"request_id": request_id,
|
|
"response_model": response_model,
|
|
"reasoning": reasoning,
|
|
}
|
|
)
|
|
|
|
|
|
class ResponsesFakeProvider:
|
|
def __init__(self) -> None:
|
|
self.stream_calls: list[dict[str, object]] = []
|
|
self.stream_close_calls = 0
|
|
|
|
async def stream_messages(
|
|
self,
|
|
request: MessagesRequest,
|
|
*,
|
|
input_tokens: int,
|
|
request_id: str,
|
|
response_model: str,
|
|
reasoning: ReasoningPolicy,
|
|
request_headers: Mapping[str, str] | None = None,
|
|
model_info: ProviderModelInfo | None = None,
|
|
continuation: ContinuationSeed | None = None,
|
|
) -> AsyncIterator[str]:
|
|
raise AssertionError("Responses test provider received a Messages request")
|
|
yield ""
|
|
|
|
async def stream_responses(
|
|
self,
|
|
request: OpenAIResponsesRequest,
|
|
*,
|
|
input_tokens: int,
|
|
request_id: str,
|
|
response_model: str,
|
|
reasoning: ReasoningPolicy,
|
|
request_headers: Mapping[str, str] | None = None,
|
|
model_info: ProviderModelInfo | None = None,
|
|
continuation: ContinuationSeed | None = None,
|
|
) -> AsyncIterator[str]:
|
|
self.stream_calls.append(
|
|
{
|
|
"request": request,
|
|
"input_tokens": input_tokens,
|
|
"request_id": request_id,
|
|
"response_model": response_model,
|
|
"reasoning": reasoning,
|
|
}
|
|
)
|
|
try:
|
|
yield "event: response.completed\ndata: {}\n\n"
|
|
finally:
|
|
self.stream_close_calls += 1
|
|
|
|
|
|
class WaitStep:
|
|
def __init__(self) -> None:
|
|
self.started = asyncio.Event()
|
|
self.release = asyncio.Event()
|
|
|
|
|
|
class EmptyForever:
|
|
pass
|
|
|
|
|
|
EMPTY_FOREVER = EmptyForever()
|
|
|
|
|
|
class DelayStep:
|
|
def __init__(self, seconds: float) -> None:
|
|
self.seconds = seconds
|
|
|
|
|
|
type StreamStep = str | WaitStep | DelayStep | EmptyForever | BaseException
|
|
|
|
|
|
class ControlledProvider(FakeProvider):
|
|
def __init__(self, steps: list[StreamStep]) -> None:
|
|
super().__init__()
|
|
self._steps = steps
|
|
|
|
async def stream_messages(
|
|
self,
|
|
request: MessagesRequest,
|
|
input_tokens: int = 0,
|
|
*,
|
|
request_id: str | None = None,
|
|
response_model: str | None = None,
|
|
reasoning: ReasoningPolicy,
|
|
request_headers: Mapping[str, str] | None = None,
|
|
model_info: ProviderModelInfo | None = None,
|
|
continuation: ContinuationSeed | None = None,
|
|
) -> AsyncIterator[str]:
|
|
self._record_stream_call(
|
|
request,
|
|
input_tokens=input_tokens,
|
|
request_id=request_id,
|
|
response_model=response_model,
|
|
reasoning=reasoning,
|
|
)
|
|
try:
|
|
for step in self._steps:
|
|
if isinstance(step, str):
|
|
yield step
|
|
elif isinstance(step, WaitStep):
|
|
step.started.set()
|
|
await step.release.wait()
|
|
elif isinstance(step, DelayStep):
|
|
await asyncio.sleep(step.seconds)
|
|
elif isinstance(step, EmptyForever):
|
|
while True:
|
|
yield ""
|
|
else:
|
|
raise step
|
|
finally:
|
|
self.stream_close_calls += 1
|
|
|
|
|
|
class FailingValidationProvider(FakeProvider):
|
|
def stream_messages(
|
|
self,
|
|
request: MessagesRequest,
|
|
input_tokens: int = 0,
|
|
*,
|
|
reasoning: ReasoningPolicy,
|
|
model_info: ProviderModelInfo | None = None,
|
|
**kwargs: object,
|
|
) -> AsyncIterator[str]:
|
|
raise ValueError("invalid provider request")
|
|
|
|
|
|
class ApplicationErrorValidationProvider(FakeProvider):
|
|
def __init__(self, error: InvalidRequestError) -> None:
|
|
super().__init__()
|
|
self._error = error
|
|
|
|
def stream_messages(
|
|
self,
|
|
request: MessagesRequest,
|
|
input_tokens: int = 0,
|
|
*,
|
|
reasoning: ReasoningPolicy,
|
|
model_info: ProviderModelInfo | None = None,
|
|
**kwargs: object,
|
|
) -> AsyncIterator[str]:
|
|
raise self._error
|
|
|
|
|
|
class FailingStreamConstructionProvider(FakeProvider):
|
|
def stream_messages(
|
|
self,
|
|
request: MessagesRequest,
|
|
input_tokens: int = 0,
|
|
*,
|
|
request_id: str | None = None,
|
|
response_model: str | None = None,
|
|
reasoning: ReasoningPolicy,
|
|
request_headers: Mapping[str, str] | None = None,
|
|
model_info: ProviderModelInfo | None = None,
|
|
continuation: ContinuationSeed | None = None,
|
|
) -> AsyncIterator[str]:
|
|
raise RuntimeError("stream construction failed")
|
|
|
|
|
|
class ExecutionFailureStreamConstructionProvider(FakeProvider):
|
|
def __init__(self, failure: ExecutionFailure) -> None:
|
|
super().__init__()
|
|
self._failure = failure
|
|
|
|
def stream_messages(
|
|
self,
|
|
request: MessagesRequest,
|
|
input_tokens: int = 0,
|
|
*,
|
|
request_id: str | None = None,
|
|
response_model: str | None = None,
|
|
reasoning: ReasoningPolicy,
|
|
request_headers: Mapping[str, str] | None = None,
|
|
model_info: ProviderModelInfo | None = None,
|
|
continuation: ContinuationSeed | None = None,
|
|
) -> AsyncIterator[str]:
|
|
del request, input_tokens, request_id, response_model, reasoning
|
|
raise self._failure
|
|
|
|
|
|
class CloseControlledProvider(FakeProvider):
|
|
def __init__(
|
|
self,
|
|
failure: ExecutionFailure,
|
|
*,
|
|
close_delay_seconds: float = 0.0,
|
|
close_error: Exception | None = None,
|
|
) -> None:
|
|
super().__init__()
|
|
self._failure = failure
|
|
self._close_delay_seconds = close_delay_seconds
|
|
self._close_error = close_error
|
|
self._read = False
|
|
|
|
def stream_messages(
|
|
self,
|
|
request: MessagesRequest,
|
|
input_tokens: int = 0,
|
|
*,
|
|
request_id: str | None = None,
|
|
response_model: str | None = None,
|
|
reasoning: ReasoningPolicy,
|
|
request_headers: Mapping[str, str] | None = None,
|
|
model_info: ProviderModelInfo | None = None,
|
|
continuation: ContinuationSeed | None = None,
|
|
) -> AsyncIterator[str]:
|
|
self._record_stream_call(
|
|
request,
|
|
input_tokens=input_tokens,
|
|
request_id=request_id,
|
|
response_model=response_model,
|
|
reasoning=reasoning,
|
|
)
|
|
return self
|
|
|
|
def __aiter__(self) -> AsyncIterator[str]:
|
|
return self
|
|
|
|
async def __anext__(self) -> str:
|
|
if self._read:
|
|
raise StopAsyncIteration
|
|
self._read = True
|
|
raise self._failure
|
|
|
|
async def aclose(self) -> None:
|
|
self.stream_close_calls += 1
|
|
if self._close_delay_seconds:
|
|
await asyncio.sleep(self._close_delay_seconds)
|
|
if self._close_error is not None:
|
|
raise self._close_error
|
|
|
|
|
|
def _target(provider_id: str, provider_model: str) -> ProviderModelTarget:
|
|
return ProviderModelTarget(
|
|
provider_id=provider_id,
|
|
provider_model=provider_model,
|
|
provider_model_ref=f"{provider_id}/{provider_model}",
|
|
)
|
|
|
|
|
|
def _routed_request(
|
|
*fallbacks: ProviderModelTarget,
|
|
) -> RoutedMessagesRequest:
|
|
request = MessagesRequest(
|
|
model="provider-model",
|
|
messages=[Message(role="user", content="hello")],
|
|
)
|
|
return RoutedMessagesRequest(
|
|
request=request,
|
|
resolved=ResolvedModelRoute(
|
|
original_model="gateway-model",
|
|
primary=_target("provider", "provider-model"),
|
|
fallbacks=tuple(ProviderModelCandidate(target) for target in fallbacks),
|
|
reasoning_preference=ReasoningPreference.CLIENT,
|
|
),
|
|
reasoning=ReasoningPolicy.on(),
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("ingress", ["messages", "responses"])
|
|
@pytest.mark.parametrize("unknown_fallback", [False, True])
|
|
async def test_fallback_receives_its_own_cached_capability_in_stream(
|
|
ingress, unknown_fallback
|
|
):
|
|
seen = []
|
|
resolved = set()
|
|
|
|
class MetadataProvider(ControlledProvider):
|
|
async def stream_messages(self, request, input_tokens=0, **kwargs):
|
|
seen.append(("stream", request.model, kwargs["model_info"]))
|
|
async for event in super().stream_messages(request, input_tokens, **kwargs):
|
|
yield event
|
|
|
|
stream_responses = stream_messages
|
|
|
|
primary_info = ProviderModelInfo(
|
|
"provider/provider-model", reasoning_capability=ReasoningCapability.NONE
|
|
)
|
|
fallback_info = (
|
|
ProviderModelInfo(
|
|
"fallback/fallback-model", reasoning_capability=ReasoningCapability.REQUIRED
|
|
)
|
|
if not unknown_fallback
|
|
else None
|
|
)
|
|
primary = MetadataProvider(
|
|
[ExecutionFailure(FailureKind.UNAVAILABLE, 503, "unavailable", True)]
|
|
)
|
|
fallback = MetadataProvider(["verdict"])
|
|
providers = {"provider": primary, "fallback": fallback}
|
|
|
|
async def resolve(name, _api_key_number):
|
|
resolved.add(name)
|
|
return providers[name]
|
|
|
|
def lookup(provider_id, model_id):
|
|
assert provider_id in resolved
|
|
return {
|
|
"provider/provider-model": primary_info,
|
|
"fallback/fallback-model": fallback_info,
|
|
}.get(f"{provider_id}/{model_id}")
|
|
|
|
executor = ProviderExecutor(
|
|
resolve,
|
|
progress_timeout_seconds=10,
|
|
model_info_lookup=lookup,
|
|
)
|
|
route = _routed_request if ingress == "messages" else _routed_responses_request
|
|
output = [
|
|
event
|
|
async for event in getattr(executor, f"stream_{ingress}")(
|
|
route(_target("fallback", "fallback-model")),
|
|
raw_log_payload=dict,
|
|
request_id="capability-fallback",
|
|
)
|
|
]
|
|
assert output == ["verdict"]
|
|
assert seen == [
|
|
("stream", "provider-model", primary_info),
|
|
("stream", "fallback-model", fallback_info),
|
|
]
|
|
|
|
|
|
def _routed_responses_request(
|
|
*fallbacks: ProviderModelTarget,
|
|
) -> RoutedResponsesRequest:
|
|
request = OpenAIResponsesRequest(
|
|
model="provider-model",
|
|
input=[{"role": "user", "content": "hello"}],
|
|
reasoning={"effort": "high"},
|
|
)
|
|
return RoutedResponsesRequest(
|
|
request=request,
|
|
resolved=ResolvedModelRoute(
|
|
original_model="gateway-model",
|
|
primary=_target("provider", "provider-model"),
|
|
fallbacks=tuple(ProviderModelCandidate(target) for target in fallbacks),
|
|
reasoning_preference=ReasoningPreference.CLIENT,
|
|
),
|
|
reasoning=ReasoningPolicy.on(),
|
|
)
|
|
|
|
|
|
def _executor_stream(
|
|
provider: FakeProvider,
|
|
*,
|
|
timeout_seconds: float,
|
|
request_id: str,
|
|
) -> AsyncIterator[str]:
|
|
executor = ProviderExecutor(
|
|
AsyncMock(side_effect=lambda _provider_id, _api_key_number: provider),
|
|
token_counter=lambda _messages, _system, _tools: 17,
|
|
progress_timeout_seconds=timeout_seconds,
|
|
)
|
|
return executor.stream_messages(
|
|
_routed_request(),
|
|
raw_log_payload=dict,
|
|
request_id=request_id,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_executor_routes_native_responses_without_messages_conversion() -> None:
|
|
provider = ResponsesFakeProvider()
|
|
routed = _routed_responses_request()
|
|
request = routed.request
|
|
executor = ProviderExecutor(
|
|
AsyncMock(side_effect=lambda _provider_id, _api_key_number: provider),
|
|
progress_timeout_seconds=60.0,
|
|
responses_token_counter=lambda _request: 23,
|
|
)
|
|
|
|
stream = executor.stream_responses(
|
|
routed,
|
|
raw_log_payload=lambda: request.model_dump(mode="json"),
|
|
request_id="req_responses_application",
|
|
)
|
|
|
|
assert provider.stream_calls == []
|
|
assert [chunk async for chunk in stream] == [
|
|
"event: response.completed\ndata: {}\n\n"
|
|
]
|
|
assert provider.stream_calls == [
|
|
{
|
|
"request": request,
|
|
"input_tokens": 23,
|
|
"request_id": "req_responses_application",
|
|
"response_model": "gateway-model",
|
|
"reasoning": ReasoningPolicy.on(),
|
|
}
|
|
]
|
|
assert provider.stream_close_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_executor_uses_structural_provider_port_and_defers_stream_startup() -> (
|
|
None
|
|
):
|
|
provider = FakeProvider()
|
|
routed = _routed_request()
|
|
request = routed.request
|
|
executor = ProviderExecutor(
|
|
AsyncMock(side_effect=lambda _provider_id, _api_key_number: provider),
|
|
progress_timeout_seconds=60.0,
|
|
token_counter=lambda _messages, _system, _tools: 17,
|
|
)
|
|
|
|
stream = executor.stream_messages(
|
|
routed,
|
|
raw_log_payload=request.model_dump,
|
|
request_id="req_application",
|
|
)
|
|
|
|
assert provider.stream_calls == []
|
|
assert [chunk async for chunk in stream] == ["event: message_stop\ndata: {}\n\n"]
|
|
assert provider.stream_calls == [
|
|
{
|
|
"request": request,
|
|
"input_tokens": 17,
|
|
"request_id": "req_application",
|
|
"response_model": "gateway-model",
|
|
"reasoning": ReasoningPolicy.on(),
|
|
}
|
|
]
|
|
assert provider.stream_close_calls == 1
|
|
|
|
|
|
def _execution_failure(
|
|
message: str,
|
|
*,
|
|
retryable: bool = True,
|
|
) -> ExecutionFailure:
|
|
return ExecutionFailure(
|
|
kind=FailureKind.OVERLOADED,
|
|
status_code=529,
|
|
message=message,
|
|
retryable=retryable,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_primary_success_never_resolves_fallback() -> None:
|
|
primary = FakeProvider()
|
|
fallback = FakeProvider()
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return {"provider": primary, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(resolve, progress_timeout_seconds=60.0)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "fallback-model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_primary_success",
|
|
)
|
|
|
|
assert [chunk async for chunk in stream] == ["event: message_stop\ndata: {}\n\n"]
|
|
assert resolved_ids == ["provider"]
|
|
assert fallback.stream_calls == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retryable_preframe_failure_selects_fallback_after_closing_primary() -> (
|
|
None
|
|
):
|
|
order: list[str] = []
|
|
primary = ControlledProvider([_execution_failure("primary overloaded")])
|
|
fallback = ControlledProvider(["fallback-frame"])
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
if provider_id == "fallback":
|
|
assert primary.stream_close_calls == 1
|
|
order.append("fallback-resolved")
|
|
return {"provider": primary, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(
|
|
resolve,
|
|
token_counter=lambda _messages, _system, _tools: 17,
|
|
progress_timeout_seconds=60.0,
|
|
generation_id=7,
|
|
)
|
|
routed = _routed_request(_target("fallback", "fallback-model"))
|
|
|
|
with patch("free_claude_code.application.execution.trace_event") as trace_mock:
|
|
chunks = [
|
|
chunk
|
|
async for chunk in executor.stream_messages(
|
|
routed,
|
|
raw_log_payload=dict,
|
|
request_id="req_fallback",
|
|
)
|
|
]
|
|
|
|
assert chunks == ["fallback-frame"]
|
|
assert order == ["fallback-resolved"]
|
|
assert primary.stream_close_calls == 1
|
|
assert fallback.stream_close_calls == 1
|
|
fallback_request = fallback.stream_calls[0]["request"]
|
|
assert isinstance(fallback_request, MessagesRequest)
|
|
assert fallback_request.model == "fallback-model"
|
|
assert fallback.stream_calls[0] == {
|
|
"request": fallback_request,
|
|
"input_tokens": 17,
|
|
"request_id": "req_fallback",
|
|
"response_model": "gateway-model",
|
|
"reasoning": ReasoningPolicy.on(),
|
|
}
|
|
events = [call.kwargs for call in trace_mock.call_args_list]
|
|
started = next(
|
|
event
|
|
for event in events
|
|
if event.get("event") == "free_claude_code.model_fallback.started"
|
|
)
|
|
assert started == {
|
|
"stage": "execution",
|
|
"event": "free_claude_code.model_fallback.started",
|
|
"source": "application",
|
|
"request_id": "req_fallback",
|
|
"wire_api": "messages",
|
|
"from_provider_model_ref": "provider/provider-model",
|
|
"to_provider_model_ref": "fallback/fallback-model",
|
|
"from_api_key_number": 1,
|
|
"to_api_key_number": 1,
|
|
"candidate_index": 2,
|
|
"candidate_count": 2,
|
|
"failure_kind": "overloaded",
|
|
"status_code": 529,
|
|
"provider_retryable": True,
|
|
"recovery_action": "restart",
|
|
"recovery_reason": "no_candidate_output",
|
|
"generation_id": 7,
|
|
}
|
|
selected = next(
|
|
event
|
|
for event in events
|
|
if event.get("event") == "free_claude_code.model_fallback.selected"
|
|
)
|
|
assert selected["selected_provider_model_ref"] == "fallback/fallback-model"
|
|
assert selected["candidate_index"] == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fallback_chain_preserves_exact_last_failure() -> None:
|
|
first = _execution_failure("first")
|
|
second = _execution_failure("second")
|
|
providers = {
|
|
"provider": ControlledProvider([first]),
|
|
"second": ControlledProvider([second]),
|
|
}
|
|
executor = ProviderExecutor(
|
|
AsyncMock(
|
|
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
|
|
),
|
|
progress_timeout_seconds=60.0,
|
|
)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("second", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_exhausted",
|
|
)
|
|
|
|
with pytest.raises(ExecutionFailure) as exc_info:
|
|
await anext(stream)
|
|
|
|
assert exc_info.value is second
|
|
assert all(provider.stream_close_calls == 1 for provider in providers.values())
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_multiple_retryable_failures_walk_fallbacks_in_order() -> None:
|
|
providers = {
|
|
"provider": ControlledProvider([_execution_failure("primary")]),
|
|
"second": ControlledProvider([_execution_failure("second")]),
|
|
"third": ControlledProvider(["third-frame"]),
|
|
}
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return providers[provider_id]
|
|
|
|
executor = ProviderExecutor(resolve, progress_timeout_seconds=60.0)
|
|
stream = executor.stream_messages(
|
|
_routed_request(
|
|
_target("second", "second-model"),
|
|
_target("third", "third-model"),
|
|
),
|
|
raw_log_payload=dict,
|
|
request_id="req_ordered_fallbacks",
|
|
)
|
|
|
|
assert [chunk async for chunk in stream] == ["third-frame"]
|
|
assert resolved_ids == ["provider", "second", "third"]
|
|
assert all(provider.stream_close_calls == 1 for provider in providers.values())
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_primary_completion_does_not_select_fallback() -> None:
|
|
primary = ControlledProvider([])
|
|
fallback = FakeProvider()
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return {"provider": primary, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(resolve, progress_timeout_seconds=60.0)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_empty_primary",
|
|
)
|
|
|
|
assert [chunk async for chunk in stream] == []
|
|
assert resolved_ids == ["provider"]
|
|
assert primary.stream_close_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unexpected_primary_failure_does_not_select_fallback() -> None:
|
|
primary = ControlledProvider([RuntimeError("unexpected")])
|
|
fallback = FakeProvider()
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return {"provider": primary, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(resolve, progress_timeout_seconds=60.0)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_unexpected_primary",
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="unexpected"):
|
|
await anext(stream)
|
|
|
|
assert resolved_ids == ["provider"]
|
|
assert primary.stream_close_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_invalid_none_stream_remains_terminal() -> None:
|
|
primary = FakeProvider()
|
|
fallback = FakeProvider()
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return {"provider": primary, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(resolve, progress_timeout_seconds=60.0)
|
|
with patch.object(primary, "stream_messages", return_value=None):
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_invalid_stream",
|
|
)
|
|
with pytest.raises(TypeError):
|
|
await anext(stream)
|
|
|
|
assert resolved_ids == ["provider"]
|
|
assert fallback.stream_calls == []
|
|
|
|
|
|
@pytest.mark.parametrize("failure_kind", tuple(FailureKind))
|
|
@pytest.mark.asyncio
|
|
async def test_nonretryable_provider_failure_selects_fallback_before_first_frame(
|
|
failure_kind: FailureKind,
|
|
) -> None:
|
|
primary_failure = ExecutionFailure(
|
|
kind=failure_kind,
|
|
status_code=500,
|
|
message="provider rejected candidate",
|
|
retryable=False,
|
|
)
|
|
primary = ControlledProvider([primary_failure])
|
|
fallback = ControlledProvider(["fallback-frame"])
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return {"provider": primary, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(resolve, progress_timeout_seconds=60.0)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_rejected",
|
|
)
|
|
|
|
with patch("free_claude_code.application.execution.trace_event") as trace_mock:
|
|
chunks = [chunk async for chunk in stream]
|
|
|
|
assert chunks == ["fallback-frame"]
|
|
assert resolved_ids == ["provider", "fallback"]
|
|
fallback_started = next(
|
|
call.kwargs
|
|
for call in trace_mock.call_args_list
|
|
if call.kwargs.get("event") == "free_claude_code.model_fallback.started"
|
|
)
|
|
assert fallback_started["failure_kind"] == failure_kind.value
|
|
assert fallback_started["provider_retryable"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_primary_canonical_startup_failure_selects_fallback() -> None:
|
|
primary_failure = ExecutionFailure(
|
|
kind=FailureKind.CONTEXT_WINDOW_EXCEEDED,
|
|
status_code=400,
|
|
message="primary context exceeded",
|
|
retryable=False,
|
|
)
|
|
primary = ExecutionFailureStreamConstructionProvider(primary_failure)
|
|
fallback = ControlledProvider(["fallback-frame"])
|
|
providers = {"provider": primary, "fallback": fallback}
|
|
executor = ProviderExecutor(
|
|
AsyncMock(
|
|
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
|
|
),
|
|
progress_timeout_seconds=60.0,
|
|
)
|
|
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "fallback-model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_startup_fallback",
|
|
)
|
|
|
|
assert [chunk async for chunk in stream] == ["fallback-frame"]
|
|
assert primary.stream_calls == []
|
|
fallback_request = fallback.stream_calls[0]["request"]
|
|
assert isinstance(fallback_request, MessagesRequest)
|
|
assert fallback_request.model == "fallback-model"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_nonretryable_stream_construction_failure_selects_fallback() -> None:
|
|
primary_failure = ExecutionFailure(
|
|
kind=FailureKind.PERMISSION,
|
|
status_code=403,
|
|
message="provider denied candidate",
|
|
retryable=False,
|
|
)
|
|
primary = ExecutionFailureStreamConstructionProvider(primary_failure)
|
|
fallback = ControlledProvider(["fallback-frame"])
|
|
providers = {"provider": primary, "fallback": fallback}
|
|
executor = ProviderExecutor(
|
|
AsyncMock(
|
|
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
|
|
),
|
|
progress_timeout_seconds=60.0,
|
|
)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_construction_failure",
|
|
)
|
|
|
|
assert [chunk async for chunk in stream] == ["fallback-frame"]
|
|
assert fallback.stream_calls[0]["request_id"] == "req_construction_failure"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_failure_after_first_frame_never_selects_fallback() -> None:
|
|
primary_failure = _execution_failure("after frame")
|
|
primary = ControlledProvider(["first-frame", primary_failure])
|
|
fallback = FakeProvider()
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return {"provider": primary, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(resolve, progress_timeout_seconds=60.0)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_committed",
|
|
)
|
|
|
|
assert await anext(stream) == "first-frame"
|
|
with pytest.raises(ExecutionFailure) as exc_info:
|
|
await anext(stream)
|
|
|
|
assert exc_info.value is primary_failure
|
|
assert resolved_ids == ["provider"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_lazy_fallback_startup_application_error_stops_unchanged() -> None:
|
|
primary = ControlledProvider([_execution_failure("primary")])
|
|
startup_error = InvalidRequestError("fallback is misconfigured")
|
|
fallback = ApplicationErrorValidationProvider(startup_error)
|
|
providers = {"provider": primary, "fallback": fallback}
|
|
executor = ProviderExecutor(
|
|
AsyncMock(
|
|
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
|
|
),
|
|
progress_timeout_seconds=60.0,
|
|
)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_startup",
|
|
)
|
|
|
|
with pytest.raises(InvalidRequestError) as exc_info:
|
|
await anext(stream)
|
|
|
|
assert exc_info.value is startup_error
|
|
assert primary.stream_close_calls == 1
|
|
assert fallback.stream_calls == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_candidate_requests_are_isolated_from_provider_mutation() -> None:
|
|
class MutatingProvider(ControlledProvider):
|
|
async def stream_messages(
|
|
self,
|
|
request: MessagesRequest,
|
|
input_tokens: int = 0,
|
|
*,
|
|
request_id: str | None = None,
|
|
response_model: str | None = None,
|
|
reasoning: ReasoningPolicy,
|
|
request_headers: Mapping[str, str] | None = None,
|
|
model_info: ProviderModelInfo | None = None,
|
|
continuation: ContinuationSeed | None = None,
|
|
) -> AsyncIterator[str]:
|
|
request.messages[0].content = "mutated"
|
|
async for chunk in super().stream_messages(
|
|
request,
|
|
input_tokens,
|
|
request_id=request_id,
|
|
response_model=response_model,
|
|
reasoning=reasoning,
|
|
):
|
|
yield chunk
|
|
|
|
primary = MutatingProvider([_execution_failure("primary")])
|
|
fallback = FakeProvider()
|
|
providers = {"provider": primary, "fallback": fallback}
|
|
routed = _routed_request(_target("fallback", "model"))
|
|
executor = ProviderExecutor(
|
|
AsyncMock(
|
|
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
|
|
),
|
|
progress_timeout_seconds=60.0,
|
|
)
|
|
|
|
assert [
|
|
chunk
|
|
async for chunk in executor.stream_messages(
|
|
routed,
|
|
raw_log_payload=dict,
|
|
request_id="req_isolated",
|
|
)
|
|
]
|
|
fallback_request = fallback.stream_calls[0]["request"]
|
|
assert isinstance(fallback_request, MessagesRequest)
|
|
assert fallback_request.messages[0].content == "hello"
|
|
assert routed.request.messages[0].content == "hello"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_closing_executor_stream_closes_provider_stream_once() -> None:
|
|
provider = FakeProvider()
|
|
routed = _routed_request()
|
|
executor = ProviderExecutor(
|
|
AsyncMock(side_effect=lambda _provider_id, _api_key_number: provider),
|
|
progress_timeout_seconds=60.0,
|
|
token_counter=lambda _messages, _system, _tools: 17,
|
|
)
|
|
stream = executor.stream_messages(
|
|
routed,
|
|
raw_log_payload=dict,
|
|
request_id="req_early_close",
|
|
)
|
|
|
|
assert await anext(stream) == "event: message_stop\ndata: {}\n\n"
|
|
assert isinstance(stream, AsyncCloseable)
|
|
await stream.aclose()
|
|
|
|
assert provider.stream_close_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stream_construction_failure_remains_deferred_to_iteration() -> None:
|
|
provider = FailingStreamConstructionProvider()
|
|
fallback = FakeProvider()
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return {"provider": provider, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(
|
|
resolve,
|
|
progress_timeout_seconds=60.0,
|
|
token_counter=lambda _messages, _system, _tools: 17,
|
|
)
|
|
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_deferred_construction",
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="stream construction failed"):
|
|
await anext(stream)
|
|
|
|
assert resolved_ids == ["provider"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_executor_validation_is_deferred_until_iteration() -> None:
|
|
provider = FailingValidationProvider()
|
|
token_counter = MagicMock(return_value=17)
|
|
executor = ProviderExecutor(
|
|
AsyncMock(side_effect=lambda _provider_id, _api_key_number: provider),
|
|
progress_timeout_seconds=60.0,
|
|
token_counter=token_counter,
|
|
)
|
|
|
|
stream = executor.stream_messages(
|
|
_routed_request(),
|
|
raw_log_payload=dict,
|
|
request_id="req_application",
|
|
)
|
|
token_counter.assert_called_once()
|
|
with pytest.raises(ValueError, match="invalid provider request"):
|
|
await anext(stream)
|
|
assert provider.stream_calls == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"timeout_seconds",
|
|
[0.0, -1.0, float("inf"), float("nan")],
|
|
)
|
|
def test_executor_rejects_invalid_progress_timeout(timeout_seconds: float) -> None:
|
|
with pytest.raises(ValueError, match="finite and positive"):
|
|
ProviderExecutor(
|
|
AsyncMock(side_effect=lambda _provider_id, _api_key_number: FakeProvider()),
|
|
progress_timeout_seconds=timeout_seconds,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_progress_timeout_before_first_chunk_is_canonical_and_correlated() -> (
|
|
None
|
|
):
|
|
provider = ControlledProvider([WaitStep()])
|
|
request_id = "req_progress_timeout"
|
|
|
|
with patch("free_claude_code.application.execution.trace_event") as trace_mock:
|
|
stream = _executor_stream(
|
|
provider,
|
|
timeout_seconds=0.02,
|
|
request_id=request_id,
|
|
)
|
|
with pytest.raises(ExecutionFailure) as exc_info:
|
|
await anext(stream)
|
|
|
|
failure = exc_info.value
|
|
assert failure.kind == FailureKind.TIMEOUT
|
|
assert failure.status_code == 504
|
|
assert failure.retryable is False
|
|
assert failure.message == (
|
|
"Provider execution made no progress for 0.02 seconds.\n\n"
|
|
f"Request ID: {request_id}"
|
|
)
|
|
assert provider.stream_close_calls == 1
|
|
timeout_traces = [
|
|
call.kwargs
|
|
for call in trace_mock.call_args_list
|
|
if call.kwargs.get("event") == "free_claude_code.provider.progress_timeout"
|
|
]
|
|
assert timeout_traces == [
|
|
{
|
|
"stage": "execution",
|
|
"event": "free_claude_code.provider.progress_timeout",
|
|
"source": "application",
|
|
"request_id": request_id,
|
|
"provider_id": "provider",
|
|
"timeout_seconds": 0.02,
|
|
}
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_application_progress_timeout_never_resolves_fallback() -> None:
|
|
primary = ControlledProvider([WaitStep()])
|
|
fallback = FakeProvider()
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return {"provider": primary, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(resolve, progress_timeout_seconds=0.02)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_terminal_progress_timeout",
|
|
)
|
|
|
|
with pytest.raises(ExecutionFailure) as exc_info:
|
|
await anext(stream)
|
|
|
|
assert exc_info.value.kind is FailureKind.TIMEOUT
|
|
assert exc_info.value.status_code == 504
|
|
assert exc_info.value.retryable is False
|
|
assert resolved_ids == ["provider"]
|
|
assert primary.stream_close_calls == 1
|
|
assert fallback.stream_calls == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_provider_cleanup_cannot_delay_fallback_past_progress_deadline() -> None:
|
|
primary = CloseControlledProvider(
|
|
_execution_failure("primary overloaded"),
|
|
close_delay_seconds=0.05,
|
|
)
|
|
fallback = FakeProvider()
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return {"provider": primary, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(resolve, progress_timeout_seconds=0.02)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_cleanup_timeout",
|
|
)
|
|
|
|
with (
|
|
patch("free_claude_code.application.execution.trace_event") as trace_mock,
|
|
pytest.raises(ExecutionFailure) as exc_info,
|
|
):
|
|
await anext(stream)
|
|
|
|
assert exc_info.value.kind is FailureKind.TIMEOUT
|
|
assert exc_info.value.status_code == 504
|
|
assert resolved_ids == ["provider"]
|
|
assert primary.stream_close_calls == 1
|
|
assert fallback.stream_calls == []
|
|
assert all(
|
|
call.kwargs.get("event") != "free_claude_code.model_fallback.started"
|
|
for call in trace_mock.call_args_list
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cleanup_warning_names_preserved_provider_failure(caplog) -> None:
|
|
failure = _execution_failure("primary overloaded")
|
|
provider = CloseControlledProvider(
|
|
failure,
|
|
close_error=RuntimeError("close failed"),
|
|
)
|
|
stream = _executor_stream(
|
|
provider,
|
|
timeout_seconds=1.0,
|
|
request_id="req_close_failure_trace",
|
|
)
|
|
|
|
with pytest.raises(ExecutionFailure) as exc_info:
|
|
await anext(stream)
|
|
|
|
assert exc_info.value is failure
|
|
warnings = [record for record in caplog.records if record.levelname == "WARNING"]
|
|
assert len(warnings) == 1
|
|
assert warnings[0].extra["event"] == "stream.input.close_failed"
|
|
assert warnings[0].extra["close_exc_type"] == "RuntimeError"
|
|
assert warnings[0].extra["preserved_exc_type"] == "ExecutionFailure"
|
|
assert str(warnings[0].exc_info[1]) == "close failed"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_progress_timeout_renews_after_output_without_crossing_yield() -> None:
|
|
second_wait = WaitStep()
|
|
third_wait = WaitStep()
|
|
provider = ControlledProvider(["first", second_wait, "second", third_wait, "third"])
|
|
stream = _executor_stream(
|
|
provider,
|
|
timeout_seconds=0.02,
|
|
request_id="req_progress_renewal",
|
|
)
|
|
|
|
assert await anext(stream) == "first"
|
|
await asyncio.sleep(0.05)
|
|
second_wait.release.set()
|
|
assert await anext(stream) == "second"
|
|
|
|
with pytest.raises(ExecutionFailure) as exc_info:
|
|
await anext(stream)
|
|
|
|
assert third_wait.started.is_set()
|
|
assert exc_info.value.kind == FailureKind.TIMEOUT
|
|
assert provider.stream_close_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_chunks_do_not_renew_provider_progress() -> None:
|
|
provider = ControlledProvider([EMPTY_FOREVER])
|
|
stream = _executor_stream(
|
|
provider,
|
|
timeout_seconds=0.02,
|
|
request_id="req_empty_progress",
|
|
)
|
|
|
|
with pytest.raises(ExecutionFailure) as exc_info:
|
|
await anext(stream)
|
|
|
|
assert exc_info.value.kind == FailureKind.TIMEOUT
|
|
assert provider.stream_close_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fallback_transition_does_not_reset_shared_progress_deadline() -> None:
|
|
primary_wait = WaitStep()
|
|
primary = ControlledProvider(
|
|
[primary_wait, _execution_failure("primary overloaded")]
|
|
)
|
|
fallback_wait = WaitStep()
|
|
fallback = ControlledProvider([fallback_wait])
|
|
providers = {"provider": primary, "fallback": fallback}
|
|
executor = ProviderExecutor(
|
|
AsyncMock(
|
|
side_effect=lambda provider_id, _api_key_number: providers[provider_id]
|
|
),
|
|
progress_timeout_seconds=60,
|
|
)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_shared_deadline",
|
|
)
|
|
deadlines: list[float | None] = []
|
|
timeouts: list[asyncio.Timeout] = []
|
|
|
|
def controlled_timeout_at(deadline):
|
|
deadlines.append(deadline)
|
|
timeout = asyncio.timeout(None)
|
|
timeouts.append(timeout)
|
|
return timeout
|
|
|
|
with (
|
|
patch(
|
|
"free_claude_code.application.execution.monotonic", side_effect=[0, 1, 2, 4]
|
|
),
|
|
patch("free_claude_code.application.execution.trace_event") as trace_mock,
|
|
patch(
|
|
"free_claude_code.application.execution.asyncio.timeout_at",
|
|
controlled_timeout_at,
|
|
),
|
|
):
|
|
reading = asyncio.ensure_future(anext(stream))
|
|
await primary_wait.started.wait()
|
|
primary_wait.release.set()
|
|
await fallback_wait.started.wait()
|
|
# Expire the actual fallback read context after observing its admission;
|
|
# machine load must not determine whether the shared deadline was reused.
|
|
timeouts[-1].reschedule(asyncio.get_running_loop().time())
|
|
with pytest.raises(ExecutionFailure) as exc_info:
|
|
await reading
|
|
|
|
read_deadlines = [deadline for deadline in deadlines if deadline is not None]
|
|
assert len(read_deadlines) >= 3
|
|
assert read_deadlines[0] == read_deadlines[1]
|
|
assert read_deadlines[-1] == read_deadlines[0] + 2
|
|
assert exc_info.value.kind == FailureKind.TIMEOUT
|
|
assert primary.stream_close_calls == fallback.stream_close_calls == 1
|
|
timeout_trace = next(
|
|
call.kwargs
|
|
for call in trace_mock.call_args_list
|
|
if call.kwargs.get("event") == "free_claude_code.provider.progress_timeout"
|
|
)
|
|
assert timeout_trace["provider_id"] == "fallback"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_provider_owned_timeout_is_not_reclassified() -> None:
|
|
provider = ControlledProvider([TimeoutError("provider-owned timeout")])
|
|
stream = _executor_stream(
|
|
provider,
|
|
timeout_seconds=1.0,
|
|
request_id="req_provider_timeout",
|
|
)
|
|
|
|
with pytest.raises(TimeoutError, match="provider-owned timeout"):
|
|
await anext(stream)
|
|
|
|
assert provider.stream_close_calls == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cancelling_progress_wait_remains_cancellation() -> None:
|
|
wait = WaitStep()
|
|
provider = ControlledProvider([wait])
|
|
fallback = FakeProvider()
|
|
resolved_ids: list[str] = []
|
|
|
|
async def resolve(provider_id: str, _api_key_number: int) -> FakeProvider:
|
|
resolved_ids.append(provider_id)
|
|
return {"provider": provider, "fallback": fallback}[provider_id]
|
|
|
|
executor = ProviderExecutor(
|
|
resolve,
|
|
progress_timeout_seconds=1.0,
|
|
)
|
|
stream = executor.stream_messages(
|
|
_routed_request(_target("fallback", "model")),
|
|
raw_log_payload=dict,
|
|
request_id="req_cancelled_progress",
|
|
)
|
|
task = asyncio.ensure_future(anext(stream))
|
|
await wait.started.wait()
|
|
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
|
|
assert provider.stream_close_calls == 1
|
|
assert resolved_ids == ["provider"]
|