mirror of
https://github.com/Alishahryar1/free-claude-code.git
synced 2026-10-19 15:33:34 +00:00
4e99ddaa5c
Some checks are pending
## Why
Fallback rows could choose a model and reasoning effort but always used
the provider's first API key. Users need to choose which numbered key
each fallback uses, including different keys for the same model.
## How
Add a Key N selector and persist a validated `api_key_number`,
defaulting existing entries to Key 1. Reject duplicate model/key pairs
and key removals that leave a saved fallback's selection unavailable.
Carry the number through routing and the request's retained runtime
generation. Cache and construct clients per provider/key pair so
concurrent attempts keep their selected credentials.
Share rate and concurrency capacity across a provider's keys, while
recovery belongs to the actual credential. A healthy selected fallback
key receives an attempt after another key exhausts rate-limit retries.
Reordering a key preserves its recovery state, and replacing it uses a
separate state. Keep the existing retry classification, attempt count,
and backoff.
Refresh key choices after provider edits while preserving unsaved model
drafts. Display removed numbers as unavailable and reset the choice to
Key 1 when a row changes provider. Numbers follow the current order in
the provider's settings. Providers with another authentication method
show a disabled default selection.
The Admin editor targets desktop viewports, as requested. Its existing
rows and add row keep the three selections aligned horizontally.
Local CI passes, including 9,049 Python tests and 277 browser tests.
Added regressions capture actual authentication headers after 403 and
exhausted 429 failures across both public APIs and all three custom
endpoint formats, and check shared capacity, client coalescing, retained
generations, credential reorder/replacement, and Admin edits.
<!-- greptile_comment -->
<!-- greptile_summary -->
<h2><a
href="https://app.greptile.com/api/retrigger?id=79105240"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://greptile-static-assets.s3.amazonaws.com/badges/RetriggerDark.svg?v=2"><source
media="(prefers-color-scheme: light)"
srcset="https://greptile-static-assets.s3.amazonaws.com/badges/Retrigger.svg?v=2"><img
alt="Retrigger"
src="https://greptile-static-assets.s3.amazonaws.com/badges/Retrigger.svg?v=2"
align="right"></picture></a>Confidence Score: 5/5</h2>
<!-- greptile-impact-verdict --> No outstanding blocking findings were
identified. The reviewed changes are safe to merge.
<details open><summary><h3>Summary</h3></summary>
Adds a numbered API-key choice to each fallback model and carries that
choice through routing and provider construction.
- Fallback rows can choose a numbered API key.
- Fallback attempts send requests with their selected API keys.
- Provider clients keep separate keys while sharing provider limits.
</details>
<!-- greptile_confidence_score:5 -->
<sub>Reviews (3) · Last reviewed commit: ["Keep credential recovery
separate from
p..."](c7c33ac61b)
· [Reviewed by
Greptile](https://www.greptile.com/?utm_source=greptile_expert&utm_medium=github&utm_campaign=code_reviews&utm_content=alishahryar1/free-claude-code)</sub>
<!-- /greptile_comment -->
507 lines
17 KiB
Python
507 lines
17 KiB
Python
"""Behavioral checks for progressively available, owned startup work."""
|
|
|
|
import asyncio
|
|
import threading
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from free_claude_code.api.app import create_app
|
|
from free_claude_code.api.ports import ApiServices
|
|
from free_claude_code.application.errors import ApplicationUnavailableError
|
|
from free_claude_code.application.model_metadata import ProviderModelInfo
|
|
from free_claude_code.application.readiness import InitializationWait
|
|
from free_claude_code.config.api_keys import normalize_api_keys
|
|
from free_claude_code.config.loader import ManagedConfigStore
|
|
from free_claude_code.config.settings import Settings
|
|
from free_claude_code.core.async_tasks import run_sync_owned
|
|
from free_claude_code.providers.base import BaseProvider
|
|
from free_claude_code.providers.runtime.runtime import ProviderRuntime
|
|
from free_claude_code.runtime.application import ApplicationRuntime
|
|
from free_claude_code.runtime.codex_catalog import CodexModelCatalogPublisher
|
|
from free_claude_code.runtime.configuration import ConfigurationService
|
|
from free_claude_code.runtime.provider_manager import ProviderRuntimeManager
|
|
from tests.web_tools_support import StubWebToolsClient
|
|
|
|
|
|
def _settings():
|
|
return Settings().model_copy(
|
|
update={
|
|
"model": "nvidia_nim/one",
|
|
"nvidia_nim_api_keys": normalize_api_keys("test"),
|
|
"groq_api_keys": normalize_api_keys("test"),
|
|
"messaging_platform": "none",
|
|
}
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_http_and_admin_serve_while_selected_catalog_is_held():
|
|
entered, release = asyncio.Event(), asyncio.Event()
|
|
provider = MagicMock(spec=BaseProvider)
|
|
|
|
async def list_models():
|
|
entered.set()
|
|
await release.wait()
|
|
return frozenset({ProviderModelInfo("one")})
|
|
|
|
provider.list_model_infos = AsyncMock(side_effect=list_models)
|
|
|
|
async def construct(_provider_id, _settings, _admission_registry, _api_key_number):
|
|
return provider
|
|
|
|
manager = ProviderRuntimeManager(
|
|
_settings(),
|
|
runtime_factory=lambda settings, admission_registry: ProviderRuntime(
|
|
settings,
|
|
admission_registry,
|
|
provider_constructor=construct,
|
|
),
|
|
)
|
|
runtime = ApplicationRuntime(
|
|
manager,
|
|
configuration=ConfigurationService(ManagedConfigStore()),
|
|
transcriber=None,
|
|
)
|
|
app = create_app(
|
|
ApiServices(manager, runtime, runtime, web_tools=StubWebToolsClient())
|
|
)
|
|
try:
|
|
await asyncio.wait_for(runtime.start(), 1)
|
|
await asyncio.wait_for(entered.wait(), 1)
|
|
async with httpx.AsyncClient(
|
|
transport=httpx.ASGITransport(app=app), base_url="http://127.0.0.1"
|
|
) as client:
|
|
assert (await client.get("/health")).status_code == 200
|
|
status = await client.get("/admin/api/status")
|
|
assert status.status_code == 200
|
|
assert status.json()["startup"]["catalog"] == "starting"
|
|
finally:
|
|
release.set()
|
|
await runtime.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_independent_provider_and_cancelled_waiter_share_initialization():
|
|
entered, release = asyncio.Event(), asyncio.Event()
|
|
slow, fast = MagicMock(spec=BaseProvider), MagicMock(spec=BaseProvider)
|
|
|
|
async def slow_models():
|
|
entered.set()
|
|
await release.wait()
|
|
return frozenset({ProviderModelInfo("slow")})
|
|
|
|
slow.list_model_infos = AsyncMock(side_effect=slow_models)
|
|
fast.list_model_infos = AsyncMock(
|
|
return_value=frozenset({ProviderModelInfo("one")})
|
|
)
|
|
|
|
async def construct(provider_id, _settings, _admission_registry, _api_key_number):
|
|
return slow if provider_id == "groq" else fast
|
|
|
|
manager = ProviderRuntimeManager(
|
|
_settings(),
|
|
runtime_factory=lambda settings, admission_registry: ProviderRuntime(
|
|
settings,
|
|
admission_registry,
|
|
provider_constructor=construct,
|
|
),
|
|
)
|
|
manager.start_model_list_refresh()
|
|
leases = [await manager.acquire() for _ in range(3)]
|
|
waiting = [
|
|
asyncio.create_task(lease.resolve_provider("groq")) for lease in leases[:2]
|
|
]
|
|
try:
|
|
await entered.wait()
|
|
assert await leases[2].resolve_provider("nvidia_nim") is fast
|
|
assert leases[2].model_info("nvidia_nim", "one") is not None
|
|
waiting[0].cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await waiting[0]
|
|
assert not waiting[1].done()
|
|
release.set()
|
|
assert await waiting[1] is slow
|
|
assert slow.list_model_infos.await_count == 1
|
|
finally:
|
|
release.set()
|
|
await asyncio.gather(*waiting, return_exceptions=True)
|
|
for lease in leases:
|
|
await lease.release()
|
|
await manager.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_wait_timeout_does_not_cancel_initializer_and_settled_work_still_works():
|
|
release = asyncio.Event()
|
|
task = asyncio.create_task(release.wait())
|
|
wait = InitializationWait(0.01)
|
|
try:
|
|
with pytest.raises(ApplicationUnavailableError, match="still starting"):
|
|
await wait.wait(task)
|
|
assert not task.done()
|
|
release.set()
|
|
await task
|
|
assert await wait.wait(task) is True
|
|
finally:
|
|
release.set()
|
|
await task
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("refresh", [False, True])
|
|
async def test_failed_discovery_construction_recovers_in_same_generation(refresh):
|
|
provider = MagicMock(spec=BaseProvider)
|
|
provider.list_model_infos = AsyncMock(
|
|
return_value=frozenset({ProviderModelInfo("one")})
|
|
)
|
|
attempts = 0
|
|
|
|
async def construct(_id, _settings, _admission_registry, _api_key_number):
|
|
nonlocal attempts
|
|
attempts += 1
|
|
if attempts == 1:
|
|
raise RuntimeError("construction failed")
|
|
return provider
|
|
|
|
manager = ProviderRuntimeManager(
|
|
_settings(),
|
|
runtime_factory=lambda settings, admission_registry: ProviderRuntime(
|
|
settings,
|
|
admission_registry,
|
|
provider_constructor=construct,
|
|
),
|
|
)
|
|
generation = manager.current_generation_id
|
|
try:
|
|
result = await manager.refresh_provider("nvidia_nim")
|
|
assert result.failed_provider_ids == ("nvidia_nim",)
|
|
if refresh:
|
|
result = await manager.refresh_provider("nvidia_nim")
|
|
assert result.refreshed_provider_ids == ("nvidia_nim",)
|
|
async with await manager.acquire() as lease:
|
|
assert await lease.resolve_provider("nvidia_nim") is provider
|
|
assert (lease.model_info("nvidia_nim", "one") is not None) is refresh
|
|
assert manager.current_generation_id == generation
|
|
assert attempts == 2
|
|
finally:
|
|
await manager.close()
|
|
provider.cleanup.assert_awaited_once_with()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("cancel", [False, True])
|
|
async def test_recovery_wait_uses_remaining_budget_without_orphaning_waiter(
|
|
monkeypatch, cancel
|
|
):
|
|
entered, release = asyncio.Event(), asyncio.Event()
|
|
provider = MagicMock(spec=BaseProvider)
|
|
attempts = 0
|
|
construction = None
|
|
|
|
async def construct(_id, _settings, _admission_registry, _api_key_number):
|
|
nonlocal attempts, construction
|
|
attempts += 1
|
|
if attempts == 1:
|
|
raise RuntimeError("construction failed")
|
|
construction = asyncio.current_task()
|
|
entered.set()
|
|
await release.wait()
|
|
return provider
|
|
|
|
manager = ProviderRuntimeManager(
|
|
_settings(),
|
|
runtime_factory=lambda settings, admission_registry: ProviderRuntime(
|
|
settings,
|
|
admission_registry,
|
|
provider_constructor=construct,
|
|
),
|
|
)
|
|
wait = InitializationWait(1 if cancel else 0.01)
|
|
monkeypatch.setattr(
|
|
"free_claude_code.runtime.provider_manager.InitializationWait", lambda: wait
|
|
)
|
|
lease = await manager.acquire()
|
|
waiting = None
|
|
try:
|
|
result = await manager.refresh_provider("nvidia_nim")
|
|
assert result.failed_provider_ids == ("nvidia_nim",)
|
|
existing_tasks = asyncio.all_tasks()
|
|
waiting = asyncio.create_task(lease.resolve_provider("nvidia_nim"))
|
|
await asyncio.wait_for(entered.wait(), 1)
|
|
if cancel:
|
|
waiting.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await waiting
|
|
else:
|
|
with pytest.raises(ApplicationUnavailableError, match="still starting"):
|
|
await asyncio.wait_for(asyncio.shield(waiting), 1)
|
|
assert construction is not None and not construction.done()
|
|
assert asyncio.all_tasks() <= existing_tasks | {construction}
|
|
release.set()
|
|
await construction
|
|
wait.remaining = 0
|
|
assert await lease.resolve_provider("nvidia_nim") is provider
|
|
assert attempts == 2
|
|
finally:
|
|
release.set()
|
|
if waiting is not None:
|
|
waiting.cancel()
|
|
await asyncio.gather(waiting, return_exceptions=True)
|
|
await lease.release()
|
|
await manager.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cancellation_drains_the_actual_worker():
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
def worker():
|
|
entered.set()
|
|
release.wait()
|
|
|
|
task = asyncio.create_task(run_sync_owned(worker))
|
|
try:
|
|
await asyncio.to_thread(entered.wait)
|
|
task.cancel()
|
|
await asyncio.sleep(0)
|
|
assert not task.done()
|
|
finally:
|
|
release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_late_retired_discovery_cannot_replace_current_metadata():
|
|
entered, release = asyncio.Event(), asyncio.Event()
|
|
providers = []
|
|
|
|
def factory(settings, admission_registry):
|
|
provider = MagicMock(spec=BaseProvider)
|
|
providers.append(provider)
|
|
ordinal = len(providers)
|
|
|
|
async def models():
|
|
if ordinal == 1:
|
|
entered.set()
|
|
await release.wait()
|
|
return frozenset({ProviderModelInfo(f"model-{ordinal}")})
|
|
|
|
provider.list_model_infos = AsyncMock(side_effect=models)
|
|
|
|
async def construct(
|
|
_provider_id, _settings, _admission_registry, _api_key_number
|
|
):
|
|
return provider
|
|
|
|
return ProviderRuntime(
|
|
settings,
|
|
admission_registry,
|
|
provider_constructor=construct,
|
|
)
|
|
|
|
manager = ProviderRuntimeManager(_settings(), runtime_factory=factory)
|
|
old = await manager.acquire()
|
|
waiting = asyncio.create_task(old.resolve_provider("nvidia_nim"))
|
|
new = None
|
|
try:
|
|
await entered.wait()
|
|
await manager.replace(_settings(), commit=AsyncMock())
|
|
new = await manager.acquire()
|
|
await new.resolve_provider("nvidia_nim")
|
|
release.set()
|
|
await waiting
|
|
assert old.model_info("nvidia_nim", "model-1") is not None
|
|
assert new.model_info("nvidia_nim", "model-2") is not None
|
|
assert manager.cached_model_info("nvidia_nim", "model-1") is None
|
|
finally:
|
|
release.set()
|
|
await asyncio.gather(waiting, return_exceptions=True)
|
|
await old.release()
|
|
if new:
|
|
await new.release()
|
|
await manager.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_complete_catalog_publication_preserves_prior_file_until_all_attempts(
|
|
tmp_path,
|
|
):
|
|
entered, release = asyncio.Event(), asyncio.Event()
|
|
path = tmp_path / "catalog.json"
|
|
path.write_text("prior complete catalog", encoding="utf-8")
|
|
provider = MagicMock(spec=BaseProvider)
|
|
|
|
async def models():
|
|
entered.set()
|
|
await release.wait()
|
|
return frozenset({ProviderModelInfo("new-model")})
|
|
|
|
provider.list_model_infos = AsyncMock(side_effect=models)
|
|
|
|
async def construct(_provider_id, _settings, _admission_registry, _api_key_number):
|
|
return provider
|
|
|
|
manager = ProviderRuntimeManager(
|
|
_settings(),
|
|
runtime_factory=lambda settings, admission_registry: ProviderRuntime(
|
|
settings,
|
|
admission_registry,
|
|
provider_constructor=construct,
|
|
),
|
|
model_catalog_publisher=CodexModelCatalogPublisher(path),
|
|
)
|
|
waiter = asyncio.create_task(manager.wait_for_catalog())
|
|
try:
|
|
await entered.wait()
|
|
assert not waiter.done()
|
|
assert path.read_text(encoding="utf-8") == "prior complete catalog"
|
|
release.set()
|
|
catalog = await waiter
|
|
assert catalog.cached_model_info("nvidia_nim", "new-model") is not None
|
|
assert "new-model" in path.read_text(encoding="utf-8")
|
|
finally:
|
|
release.set()
|
|
await asyncio.gather(waiter, return_exceptions=True)
|
|
await manager.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("retained_request", [False, True])
|
|
@pytest.mark.parametrize("catalog_file", [False, True])
|
|
async def test_catalog_wait_follows_replacement(
|
|
tmp_path, retained_request, catalog_file
|
|
):
|
|
entered, release = asyncio.Event(), asyncio.Event()
|
|
count = 0
|
|
|
|
def runtime(settings, admission_registry):
|
|
nonlocal count
|
|
count += 1
|
|
old = count == 1
|
|
provider = MagicMock(spec=BaseProvider)
|
|
|
|
async def models():
|
|
if old:
|
|
entered.set()
|
|
await release.wait()
|
|
return frozenset({ProviderModelInfo("old" if old else "current")})
|
|
|
|
provider.list_model_infos = AsyncMock(side_effect=models)
|
|
|
|
async def construct(_id, _settings, _admission_registry, _api_key_number):
|
|
return provider
|
|
|
|
return ProviderRuntime(
|
|
settings,
|
|
admission_registry,
|
|
provider_constructor=construct,
|
|
)
|
|
|
|
path = tmp_path / "catalog.json"
|
|
manager = ProviderRuntimeManager(
|
|
_settings(),
|
|
runtime_factory=runtime,
|
|
model_catalog_publisher=CodexModelCatalogPublisher(path),
|
|
)
|
|
lease = await manager.acquire() if retained_request else None
|
|
request = (
|
|
asyncio.create_task(lease.resolve_provider("nvidia_nim")) if lease else None
|
|
)
|
|
waiting = asyncio.create_task(
|
|
manager.wait_for_catalog_file(InitializationWait())
|
|
if catalog_file
|
|
else manager.wait_for_catalog()
|
|
)
|
|
try:
|
|
await entered.wait()
|
|
settings = _settings().model_copy(update={"model": "nvidia_nim/current"})
|
|
await manager.replace(settings, commit=AsyncMock())
|
|
await manager.wait_for_catalog()
|
|
result = await asyncio.wait_for(asyncio.shield(waiting), 1)
|
|
if catalog_file:
|
|
assert isinstance(result, int)
|
|
assert result == manager.current_generation_id
|
|
assert "nvidia_nim/current" in path.read_text(encoding="utf-8")
|
|
else:
|
|
assert not isinstance(result, int)
|
|
assert result.settings is settings
|
|
assert result.cached_model_info("nvidia_nim", "current") is not None
|
|
if request is not None:
|
|
assert lease is not None
|
|
assert not request.done()
|
|
release.set()
|
|
await request
|
|
assert lease.model_info("nvidia_nim", "old") is not None
|
|
finally:
|
|
release.set()
|
|
waiting.cancel()
|
|
await asyncio.gather(waiting, return_exceptions=True)
|
|
if request is not None:
|
|
await asyncio.gather(request, return_exceptions=True)
|
|
if lease is not None:
|
|
await lease.release()
|
|
await manager.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("catalog_file", [False, True])
|
|
@pytest.mark.parametrize("cancel", [False, True])
|
|
async def test_catalog_waiter_exit_preserves_shared_discovery(
|
|
monkeypatch, catalog_file, cancel
|
|
):
|
|
existing_tasks = asyncio.all_tasks()
|
|
entered, release = asyncio.Event(), asyncio.Event()
|
|
provider = MagicMock(spec=BaseProvider)
|
|
|
|
async def models():
|
|
entered.set()
|
|
await release.wait()
|
|
return frozenset({ProviderModelInfo("one")})
|
|
|
|
provider.list_model_infos = AsyncMock(side_effect=models)
|
|
|
|
async def construct(_id, _settings, _admission_registry, _api_key_number):
|
|
return provider
|
|
|
|
manager = ProviderRuntimeManager(
|
|
_settings(),
|
|
runtime_factory=lambda settings, admission_registry: ProviderRuntime(
|
|
settings,
|
|
admission_registry,
|
|
provider_constructor=construct,
|
|
),
|
|
)
|
|
wait = InitializationWait(1 if cancel else 0.01)
|
|
monkeypatch.setattr(
|
|
"free_claude_code.runtime.provider_manager.InitializationWait", lambda: wait
|
|
)
|
|
waiting = asyncio.create_task(
|
|
manager.wait_for_catalog_file(wait)
|
|
if catalog_file
|
|
else manager.wait_for_catalog()
|
|
)
|
|
try:
|
|
await entered.wait()
|
|
if cancel:
|
|
waiting.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await waiting
|
|
else:
|
|
with pytest.raises(ApplicationUnavailableError, match="still starting"):
|
|
await waiting
|
|
monkeypatch.setattr(
|
|
"free_claude_code.runtime.provider_manager.InitializationWait",
|
|
InitializationWait,
|
|
)
|
|
release.set()
|
|
snapshot = await manager.wait_for_catalog()
|
|
assert snapshot.cached_model_info("nvidia_nim", "one") is not None
|
|
finally:
|
|
release.set()
|
|
waiting.cancel()
|
|
await asyncio.gather(waiting, return_exceptions=True)
|
|
await manager.close()
|
|
assert asyncio.all_tasks() <= existing_tasks
|