From 00ecdf4c94c8aa40c5294f1ae1d03a8e6cf7a565 Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:02:58 +0300 Subject: [PATCH 01/12] fix: preserve dialogue through model outages and restore primary routes Use configured alternatives after eligible unknown outcomes, retain same-operation custody, and let cognition return to or wait on its primary binding. Keep resource waits honest and interruptible across direct and queued work. --- docs/DESIGN.md | 9 +- docs/architecture/06-agent-core.md | 23 +- docs/development/06-rules-by-change-class.md | 22 +- docs/inventories/FACADE_INVENTORY.md | 4 +- ouroboros/llm_claudexor.py | 26 +- ouroboros/loop.py | 71 ++- ouroboros/loop_llm_call.py | 62 +-- ouroboros/loop_model_call.py | 262 +++++++++-- ouroboros/loop_transport.py | 211 +++++---- ouroboros/model_slots.py | 21 + ouroboros/model_wait.py | 61 ++- ouroboros/provider_models.py | 10 +- ouroboros/size_ratchet_manifest.py | 1 + ouroboros/tools/control.py | 11 +- ouroboros/tools/control_runtime.py | 32 +- tests/test_llm_claudexor.py | 2 +- tests/test_loop_compaction.py | 1 + tests/test_loop_transport_wait.py | 177 +++---- tests/test_loop_transport_wait_interactive.py | 31 +- tests/test_model_wait.py | 21 +- tests/test_model_wait_browser.py | 48 ++ tests/test_model_wait_controls.py | 2 + tests/test_module_handle_extraction.py | 4 +- ...test_native_fallback_binding_acceptance.py | 309 ++++++++++++ tests/test_quota_rotation_fallback.py | 90 +++- tests/test_transport_death_retry.py | 35 +- tests/test_transport_repeat_owner_stop.py | 2 + tests/test_transport_unknown_continuation.py | 22 +- .../test_transport_wait_repeat_interaction.py | 104 ++-- tests/test_unknown_fallback_first.py | 445 ++++++++++++++++++ web/modules/model_roles.js | 2 +- web/modules/model_wait.js | 29 +- web/tests/model_wait.test.js | 10 +- 33 files changed, 1646 insertions(+), 514 deletions(-) create mode 100644 tests/test_native_fallback_binding_acceptance.py create mode 100644 tests/test_unknown_fallback_first.py diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 0e8ceeaed..ec59b546e 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -1001,9 +1001,12 @@ Permanent group-wide Local changes remain in Models. A pool confirmed to contain both accounts requiring sign-in and accounts waiting for quota says "Waiting for access" and names both causes. It keeps automatic continuation and opens Accounts without selecting a profile or initiating login. -Unknown pool availability is not this state. The confirmed quota component uses -the same execution-clock pause; an authentication-only wait still consumes that -clock, and calendar deadlines stay fixed. +A dated pool refusal without a confirmed cause says "Waiting for an account", +retains the engine's reset forecast and keeps automatic continuation without +claiming quota exhaustion or guaranteed availability. It never offers sign-in. +Only confirmed quota pauses the execution clock; authentication and unconfirmed +availability waits consume it. Calendar deadlines remain fixed. Configured +fallback routes are tried before a wait card is shown. A submitted action is shown as pending until the task reports its application. A saved Settings change and a still-pending task change are disclosed separately. diff --git a/docs/architecture/06-agent-core.md b/docs/architecture/06-agent-core.md index 6fe7a03bb..35d1389ac 100644 --- a/docs/architecture/06-agent-core.md +++ b/docs/architecture/06-agent-core.md @@ -158,11 +158,11 @@ Ouroboros stores one provider-neutral, function-shaped conversation. Before requ Retry budgets are failure-class specific: empty/incomplete responses and transient 429/5xx failures may retry the same model with deadline-bounded backoff; auth, quota, permanent bad requests and confirmed oversize fail fast; exhausted compatibility returns to the configured model fallback chain rather than becoming a second router. `LLMClient` keeps leading system messages authoritative and demotes later notices to visibly marked user notices; on the OpenAI family and the Claudexor route (`llm_claudexor._request`) the LEADING system message's builder-declared mutable blocks are also projected to one such notice placed before the task (prompt caching, below). -Transport failures are classified by physical-attempt custody, each class with its own owner and rails. A REMOTE pre-dispatch failure is `transport_unavailable` (`loop_llm_call.classify_llm_exception`: released custody, $0, non-local provider), one physical attempt per call; pacing belongs to the round-level wait episode (`ouroboros/loop_transport.py`), which tries the fallback chain once only when `USE_LOCAL_FALLBACK` makes it local (remote candidates never dial over a proven dead egress), waits (durable `network_wait` events, progress keeping the idle rail alive, owner-interruptible backoff), then redials the SAME round for free. A managed task waits as long as its existing rails allow — owner deadline minus the dispatch-admission reserve, budget, Stop, absolute ceiling; never a new setting, and `OUROBOROS_TRANSIENT_RETRY_MAX` bounds only transient PROVIDER failures — and exhaustion is its terminal result. The interactive class (every direct-chat-stamped turn: owner chat, Presence turns, consciousness wake-ups) waits the same way without queue rails and ordinarily without an owner deadline, so the task idle timeout (`OUROBOROS_TASK_IDLE_TIMEOUT_SEC`), measured from each episode's entry, bounds its episode: it limits idle WAITING only, never cancels in-flight work, the shorter of it and an explicit deadline binds, the durable `ended` detail names the expired rail (`interactive_wait_window_exhausted`, or the deadline's own), and exhaustion emits an owner note beside the provider-outage terminal. Closing `network_wait` rows exist only for cooperative exits (an external kill, Panic, crash or direct-turn hard stop leaves none), so correlate a confirmed `task_done` or durable terminal result; silence never proves completion. +Physical custody classifies failures. A remote pre-dispatch failure is `transport_unavailable` (`loop_llm_call.classify_llm_exception`: released, $0, non-local), one physical attempt per call. One provider's failure does not prove dead egress: `loop_model_call._recover_failed_round` tries configured alternatives, remote included, before the existing `loop_transport.py` episode waits and redials the SAME round. Alternatives remain eligible during paced recovery; no one-pass limit is imposed. The episode owns interruptible backoff, durable `network_wait` rows and progress that keeps the idle rail alive. Managed work retains budget, Stop, owner deadline minus dispatch reserve and absolute ceiling; `OUROBOROS_TRANSIENT_RETRY_MAX` bounds transient PROVIDER failures only. Direct-chat-stamped turns (owner chat, Presence, consciousness) instead bound idle WAITING by `OUROBOROS_TASK_IDLE_TIMEOUT_SEC` from episode entry, or a shorter explicit deadline. A dispatched response remains eligible; the wait bound never cancels in-flight cognition. Expiry takes the no-resend terminal, with `interactive_wait_window_exhausted` or the deadline's detail; direct turns also receive an owner note. Closing rows prove cooperative exits only: external kill, Panic, crash or hard Stop require a confirmed terminal result, never inference from silence. -The mid-flight class is `provider_outcome_unknown`: a dispatched request with no terminal provider fact keeps its unresolved monetary bound and is never resent. A NEW logical request afterwards needs a unique host-attested input, granted per caller class. Ordinary managed tasks and native API children use the upstream-observation continuation (§6 Review delivery), whose wait admits a marked new physical attempt on observed connectivity. A configured-session supervisor with exactly one live delegated leaf latches a durable unknown-provider hold (`ouroboros/delegate_hold.py`, closing any transport episode as `network_wait ended / hold_latched`) and parks in the ordinary `supervised_wait`, resuming in a NEW round only on a meaningful leaf wake whose receipt is that input (control wakes and an ineligible hold grant nothing; budget admission stays fail-closed on the unresolved bound); without a hold it uses the managed continuation. Interactive primary rounds get the bounded transport-death repeat below: no-effect by construction (no Host-executed tool or correspondent action can stem from the failed attempt; priced read-only provider-owned retrieval may rerun; §12), not an event retry. Every other surface — forced-final, fallback-chain candidates, review actors, safety, probes, web search, consolidation/summary/reflection, external-harness delegated runs — keeps no-resend (budget 0, or never enters `call_llm_with_retry`); an unknown outcome not typed as a transport death is never resent. +`provider_outcome_unknown` retains the dispatched attempt's unresolved monetary bound; the same request is not resent. `loop_transport.new_generation_after_unknown` admits a NEW, possibly charged task request only after receiving has ended without a usable answer or local producer and no same-operation rejoin is viable. Engine-terminal unknown and ended API header/body reads, including timeouts, qualify; remote computation may still run. Failed Claudexor reads or lost create replies remain `same_operation_recoverable`, reusing the operation or idempotency key. Main/Project, queued/native-child and consciousness cognition tries alternatives, then existing observation/wait continuation; an unknown fallback may advance too. `append_unknown_recovery_input` binds each new attempt to the old id/route/unknown cost. Only the current accepted message executes tools; late bytes stay in CAS/accounting, and provider retrieval may rerun. One live delegated leaf makes its configured supervisor enter `delegate_hold.py` (`network_wait ended / hold_latched`): only a meaningful leaf wake supplies new-round input; controls, ineligible holds and unresolved budget grant no resend. Without a hold it uses ordinary recovery. Inline Presence retains its bounded typed-death repeat below and otherwise its unknown no-resend terminal (§12). Forced-final, review, safety, probes, search, consolidation/reflection and external Agent runs retain their own no-resend contracts. -The transport-death repeat rail: a typed death (`transport_custody.is_retryable_transport_death` — httpx `ReadError`/`WriteError`/`RemoteProtocolError` through the explicit `__cause__` chain, or a requests wrapper carrying `ProtocolError`/`RemoteDisconnected`; never a timeout, a provider status/body error, a pre-dispatch failure, a local provider or a loopback route) lets the PRIMARY main-loop round dispatch alone (`_dispatch_round_model`, `transport_death_retries`) repeat the SAME logical request at most twice per round. Each repeat is a NEW physical attempt with its own ledger row, and the earlier rows stay `unresolved` at their upper bound (a fully dead round reserves up to three: honest accounting over a cheaper rail). A round record (`execution_id:round:round_idx`) counts the repeats and names the class the terminal reports; `llm_non_retryable_same_request` marks only the exhaustion. A current typed `finalize_now` control or the deadline can refuse a not-yet-sent repeat (`finalize_control_pending` names the control case); the round then exits through the unknown no-resend terminal. A round holding a repeat record sends nothing but those typed-death repeats: a repeat failing with any other class ends the round on the same terminal — no transient burst, empty-response retry, compaction retry, forced-final dial, fallback chain or local-only pass — because the earlier request may still be live; only a repeat that never left the host (released custody) stays the free wait episode's to redial, and when that window closes the terminal is still `provider_outcome_unknown_no_resend`. +Inline Presence alone opts its primary round into the transport-death repeat (`_dispatch_round_model`, `transport_death_retries`). `transport_custody.is_retryable_transport_death` accepts httpx ReadError/WriteError/RemoteProtocolError through explicit causes, or requests ProtocolError/RemoteDisconnected; timeout, provider status/body error, pre-dispatch failure, local and loopback are excluded. At most two repeats of the same logical request create separate ledger attempts, leaving up to three unresolved upper bounds. The execution/round-keyed record counts repeats and preserves their failure class; `llm_non_retryable_same_request` marks exhaustion. Pending `finalize_now` (`finalize_control_pending`) or deadline refuses unsent repeats. While that record remains, only its typed-death repeats may send: any other failed response ends without transient/empty/compaction retries, forced final or fallback. A released repeat alone can redial through its free wait, whose exhaustion still reports `provider_outcome_unknown_no_resend`. A usable response clears the record. Retry rails nest, each with its own owner and bound; the table exists so the multiplication is visible in one place (a physical send is always its own `execute_physical_attempt` lifecycle, whichever rail asked for it): @@ -170,9 +170,10 @@ Retry rails nest, each with its own owner and bound; the table exists so the mul |---|---|---|---| | SDK `max_retries` | client construction (`net_transport.py`, `llm.py`) | 0 everywhere | nothing: every physical send is visible to the ledger | | request-wire recovery ladder | `request_wire_recovery.py` inside one `LLMClient.chat` | at most eight typed compatibility actions per call | a rejected request in a corrected wire shape | -| transport-death repeat | `call_llm_with_retry`, primary round dispatch only | ≤2 per round, 4 s then 8 s deadline-aware backoff | a dispatched request that died with a typed transport death, as a new attempt | +| transport-death repeat | `call_llm_with_retry`, inline Presence primary round only | ≤2 per round, 4 s then 8 s deadline-aware backoff | a dispatched request that died with a typed transport death, as a new attempt | +| configured-route walk | `loop_model_call._run_cross_model_fallback_chain` | one attempt per route per failed round, none inside a wait episode | the round on each other configured binding, the primary first when a fallback acts | | transient burst | `call_llm_with_retry` | `OUROBOROS_TRANSIENT_RETRY_MAX` attempts per call (the outer ceiling of the same loop) | typed 408/429/5xx and empty/incomplete responses | -| round redial | `loop_transport.py` wait episode | a managed task: owner deadline, budget, Stop, absolute ceiling; an interactive turn: the task idle timeout measured from episode entry (`OUROBOROS_TASK_IDLE_TIMEOUT_SEC`), or an explicit deadline when that is shorter | a released ($0) pre-dispatch failure of the same round | +| round redial | `loop_transport.py` wait episode | a managed task: owner deadline, budget, Stop, absolute ceiling; an interactive turn: the task idle timeout measured from episode entry (`OUROBOROS_TASK_IDLE_TIMEOUT_SEC`), or an explicit deadline when that is shorter | a released ($0) pre-dispatch failure of the same round, or an eligible unknown as a marked new attempt | | served-model redo | `llm_substitution.py` inside one `chat_claudexor` | `OUROBOROS_SERVED_MODEL_REDOS` per call, and none under a pin, an admitted candidate, a spent send budget or a spent owner window | a round the engine says another model answered, as a new operation. It nests inside the rails above, so a repeat granted by one of them starts a call whose redo budget begins again | | review physical rail | `review_substrate.py`, `review_native_episode.py` | 2 sends per packet or session actor on the P3/acceptance surfaces; a native-retrieval slot carries no send count (its bounds are the episode's own, enumerated below); no rail elsewhere | a released review send, never an unknown one | @@ -264,7 +265,7 @@ The startup seal audit selects its bounded manifest set before reading live atte `claudexor::=` selects a model operation in the Ouroboros-owned engine, not an Agent Run: `LLMClient` dispatches sync and async calls through `llm_claudexor.py` before the OpenAI-compatible filtering/retries, and Ouroboros keeps its SYSTEM, BIBLE, canonical messages, tool selection and execution. The raw-model adapter is Codex; connected Claude/Cursor and other harnesses keep their Agent capabilities, and direct API keys keep their existing routes. Every main-loop execution of one install sends the same Codex cache key (`llm_claudexor.cache_key_for_model`: per data root and model, projected to `prompt_cache_key` and the `session_id` header), so a new task, child or consciousness cycle is served the governance prefix its predecessors already cached; per-execution turn states ride `nativeContinuation` unchanged under that shared session. -`ClaudexorGateway` uses purpose-bound byte uploads, one idempotent operation ID, status/result reads, cancellation and explicit result acknowledgement. Claudexor performs at most one generation per operation: a lost local HTTP reply rejoins the same operation, and an unknown provider outcome never authorizes another generation. It shares its managed profile with Agent sessions — the adapter reads current access credentials while the official CLI keeps refresh ownership — and no shared external engine, relay process, copied operator auth store or public OpenAI-compatible endpoint. Metadata helpers use `read_owned_gateway` (owned-only discovery and handshake; they never install or start the engine); actual model calls and explicit lifecycle actions use `ensure_owned_gateway`. +`ClaudexorGateway` uses purpose-bound byte uploads, one idempotent operation ID, status/result reads, cancellation and explicit result acknowledgement. Claudexor performs at most one generation per operation: a lost local HTTP reply rejoins the same operation, and an unknown provider outcome never makes the engine generate again; a host NEW generation after an eligible unknown is a new operation. It shares its managed profile with Agent sessions — the adapter reads current access credentials while the official CLI keeps refresh ownership — and no shared external engine, relay process, copied operator auth store or public OpenAI-compatible endpoint. Metadata helpers use `read_owned_gateway` (owned-only discovery and handshake; they never install or start the engine); actual model calls and explicit lifecycle actions use `ensure_owned_gateway`. One pre-create catalog read negotiates `captureFailureEvidence=true`/`captureEffortEvidence=true` on `POST /v2/model-operations`; invocation manifests freeze both for same-key rejoin. Unadvertised flags stay absent; runtime pinning remains independent. Supporting engines retain all failed-response bytes (incomplete/HTTP refusals too) and processing exceptions privately. CAS retains exact result bytes before ACK; GET never projects another view. Usage/events show compact problem context; a received prefix cannot prove the missing tail. Coherent provider terminals with unusable messages keep outcome/route/usage: `response_rejected` settles usage then raises non-repeatable `stream_rejected`, without unknown recovery or marking healthy accounts failed. After classification only, `ClaudexorModelError.display_message` adds sanitized typed `vendorCode`/`parameter` to error events/terminal previews; unknown outcomes expose no provider details. @@ -286,21 +287,25 @@ Model roles bind accounts/windows through `OUROBOROS_MODEL_ACCOUNTS` and `OUROBO Host-authored sampling defaults stay separate from explicit parameters: review requests carry `default_temperature` as a hint each effective API route resolves at dispatch, raw model operations use their provider default, and an explicit `temperature` (zero included) is preserved and may be refused. Routes without structured output use the existing safety text-JSON parser, repair and refusal path — no new safety bypass. Ordinary confirmed provider errors keep each helper's previous handling, while quota, owner interruption and unknown physical custody propagate without becoming a helper result or verdict. +#### Configured routes and primary return + +Route identity (`model_slots.route_binding`) includes model, locality and configured/owner-selected account or Auto; equal model ids can name different alternatives. The turn's primary binding (task override or role slot, role, locality; `loop._initial_round_route`) stays an alternative when an adopted fallback fails (`loop_model_call._route_candidates`), without duplicating Main in Settings; routes provide cognition until a model can decide. After a host-driven switch away from the primary the acting model gets one `[ROUTE FACTS]` notice (both bindings, the dated failure, any engine reset) and chooses stay, `switch_model(primary="return")`, or `primary="wait"`. The latter retains resource/transport waits instead of paid alternatives. Direct API refusals use Retry/Switch/Stop without subscription observation, paid probes or automatic-access claims. Return restores the primary role/fit and effort intent, retaining Auto; the next real request tests it. No timer, reset or metadata proves generation success (BIBLE P5/P13). + #### Quota and auth waits -`model_wait.py` holds confirmed quota/auth waits inside the live call. Typed owner actions use the existing mailbox and the `/api/decisions` family `model_wait::`; task-result rows are projections, not restartable stack checkpoints or a second attempt ledger; metadata polling makes no generation. Auto may prefer the last successful same-route account unless it got a status-null or typed per-subject refusal this execution. A named account's HTTP vendor quota refusal is re-asked with the same payload and no preference, a pre-dispatch verdict never; only its `poolCause` proves the pool spent; an account chosen twice stops rotation, claiming nothing. Pin never rotates. A route may also answer with ANOTHER model than requested, with no error and nothing in the account's quota: the engine states it as a typed fact on its completed result, and the transport never accepts it as the round. Retained and acknowledged like any answer (its would-be horizon stays reconstructible), it adopts no turn state, runs no tool call and is replaced by a NEW operation naming no account; the engine, which deprioritises that account for a while, alone places every redo. `OUROBOROS_SERVED_MODEL_REDOS` bounds redos; none follows a pin (one named account), an admitted candidate (exact bytes), a bounded budget with no send left (a redo would surface that rail's budget error, not this refusal) or a spent owner window (no further send). An unknown outcome is never a substitution: the transport raises it before the rail sees a result, so the existing unknown-custody fence, not this budget, governs what follows. Each discarded generation's retention/acknowledgement state travels with its durable row (failure visible, not assumed) and teaches the density witness nothing: a witness belongs to its producing model. With no redo left, the typed `model_substituted` refusal reaches the caller — an ordinary cross-model fallback trigger for a root, a typed failure for an exact-route child — and never cools the requested model: one substituting account says nothing about the model. A task's first substitution of each model also becomes one timeline row of its card. The rule covers this transport only; Agent-session delegation is a separate engine capability with its own model evidence. Fallback keeps one attempt per model and a 120-second cooldown; on quota every configured route runs before the owner wait, opened from the retained refusal (no generation first); an intermediate fallback's own refusal stays the wait/retry route if a later one fails differently; nothing sleeps to a reset; inline Presence never waits (typed refusal; safe retry proof: §12). A temporary switch affects only the wait; Settings persistence, acceptance and task application stay distinct. Fallback Local is group-wide: a one-role persistence request may change model/account only with Local unchanged, a different Local choice is task-only, and permanent group changes stay in Models. The current task-attempt identity selects actionable wait rows; a failed mailbox delivery retries the same accepted request, authoring none; terminal mailbox cleanup waits for post-task synthesis and pending accepted attachment promotion to settle. +`model_wait.py` holds confirmed quota, sign-in and engine-dated pool waits inside the live call; a dated non-mixed pool is `unavailable`, never relabelled quota (undated stays the generic transport path). Typed owner actions use the existing mailbox and the `/api/decisions` family `model_wait::`; task-result rows are projections, not restartable stack checkpoints or a second attempt ledger; metadata polling makes no generation. Auto may prefer the last successful same-route account unless it got a status-null or typed per-subject refusal this execution. A named account's HTTP vendor quota refusal is re-asked with the same payload and no preference, a pre-dispatch verdict never; only its `poolCause` proves the pool spent; an account chosen twice stops rotation, claiming nothing. Pin never rotates. A route may also answer with ANOTHER model than requested, with no error and nothing in the account's quota: the engine states it as a typed fact on its completed result, and the transport never accepts it as the round. Retained and acknowledged like any answer (its would-be horizon stays reconstructible), it adopts no turn state, runs no tool call and is replaced by a NEW operation naming no account; the engine, which deprioritises that account for a while, alone places every redo. `OUROBOROS_SERVED_MODEL_REDOS` bounds redos; none follows a pin (one named account), an admitted candidate (exact bytes), a bounded budget with no send left (a redo would surface that rail's budget error, not this refusal) or a spent owner window (no further send). An unknown outcome is never a substitution: the transport raises it before the rail sees a result, so the existing unknown-custody fence, not this budget, governs what follows. Each discarded generation's retention/acknowledgement state travels with its durable row (failure visible, not assumed) and teaches the density witness nothing: a witness belongs to its producing model. With no redo left, the typed `model_substituted` refusal reaches the caller — an ordinary cross-model fallback trigger for a root, a typed failure for an exact-route child — and never cools the requested model: one substituting account says nothing about the model. A task's first substitution of each model also becomes one timeline row of its card. The rule covers this transport only; Agent-session delegation is a separate engine capability with its own model evidence. Fallback keeps one attempt per model and a 120-second cooldown; on every resource refusal (quota, sign-in, `unavailable`) every configured route runs before the owner wait, opened from the retained refusal (no generation first), and a needed sign-in stays named in the switch note; an intermediate fallback's own refusal stays the wait/retry route if a later one fails differently; nothing sleeps to a reset, even with no configured route; inline Presence never waits (typed refusal; safe retry proof: §12). A temporary switch affects only the wait; Settings persistence, acceptance and task application stay distinct. Fallback Local is group-wide: a one-role persistence request may change model/account only with Local unchanged, a different Local choice is task-only, and permanent group changes stay in Models. The current task-attempt identity selects actionable wait rows; a failed mailbox delivery retries the same accepted request, authoring none; terminal mailbox cleanup waits for post-task synthesis and pending accepted attachment promotion to settle. `ouroboros/gateway/task_model_wait.py` owns model-wait decision effects and bounded settings-writer receipts; `task_decision.answer_decision` returns `(status, payload)` to both Web and authenticated Host Service transports (they supply the installation root — no synthetic HTTP request or parallel owner registry); `supervisor/task_model_wait.py` projects worker events and quota clocks through the existing supervisor; `ouroboros/gateway/decision_contracts.py` owns the decision-family vocabulary. Quota pauses use union duration across simultaneous waits and consume no internal execution time, while explicit calendar deadlines stay fixed; the worker and project-write ownership stay held (a one-worker queue waits), completed tools stay on the live stack, reviewer waits belong to their operation through settlement (§6 Task acceptance), closing a browser never stops the wait, and stopping the Ouroboros process ends this continuation guarantee. Image tools retain the tracked VLM child: `vision_process.py` carries typed errors, physical capture, cancellation and operation checkpoints across that process boundary; a lost child result stays unknown, never free. A model-wait row naming a review operation reaches only its exact live controller through canonical `pending_action`, never the ended author's mailbox: `review_operation_controller` validates panel, slot, attempt and open pointer, and a remote controller also needs the same pid birth and custody session with no Stop/Panic or unreadable stop state. Web/Host ingress returns 202 only if that controller consumed the claim or still holds it live, else 409. `supervisor.task_model_wait` admits current-revision operation events off the queue lock without relabeling the author live; a decision affects neither sibling panels nor the author. -A consciousness wake-up (`handle_wake_direct`) shares an owner turn's direct-turn registration, model wait, Stop and bounded interactive transport-death repeats and yields an ordinary task result, with no background owner, locked memory, pause or stop event of its own. +A consciousness wake-up (`handle_wake_direct`) shares an owner turn's direct-turn registration, model wait, Stop and unknown-outcome recovery and yields an ordinary task result, with no background owner, locked memory, pause or stop event of its own. #### Streams and transport waits Main remote completions stream (`llm_stream.py`), assembled inside the physical send closure before accounting settles: a complete stream yields the same normalized response shape as JSON; a partial stream never yields a usable tool call or answer, and its exact wire bytes and partial assembly stay in private CAS through the `physical_stream` manifest and attempt id. The assembler is strict about completeness, tolerant about form: on the Chat path identity scalars and metadata keep their first value, index gaps are forgiven and every forgiven fact is disclosed in the stream receipt (`anomalies`); the native Messages assembler rejects post-terminal shape (non-contiguous blocks, an unsigned thinking block, an incomplete tool block). A complete-but-unusable stream settles with its usage as a provider error (`stream_rejected`); unknown-outcome continuation is reserved for a stream that never reached its terminal frame (a lost socket, a malformed mid-stream native frame), and a complete response with absent final usage still has unknown money. Comments/pings do not define cognitive deadlines; every recovery candidate checks inherited calendar and quota-adjusted execution bounds before reservation, after preparation and at dispatch, and HTTP phase bounds stay distinct from the overall logical wait. -An ordinary managed task whose provider outcome becomes unknown stays in the transport-wait episode (`loop_transport.py`); the old attempt and its unreported cost remain unknown. After non-generating upstream observation, a user-role `[SYSTEM NOTICE]` supplies explicit recovery input for one new physical attempt under the existing budget, cancellation, owner deadline and absolute ceiling. An attempt that fails again, is unknown after dispatch or released before it — and a free redial that crosses dispatch and dies unknown — returns to the same episode: the backoff keeps growing (4→60 s), one redial is counted per wait iteration, an unknown repeat re-arms the probe's freshness bound at that latest failure and refreshes custody, one `continued` row names the transition (`continuation_outcome_unknown`, `continuation_transport_unavailable`, `redial_outcome_unknown`), and no cap on continuations — budget, deadline and Stop are the only stops. Finished tools are retained. Configured-session supervisors prefer their live-leaf hold (`delegate_hold.py`), then use this same recovery for their own model; reading a result or disposing a patch is not a condition for cognition. Direct turns keep their separate contract; manual Restart/Panic gains no resume authority. Recovery is proved by provider provenance, never by a local handshake: direct remote endpoints are observed through HEAD under the same no-proxy policy, reusing the ordinary connection allowance for every socket phase narrowed by the owner remainder and holding no cognitive in-flight lease; subscription metadata qualifies only when the catalog reports generic `provenance="provider_http"` with an original `observedAt` after wait entry for the selected source/model and effective profile/account fingerprint — cached reuse never advances that timestamp, and a static or pre-outage catalog cannot prove recovery. Capability discovery remains authoritative, not a new core provider table. Loss of the Claudexor control connection keeps reading the same accepted model operation across endpoint rediscovery, without creating another operation. +After alternatives fail, cognition retains one `loop_transport.py` episode. A fresh non-generating upstream observation can admit a real task continuation with `[SYSTEM NOTICE]`; it proves connectivity, never the old outcome or future success. Repeated unknown/released attempts keep the episode clock, growing 4→60 s backoff and one redial per wait iteration. Unknown repeats refresh custody and the observation lower bound, with `continuation_outcome_unknown`, `continuation_transport_unavailable` or `redial_outcome_unknown` evidence. Alternatives remain eligible; no new count limit or mandatory Retry. Finished tools survive; budget, Stop, deadline, absolute ceiling and direct interactive waiting bounds remain. Configured-session supervisors prefer their live-leaf hold; patch disposition is not cognition authority, and manual Restart/Panic grants no resume. Direct API connectivity uses HEAD under ordinary no-proxy rules, each phase bounded by the connection allowance and owner remainder, without a cognitive lease. Subscription catalog evidence needs `provenance="provider_http"`, original `observedAt` after the failure and matching selected source/model/account fingerprint; cache reads never refresh that timestamp. Prior-attempt evidence never pins an unrelated next route. Capability discovery retains its existing owner. Ordinary direct and queued control-link/create/status/result-read failures, including generic 404, rejoin the same operation or create key across endpoint rediscovery. They do not prove non-dispatch, and cannot start another generation while the existing operation remains recoverable. ### Delegated subagents (Claudexor transport + the nanny) diff --git a/docs/development/06-rules-by-change-class.md b/docs/development/06-rules-by-change-class.md index 818428f4e..f991e2a41 100644 --- a/docs/development/06-rules-by-change-class.md +++ b/docs/development/06-rules-by-change-class.md @@ -586,13 +586,12 @@ Settings, accounts and shared controls. Tests: `test_owner_settings_write_seam.p existence, preview or matching prompt prose alone grants authority — missing/partial/error/mismatched custody never buys another same-operation dispatch. -- Managed unknown-outcome recovery uses the existing transport-wait owner - (`loop_transport.py`) with non-generating upstream observations (what proves - recovery and what cannot: ARCHITECTURE §6 "Caller-owned subscription model - calls"): keep the old outcome/cost unknown, apply current - budget/Stop/deadline before dispatch, and let a control-channel outage first - rejoin the same accepted operation. No scheduler, provider/model table, paid - readiness probe or automatic manual-restart recovery is introduced. +- Unknown-outcome recovery is typed (`loop_transport.new_generation_after_unknown`), + not silence: configured routes, then transport-wait observations (ARCHITECTURE §6); + inline Presence retains its contract. Old outcome/cost stay unknown; + new attempts get their own identity and facts-only input; a control-link or read + failure only rejoins (`same_operation_recoverable`). + No scheduler, pass counter or paid probe (`test_unknown_fallback_first.py`). - `delegate_wait` supervision's observation beat is separate from its HTTP read allowance, and a typed read-only-retryable transport failure is a quiet observation hole, not a wake (the per-class reasons and the once-per-episode @@ -650,7 +649,8 @@ and what enforces each. Density evidence never crosses models. Proven no-send returns the local attempt claim even if bounded ledger release fails; the reservation and capture then remain. Unknown or dispatched claims stay charged. -- Resource refusals wait inside the live call, before helper catch-all blocks, on the +- Resource refusals try configured routes first, never sleep to a reset, and + wait inside the live call, before helper catch-all blocks, on the existing task owner, mailbox, clocks and settings writer. Reviewer calls use their panel's operation wait (§6 Already-paid operations), preserving original task money and controls. Reprepare wake input against the new route's tools/capacity/reserve @@ -797,7 +797,7 @@ and what enforces each. alone to exceed the window); quota/auth/billing, hard bad-request and request-too-large are non-retryable as-is (exact category, recovery hint); a typed 408/429/5xx or a proven pre-dispatch failure may retry; a dispatched request with no - terminal outcome stops same-model and cross-model sends until reconciled. Who may + terminal outcome is never resent as is; only an eligible unknown permits a NEW generation. Who may repeat after a typed transport death, how often, on whose row, what ends the round: ARCHITECTURE §6 "Context fitting, retry, and compaction" (`tests/test_transport_death_retry.py`). Call-site rules: decide `retry_same_request` @@ -809,8 +809,8 @@ and what enforces each. untouched); generic transient/empty-response backoffs keep their contract. A budget refusal does NOT un-count: the budget rail cannot prove the repeat never left the host (`llm.chat` retries on the wire before a later reservation can refuse), so the - attempt stays booked and the budget terminal ends the round. Every caller outside the - interactive primary rail keeps `transport_death_retries=0`; no + attempt stays booked and the budget terminal ends the round. Every caller outside + inline Presence's primary rail keeps `transport_death_retries=0`; no consumed/terminal/patch-disposition predicate gates a session supervisor's cognition, and a successful live-leaf hold closes any prior transport episode so its acknowledged wake alone resumes the model. diff --git a/docs/inventories/FACADE_INVENTORY.md b/docs/inventories/FACADE_INVENTORY.md index e4b276195..e5ab1c8a5 100644 --- a/docs/inventories/FACADE_INVENTORY.md +++ b/docs/inventories/FACADE_INVENTORY.md @@ -2,7 +2,7 @@ AST-derived inventory of compatibility facades, regenerated by `python scripts/regenerate_inventories.py`. Do not edit. A facade row is any runtime module whose top-level `from import ...` statements carry the `noqa: F401` re-export marker — the codebase's declared "this binding exists for its binding, not for this module's own use" convention (reference FACADE_CONSUMERS method). Leaf domains come from `ouroboros/domains.toml`; a leaf outside the facade's domain is marked ✗ (that edge also appears in the manifest's pinned direction matrix). `tests/test_generated_inventories.py` pins byte-identity, so any re-export surface change must regenerate this file. -- facade modules: **62**; marked re-export bindings: **2347**; cross-domain facade→leaf pairs: **131** +- facade modules: **62**; marked re-export bindings: **2349**; cross-domain facade→leaf pairs: **131** | facade | domain | bindings | leaves | |---|---|---:|---| @@ -19,7 +19,7 @@ AST-derived inventory of compatibility facades, regenerated by `python scripts/r | `ouroboros/gateway/tasks.py` | D11 | 7 | `ouroboros/gateway/cost_breakdown.py` (1)
`ouroboros/gateway/task_decision.py` (1)
`ouroboros/gateway/task_events.py` (4)
`ouroboros/gateway/task_hurry.py` (1) | | `ouroboros/headless.py` | D17 | 45 | `ouroboros/contracts/task_constraint.py` (1 ✗D19)
`ouroboros/headless_status.py` (11)
`ouroboros/workspace_patch_capture.py` (20)
`ouroboros/workspace_patch_rules.py` (13) | | `ouroboros/llm.py` | D02 | 92 | `ouroboros/anthropic_native_custody.py` (7)
`ouroboros/context_budget.py` (2 ✗D03)
`ouroboros/llm_anthropic.py` (2)
`ouroboros/llm_attempt.py` (22)
`ouroboros/llm_capability_policy.py` (5)
`ouroboros/llm_fallback.py` (1)
`ouroboros/llm_gigachat.py` (1)
`ouroboros/llm_local.py` (7)
`ouroboros/llm_messages.py` (1)
`ouroboros/llm_openai_compatible.py` (5)
`ouroboros/llm_pricing.py` (4)
`ouroboros/llm_routing.py` (3)
`ouroboros/openrouter_attribution.py` (1)
`ouroboros/provider_models.py` (7)
`ouroboros/request_wire_recovery.py` (7)
`ouroboros/transport_custody.py` (1)
`ouroboros/usage_accounting.py` (14 ✗D16)
`ouroboros/utils.py` (2 ✗D18) | -| `ouroboros/loop.py` | D01 | 261 | `ouroboros/config.py` (6 ✗D12)
`ouroboros/context.py` (1 ✗D03)
`ouroboros/context_budget.py` (1 ✗D03)
`ouroboros/context_compaction.py` (2 ✗D03)
`ouroboros/context_fit.py` (2 ✗D03)
`ouroboros/deadline_utils.py` (2)
`ouroboros/llm.py` (3 ✗D02)
`ouroboros/loop_acceptance.py` (21)
`ouroboros/loop_acceptance_review.py` (29)
`ouroboros/loop_budget.py` (14)
`ouroboros/loop_delivery.py` (29)
`ouroboros/loop_forced_finalization.py` (22)
`ouroboros/loop_llm_call.py` (6)
`ouroboros/loop_messages.py` (11)
`ouroboros/loop_model_call.py` (20)
`ouroboros/loop_nudges.py` (16)
`ouroboros/loop_round_limits.py` (17)
`ouroboros/loop_tool_execution.py` (5)
`ouroboros/loop_transport.py` (10)
`ouroboros/nanny_pacing.py` (4 ✗D07)
`ouroboros/observability.py` (1 ✗D16)
`ouroboros/outcomes.py` (17)
`ouroboros/pricing.py` (1 ✗D02)
`ouroboros/review_cycles.py` (1 ✗D06)
`ouroboros/task_finalization.py` (3)
`ouroboros/task_pacing.py` (1)
`ouroboros/tool_policy.py` (4 ✗D04)
`ouroboros/usage_accounting.py` (5 ✗D16)
`ouroboros/utils.py` (3 ✗D18)
`supervisor/owner_stop.py` (4 ✗D09) | +| `ouroboros/loop.py` | D01 | 263 | `ouroboros/config.py` (6 ✗D12)
`ouroboros/context.py` (1 ✗D03)
`ouroboros/context_budget.py` (1 ✗D03)
`ouroboros/context_compaction.py` (2 ✗D03)
`ouroboros/context_fit.py` (2 ✗D03)
`ouroboros/deadline_utils.py` (2)
`ouroboros/llm.py` (3 ✗D02)
`ouroboros/loop_acceptance.py` (21)
`ouroboros/loop_acceptance_review.py` (29)
`ouroboros/loop_budget.py` (14)
`ouroboros/loop_delivery.py` (29)
`ouroboros/loop_forced_finalization.py` (22)
`ouroboros/loop_llm_call.py` (6)
`ouroboros/loop_messages.py` (11)
`ouroboros/loop_model_call.py` (22)
`ouroboros/loop_nudges.py` (16)
`ouroboros/loop_round_limits.py` (17)
`ouroboros/loop_tool_execution.py` (5)
`ouroboros/loop_transport.py` (10)
`ouroboros/nanny_pacing.py` (4 ✗D07)
`ouroboros/observability.py` (1 ✗D16)
`ouroboros/outcomes.py` (17)
`ouroboros/pricing.py` (1 ✗D02)
`ouroboros/review_cycles.py` (1 ✗D06)
`ouroboros/task_finalization.py` (3)
`ouroboros/task_pacing.py` (1)
`ouroboros/tool_policy.py` (4 ✗D04)
`ouroboros/usage_accounting.py` (5 ✗D16)
`ouroboros/utils.py` (3 ✗D18)
`supervisor/owner_stop.py` (4 ✗D09) | | `ouroboros/loop_acceptance.py` | D01 | 5 | `ouroboros/loop_messages.py` (5) | | `ouroboros/loop_acceptance_review.py` | D01 | 3 | `ouroboros/acceptance_retrieving.py` (3) | | `ouroboros/loop_llm_call.py` | D01 | 2 | `ouroboros/config.py` (2 ✗D12) | diff --git a/ouroboros/llm_claudexor.py b/ouroboros/llm_claudexor.py index 27157a2fa..191a2adbd 100644 --- a/ouroboros/llm_claudexor.py +++ b/ouroboros/llm_claudexor.py @@ -98,7 +98,7 @@ class ClaudexorModelError(RuntimeError): """A model-operation fact with its exact role, route and recovery identity.""" def __init__(self, problem: dict, *, model_role: str = "", operation_id: str = "", - route: dict | None = None, unknown: bool = False): + route: dict | None = None, unknown: bool = False, same_operation_recoverable: bool = False): self.problem = copy.deepcopy(problem) self.code = "model_outcome_unknown" if unknown else str(problem.get("code") or "model_operation_failed") super().__init__(f"{self.code}: {problem.get('message') or 'Model operation did not complete'}") @@ -117,7 +117,7 @@ class ClaudexorModelError(RuntimeError): # rejection: no unknown outcome or same-request wire repair. self.stream_rejected = self.stream_incomplete = True self.retryable = False - self.model_role = model_role + self.model_role, self.same_operation_recoverable = model_role, same_operation_recoverable # the latter: a READ failure; rejoin, never regenerate self.operation_id = operation_id self.route = copy.deepcopy(route or {}) @@ -467,12 +467,11 @@ class _ModelInvocation: except Exception as error: log.warning("Model custody observer unavailable: %s", type(error).__name__) - def error(self, problem: dict | None, detail: dict | None = None, *, unknown: bool = False): - detail = detail or {} - route = (detail.get("dispatch") or {}).get("route") or {} + def error(self, problem: dict | None, detail: dict | None = None, *, unknown: bool = False, rejoinable: bool = False): + route = ((detail or {}).get("dispatch") or {}).get("route") or {} cls = ClaudexorModelError if unknown else ClaudexorModelNotDispatched - return cls(problem or {"code": "model_operation_failed", "message": "The engine returned no model result."}, - model_role=self.role, operation_id=self.operation_id, route=route, unknown=unknown) + return cls(problem or {"code": "model_operation_failed", "message": "The engine returned no model result."}, model_role=self.role, + operation_id=self.operation_id, route=route, unknown=unknown, same_operation_recoverable=rejoinable) def receive(self) -> dict: outage_started = None @@ -537,21 +536,22 @@ class _ModelInvocation: "context": {"httpStatus": error.status_code}}) from None # A failed read after creation is never a provider connect failure. # Drop its causal HTTP chain at this boundary: even ConnectError - # means only the local control read failed, not inference un-sent. + # means only the local control read failed, not inference un-sent (a refused read, 404 too). if error.code != "daemon_unreachable" and not 500 <= error.status_code < 600: - raise self.error({"code": error.code, "message": str(error)}, detail, unknown=True) from None + if self._control_outage(): # rejoin by operation id OR the same create idempotency key + continue + raise self.error({"code": error.code, "message": str(error)}, detail, unknown=True, rejoinable=self.create_attempted) from None if outage_started is None: outage_started = time.monotonic() if self._control_outage(): continue if time.monotonic() - outage_started >= self.timeout: - raise self.error({"code": "model_control_unreachable", "message": "Control connection lost; the same model operation may still finish."}, detail, unknown=True) from None + raise self.error({"code": "model_control_unreachable", "message": "Control connection lost; the same model operation may still finish."}, detail, unknown=True, rejoinable=True) from None time.sleep(min(config.CLAUDEXOR_MODEL_POLL_INTERVAL_SEC, self.timeout)) def _control_outage(self, *, recovered: bool = False) -> bool: - """Managed calls keep the same accepted operation through local HTTP loss.""" - from ouroboros.loop_transport import ( - TransportWaitEpisode, emit_network_wait_event, managed_transport_continuation) + """Ordinary direct and queued calls keep the same operation through control loss.""" + from ouroboros.loop_transport import TransportWaitEpisode, emit_network_wait_event, managed_transport_continuation waiter = current_model_wait() ctx = getattr(waiter, "tool_context", None) if not managed_transport_continuation(ctx): diff --git a/ouroboros/loop.py b/ouroboros/loop.py index 3a2c93c81..7ac8f2c2b 100644 --- a/ouroboros/loop.py +++ b/ouroboros/loop.py @@ -63,11 +63,11 @@ from ouroboros.loop_transport import ( continue_unknown_transport as _continue_unknown_transport, TransportWaitEpisode, # noqa: F401 -- the loop module keeps its historical import surface for the L-B leaves end_episode_budget as _end_episode_budget, # noqa: F401 -- the loop module keeps its historical import surface for the L-B leaves - fallback_chain_allowed as _fallback_chain_allowed, + fallback_chain_allowed as _fallback_chain_allowed, # noqa: F401 -- resolved by loop_model_call._recover_failed_round at call time finalize_now_transport_terminal as _finalize_now_transport_terminal, # noqa: F401 -- the loop module keeps its historical import surface for the L-B leaves last_assistant_text as _last_assistant_text, provider_terminal_fallback_text as _provider_terminal_fallback_text, - reconcile_transport_wait as _reconcile_transport_wait, + reconcile_transport_wait as _reconcile_transport_wait, # noqa: F401 -- resolved by loop_model_call._recover_failed_round at call time task_deadline_epoch as _task_deadline_epoch, # noqa: F401 -- the loop module keeps its historical import surface for the L-B leaves transport_wait_step as _transport_wait_step, ) @@ -207,7 +207,7 @@ def _provider_unavailable_result( # in flight — no record, sticky kind `provider_outcome_unknown` — told the owner # its outcome was unknown while stamping `transport_unavailable_no_resend` on the # trace (#869). One question, one answer, on all three surfaces. - unknown_outcome = record or str( + unknown_outcome = record or bool(ctx.accumulated_usage.get("_pending_transport_outcome")) or str( ctx.accumulated_usage.get("_last_llm_error_kind") or "") == "provider_outcome_unknown" is_transport_wait = wait_cause == "transport_unavailable" is_context_overflow = (kind == "context_overflow" and not (record or is_transport_wait) @@ -362,14 +362,19 @@ def _reset_turn_state(ctx: Any) -> None: ctx._delivery_candidate, ctx._delivery_candidate_revision, ctx._delivery_control_required = None, 0, False ctx._delivery_evidence_revision, ctx._delivery_evidence_fingerprint = 0, "" ctx.model_turn_state, ctx._authoring_handover, ctx._pending_model_wait_handover = ModelTurnState(), None, None + ctx.route_wait_on_primary, ctx.active_role_override, ctx._route_facts_pending = False, None, "" def _initial_round_route(ctx: Any, llm: LLMClient, initial_effort: str) -> tuple: """The route this turn opens on: model, effort, local flag and context modes. Unknown routes get one honest call; no synthetic short-window capacity, so a - fit plan is adopted only while it still answers the preferred mode. + fit plan is adopted only while it still answers the preferred mode. The same + binding is recorded as the turn's primary (task override or role slot, role, + locality) for configured-route recovery and ``switch_model(primary=...)``. """ + from ouroboros.model_slots import task_model_binding + task_model_override = str(getattr(ctx, "task_model_override", "") or "").strip() local_override = getattr(ctx, "task_use_local_override", None) preferred_mode = get_context_mode() @@ -379,10 +384,12 @@ def _initial_round_route(ctx: Any, llm: LLMClient, initial_effort: str) -> tuple active_context_mode = str(getattr(context_fit_plan, "initial_mode", "") or preferred_mode) else: active_context_mode = preferred_mode - return (task_model_override or llm.default_model(), initial_effort, - (bool(local_override) if local_override is not None else - runtime_setting("USE_LOCAL_MAIN", "").lower() in ("true", "1")), - preferred_mode, active_context_mode, context_fit_plan) + model = task_model_override or llm.default_model() + use_local = (bool(local_override) if local_override is not None else + runtime_setting("USE_LOCAL_MAIN", "").lower() in ("true", "1")) + ctx.primary_route = {"model": model, "use_local": use_local, "role": task_model_binding( + {"task_metadata": getattr(ctx, "task_metadata", {})}, context_fit_plan=context_fit_plan)[0]} + return model, initial_effort, use_local, preferred_mode, active_context_mode, context_fit_plan def run_llm_loop( @@ -461,22 +468,10 @@ def run_llm_loop( ctx = tools._ctx if not pending_tool_budget: - _prev_active_route = (active_model, active_use_local) - active_model, active_use_local, active_effort = _apply_runtime_overrides( - ctx, active_model, active_use_local, active_effort, - ) - if (active_model, active_use_local) != _prev_active_route: - context_fit_plan, active_context_mode = _rebind_context_fit_plan( - context_fit_plan, tools, messages, model=active_model, - use_local=active_use_local, preferred_mode=_preferred_context_mode, - tool_schemas=tool_schemas, - ) - if active_model != _prev_active_route[0]: - # Cross-FAMILY switch: discard provider-private reasoning - # signatures before the new route sees them (same family is a no-op). - _sanitized = LLMClient.sanitize_reasoning_on_model_switch(messages, _prev_active_route[0], active_model) - if _sanitized is not messages: - messages[:] = _sanitized + (active_model, active_use_local, active_effort, context_fit_plan, + active_context_mode) = _apply_round_route_overrides( + ctx, tools, messages, (active_model, active_use_local, active_effort), context_fit_plan, + active_context_mode, _preferred_context_mode, tool_schemas) ctx.active_context_mode = active_context_mode ctx.active_model = active_model ctx.active_effort = active_effort @@ -593,26 +588,12 @@ def run_llm_loop( task_id=task_id, emit_progress=emit_progress, transport_episode=transport_wait): transport_wait = None # The leaf wake owns resumption, without a provider probe. continue - transport_wait = _reconcile_transport_wait( - transport_wait, ctx, msg_present=msg is not None, error_kind=last_error_kind, - drive_logs=drive_logs, task_id=task_id, model=active_model, emit_progress=emit_progress) - if msg is None and _fallback_chain_allowed(ctx, last_error_kind, transport_wait, accumulated_usage): - _episode_before_chain = transport_wait is not None - (msg, active_model, active_use_local, - context_fit_plan, active_context_mode) = _run_cross_model_fallback_chain( - llm=llm, ctx=ctx, tools=tools, messages=messages, active_model=active_model, - active_use_local=active_use_local, tool_schemas=tool_schemas, active_effort=active_effort, - max_retries=max_retries, drive_logs=drive_logs, task_id=task_id, round_idx=round_idx, - event_queue=event_queue, accumulated_usage=accumulated_usage, task_type=task_type, - emit_progress=emit_progress, context_fit_plan=context_fit_plan, - active_context_mode=active_context_mode) - # Post-chain reconcile with the FRESH kind: a MID-chain outage - # latches too (see reconcile_transport_wait's docstring). - transport_wait = _reconcile_transport_wait( - transport_wait, ctx, msg_present=msg is not None, - error_kind=str(accumulated_usage.get("_last_llm_error_kind") or ""), - drive_logs=drive_logs, task_id=task_id, model=active_model, - emit_progress=emit_progress, after_local_pass=_episode_before_chain) + limit_ctx.active_model, limit_ctx.active_use_local = active_model, active_use_local + # Configured routes before any wait; an active episode owns its own outcome. + (msg, active_model, active_use_local, context_fit_plan, active_context_mode, + transport_wait) = _recover_failed_round( + limit_ctx, tools, msg, transport_wait, context_fit_plan=context_fit_plan, + active_context_mode=active_context_mode, emit_progress=emit_progress) # A wait-card switch can change the route within this very call. # Delivery/finalization in the same round must use that applied route. limit_ctx.active_model = ctx.active_model = active_model @@ -792,6 +773,8 @@ from ouroboros.loop_nudges import ( # noqa: E402, F401 -- intentional public re ) from ouroboros.loop_model_call import ( # noqa: E402, F401 -- intentional public re-exports _adopt_fallback_route, + _apply_round_route_overrides, + _recover_failed_round, _snapshot_context_fit_usage, _restore_context_fit_usage, _run_cross_model_fallback_chain, diff --git a/ouroboros/loop_llm_call.py b/ouroboros/loop_llm_call.py index 6aa5403db..9e220ce2a 100644 --- a/ouroboros/loop_llm_call.py +++ b/ouroboros/loop_llm_call.py @@ -19,14 +19,13 @@ from ouroboros.llm_observability import persist_observed_call from ouroboros.config import get_finalization_grace_sec, NETWORK_WAIT_BACKOFF_MAX_SEC # noqa: F401 - legacy monkeypatch seam from ouroboros.deadline_utils import ( owner_deadline_exhausted, - seconds_until, main_transport_timeout_sec as _main_transport_timeout, ) from ouroboros.knowledge import observed_route_stamp from ouroboros.llm import LLMClient, LocalContextTooLargeError, add_usage from ouroboros.llm_claudexor import propagate_model_error, presence_refusal_unstarted from ouroboros.llm_substitution import same_route_refusal, stamp_substitutions -from ouroboros.model_wait import current_model_wait, model_wait_reason, propagate_model_control +from ouroboros.model_wait import model_wait_reason, propagate_model_control from ouroboros.openai_chat_dispatch import CUSTOM_RECEIPTS_USAGE_KEY, pop_custom_validation_receipts from ouroboros.llm_attempt import PROVIDER_POLICY_REFUSAL, _is_provider_policy_refusal # typed-refusal contract owner from ouroboros.observability import new_call_id, new_execution_id, persist_call @@ -146,8 +145,8 @@ _COOLDOWN_ERROR_KINDS = _TRANSIENT_RETRY_KINDS | frozenset({"rate_limit"}) # A subscription window that is spent but heals on a timer. SSOT for the name. # Deliberately NOT `quota_exhausted`: that class is classified PERMANENT, which is # correct for a billing refusal (402, no credits) and wrong for a plan window whose -# only cure is waiting. Scheduling follows `reset_at` (never for a confirmed quota refusal, which -# model_wait recovers, nor inline Presence), so a six-hour window never becomes sixty 1-minute retries. +# only cure is waiting. Its `reset_at` is a disclosed fact, never a sleep: configured +# routes answer first, then model_wait's visible wait (or inline Presence's typed refusal). SUBSCRIPTION_WINDOW_EXHAUSTED = "subscription_window_exhausted" _TRANSIENT_RETRY_DEFAULT = 6 _TRANSIENT_BACKOFF_CAP_SEC = NETWORK_WAIT_BACKOFF_MAX_SEC @@ -157,10 +156,10 @@ _TRANSIENT_BACKOFF_CAP_SEC = NETWORK_WAIT_BACKOFF_MAX_SEC _DEADLINE_RETRY_FLOOR_SEC = 10.0 # Bounded PAID repeat of a dispatched request whose socket died with a typed # transport death (transport_custody.is_retryable_transport_death — never a -# timeout, status, body error or pre-dispatch failure): the primary main-loop -# round dispatch alone opts in, at most this many extra physical sends per ROUND -# (≤3 unresolved upper bounds), each its own ledger attempt; backoff by death -# ordinal. Every other caller keeps the default 0 = the no-resend doctrine. +# timeout, status, body error or pre-dispatch failure): an inline Presence turn's +# primary round dispatch alone opts in, at most this many extra physical sends per +# ROUND (≤3 unresolved upper bounds), each its own ledger attempt; backoff by death +# ordinal. Every other caller passes 0: its new generation is the round's recovery. _TRANSPORT_DEATH_RETRIES = 2 _TRANSPORT_DEATH_BACKOFF_SEC = (4.0, 8.0) # Round-keyed repeat counter on the shared usage dict ({"round_id", "count"}): @@ -303,17 +302,8 @@ def _cooldown_kind_for_empty_response(body_error: Dict[str, Any], event_type: st return body_kind if body_kind in _COOLDOWN_ERROR_KINDS else event_type -def _retry_backoff_sec(accumulated_usage: Dict[str, Any], error_kind: str, attempt: int, is_transient: bool) -> float: - """Seconds to wait before retrying the same request. - - A KNOWN reset instant wins over guesswork: a spent subscription window is scheduled - against its own ``reset_at``, never through the 60s-capped exponential, so a - six-hour window never becomes sixty one-minute retries. ``_sleep_within_deadline`` - then honestly refuses when the task deadline cannot absorb that wait. - """ - retry_after = accumulated_usage.get("_last_llm_retry_after_sec") - if error_kind == SUBSCRIPTION_WINDOW_EXHAUSTED and retry_after is not None: - return max(0.0, float(retry_after)) +def _retry_backoff_sec(attempt: int, is_transient: bool) -> float: + """Seconds before retrying the same request; no reset instant ever schedules one.""" return min(2.0 ** attempt * 4, _TRANSIENT_BACKOFF_CAP_SEC if is_transient else 30.0) @@ -718,13 +708,12 @@ def classify_llm_exception(exc: Exception, safe_error: str = "") -> LlmErrorClas return LlmErrorClassification("auth_error", False, _exception_status_code(exc), exc.code) if isinstance(exc, LocalContextTooLargeError): return LlmErrorClassification("context_overflow", False) - # Structured fact, not a keyword scan (Bible P5): a transport that KNOWS its - # window is spent carries the typed code plus the reset instant; a DATED credential - # pool heals on the same timer (its code stays as evidence), an undated one never gets here. + # A reset forecast is evidence, not proof that an unavailable pool exhausted quota. wcode, reset_at = str(getattr(exc, "code", "") or ""), str(getattr(exc, "reset_at", "") or "") - if wcode == SUBSCRIPTION_WINDOW_EXHAUSTED or (wcode == "credential_pool_exhausted" and reset_at): - return LlmErrorClassification(SUBSCRIPTION_WINDOW_EXHAUSTED, True, _exception_status_code(exc), - "" if wcode == SUBSCRIPTION_WINDOW_EXHAUSTED else wcode, seconds_until(reset_at), reset_at) + if wcode == SUBSCRIPTION_WINDOW_EXHAUSTED: + return LlmErrorClassification(wcode, False, _exception_status_code(exc), reset_at=reset_at) + if wcode == "credential_pool_exhausted" and reset_at: + return LlmErrorClassification("provider_error", False, _exception_status_code(exc), wcode, reset_at=reset_at) status_code = _exception_status_code(exc) provider_code = _exception_provider_code(exc, safe) # Typed refusal (llm_attempt.ProviderPolicyRefusal): nothing upstream answered, @@ -932,11 +921,13 @@ def _record_llm_call_error( if classification.kind == "provider_outcome_unknown": ctx.accumulated_usage["_pending_transport_outcome"] = { **custody_fields, "model": ctx.model, "operation_id": str(getattr(error, "operation_id", "") or ""), + "model_role": str(getattr(error, "model_role", "") or ""), "route": dict(getattr(error, "route", {}) or {}), "outcome": "unknown", "request_ref": ctx.request_ref.get("manifest_ref") if ctx.request_ref else None, + **({"same_operation_recoverable": True} if getattr(error, "same_operation_recoverable", False) else {}), } - # Ordinary managed tasks require upstream recovery before a new attempt; - # other callers retain their bounded repeat rail. + # Only a caller with a repeat budget (inline Presence) repeats here; every other + # new generation belongs to the round's recovery (configured routes, then the wait). if ( is_retryable_transport_death(error) and repeats < ctx.transport_death_retries and ctx.attempt < ctx.transient_budget - 1 @@ -984,11 +975,9 @@ def _record_llm_call_error( }) ctx.accumulated_usage.update(_last_llm_error=_short_error_text(display_error), _last_llm_resource_refusal=model_wait_reason(error), _last_llm_error_kind=classification.kind, _last_llm_retry_same_request=will_retry) - if classification.retry_after_sec is not None: - ctx.accumulated_usage.update(_last_llm_retry_after_sec=classification.retry_after_sec, - _last_llm_reset_at=classification.reset_at) + if classification.reset_at: # a disclosed fact for the terminal hint, never a retry schedule + ctx.accumulated_usage["_last_llm_reset_at"] = classification.reset_at else: - ctx.accumulated_usage.pop("_last_llm_retry_after_sec", None) ctx.accumulated_usage.pop("_last_llm_reset_at", None) for key, value in (("_last_llm_provider_message", provider_message), ("_last_llm_status_code", classification.status_code), ("_last_llm_provider_code", classification.provider_code)): @@ -1029,10 +1018,10 @@ def _stop_after_llm_error(ctx: _LlmErrorContext) -> bool: is_transient = error_kind in _TRANSIENT_RETRY_KINDS # Non-transient retryables: max_retries capped by the loop ceiling (primary: no-op). attempt_budget = ctx.transient_budget if is_transient else min(ctx.max_retries, ctx.transient_budget) - backoff = ( # a confirmed resource refusal and inline Presence neither resend nor sleep to a reset - _retry_backoff_sec(accumulated_usage, error_kind, ctx.attempt, is_transient) + backoff = ( # a resource refusal or spent window never resends nor sleeps to a reset + _retry_backoff_sec(ctx.attempt, is_transient) if ctx.attempt < attempt_budget - 1 and not accumulated_usage.get("_last_llm_resource_refusal") - and (error_kind != SUBSCRIPTION_WINDOW_EXHAUSTED or getattr(current_model_wait(), "waits_allowed", True)) else None + and error_kind != SUBSCRIPTION_WINDOW_EXHAUSTED else None ) if backoff is not None: if _sleep_within_deadline(backoff, ctx.deadline_ts): @@ -1060,7 +1049,8 @@ def provider_no_call_source(accumulated_usage: Dict[str, Any], deadline_exhauste rail keeps its grace call, so ``deadline_exhausted`` suppresses the wall. A round still holding an unresolved attempt (its transport-death record: written at a grant, cleared only by a usable response) forbids the resend whatever the sticky kind.""" - if str(accumulated_usage.get("_last_llm_error_kind") or "") == "provider_outcome_unknown" or isinstance(accumulated_usage.get(TRANSPORT_DEATHS_KEY), dict): + if (str(accumulated_usage.get("_last_llm_error_kind") or "") == "provider_outcome_unknown" + or accumulated_usage.get("_pending_transport_outcome") or isinstance(accumulated_usage.get(TRANSPORT_DEATHS_KEY), dict)): return "provider_outcome_unknown_no_resend", False if accumulated_usage.get("resource_refusal"): return "resource_refusal_no_resend", True # typed temporary refusal: no route or owner wait recovered it if same_route_refusal(accumulated_usage): return "same_route_refusal_no_resend", False # a salvage call is the same request on the same route, and the accounts that refused it were just marked @@ -1516,7 +1506,7 @@ def call_llm_with_retry( if _empty_response_wall_spent(is_provider_glitch, permanent_body_error, usage): accumulated_usage[RETRY_WALL_EXHAUSTED_KEY] = True return None, cost - for stale in ("execution_status", "result_status", "reason_code", RETRY_WALL_EXHAUSTED_KEY, TRANSPORT_DEATHS_KEY): + for stale in ("execution_status", "result_status", "reason_code", RETRY_WALL_EXHAUSTED_KEY, TRANSPORT_DEATHS_KEY, "_pending_transport_outcome"): accumulated_usage.pop(stale, None) # a USABLE response closes the round's repeat record accumulated_usage["rounds"] = accumulated_usage.get("rounds", 0) + 1 cached_tokens = int(usage.get("cached_tokens") or 0) diff --git a/ouroboros/loop_model_call.py b/ouroboros/loop_model_call.py index a17f4701c..140a25fa1 100644 --- a/ouroboros/loop_model_call.py +++ b/ouroboros/loop_model_call.py @@ -33,7 +33,8 @@ log = logging.getLogger("ouroboros.loop") # The typed, temporary non-success of a round whose resource refusal no route recovered and # no owner wait follows (inline Presence, or a chain stopped by its own fence): nothing more is sent. RESOURCE_REFUSAL_KEY = "resource_refusal" -_CHAIN_STOP_KINDS = ("provider_outcome_unknown", "deadline_exhausted", "transport_unavailable") +# A round failure that owns a wait episode on its own route when no configured route answers. +_ROUND_WAIT_KINDS = ("transport_unavailable", "provider_outcome_unknown") def _loop(): @@ -164,16 +165,77 @@ def _restore_context_fit_usage( usage.update(snapshot) +def _route_candidates(tool_ctx: Any, active_model: str, active_use_local: bool, acting_role: str, + waiter: Any) -> List[Tuple[str, str, bool, bool]]: + """Configured routes other than the acting binding: ``(model, role, use_local, is_primary)``. + + The task's primary binding comes first whenever another route is acting (a failing + adopted fallback may return to it; the owner never duplicates Main in the chain), then + the ordered chain. Identity is the complete binding (``model_slots.route_binding``), so + a chain entry with Main's model on another account is a real alternative. + """ + from ouroboros.config import fallback_candidate_targets + from ouroboros.model_slots import route_binding + + overrides = waiter.overrides if waiter is not None else None + chain_local = runtime_setting("USE_LOCAL_FALLBACK", "").lower() in ("true", "1") + primary = getattr(tool_ctx, "primary_route", None) + rows = ([(primary["model"], primary["role"], bool(primary["use_local"]), True)] + if isinstance(primary, dict) and primary.get("model") else []) + rows += [(target.model_id, f"fallback:{index}", chain_local, False) + for index, target in enumerate(fallback_candidate_targets(preserve_slots=True))] + role = acting_role or (primary["role"] if rows and rows[0][3] and rows[0][0] == active_model else "main") + seen, candidates = {route_binding(active_model, active_use_local, role, overrides=overrides)}, [] + for model, row_role, use_local, is_primary in rows: + binding = route_binding(model, use_local, row_role, overrides=overrides) + if binding not in seen: + seen.add(binding) + candidates.append((model, row_role, use_local, is_primary)) + return candidates + + +def _route_facts_text(tool_ctx: Any, *, model: str, use_local: bool, role: str, failed_model: str, + failure: Dict[str, Any], waiter: Any) -> str: + """Facts after a host-driven switch away from the primary; the acting model decides what follows.""" + from ouroboros.model_slots import route_binding + from ouroboros.provider_models import provider_for_model + + primary = getattr(tool_ctx, "primary_route", None) + if not isinstance(primary, dict) or not primary.get("model"): + return "" + + def label(name: str, local: bool, route_role: str) -> str: + account = route_binding(name, local, route_role, overrides=waiter.overrides if waiter else None)[2] + managed = not local and provider_for_model(name) == "claudexor" + account_text = f"; account {account or 'Auto'}" if managed else "" + return f"{name}{' (local)' if local else ''} (role {route_role}{account_text})" + + reset = str(failure.get("reset_at") or "") + return (f"[ROUTE FACTS] This task's primary route is {label(primary['model'], bool(primary['use_local']), primary['role'])}; " + f"{label(model, use_local, role)} answered this round after {failed_model} failed with " + f"{failure.get('failure_code') or 'an error'} at {failure.get('ts') or 'an unrecorded time'}" + f"{f' (its engine dated a reset at {reset})' if reset else ''}. Facts, not an instruction: stay on this " + "route, return with switch_model(primary=\"return\") (the next real request tests the primary; a refusal " + "there moves on to configured routes again), or return and wait for it with switch_model(primary=\"wait\"). " + "No catalog check or timer shows whether the primary serves now.") + + def _run_cross_model_fallback_chain( *, llm, ctx, tools, messages, active_model, active_use_local, tool_schemas, active_effort, max_retries, drive_logs, task_id, round_idx, event_queue, accumulated_usage, task_type, emit_progress, context_fit_plan, active_context_mode, ) -> tuple: - """Try fallbacks; unknown dispatch stops the chain.""" + """Try the configured routes other than the acting binding before any wait begins. + + An eligible unknown outcome moves on to the next route with facts-only recovery + input (a real, possibly charged generation); an ineligible one (an accepted operation still readable, + inline Presence) or a spent deadline stops the walk. A round whose own failure is an + outage or an unknown outcome keeps that failure as its wait when none answers. + """ from ouroboros import fallback_cooldown as _fcd - from ouroboros.config import fallback_candidate_targets - from ouroboros.model_slots import MODEL_ACCOUNTS_KEY, model_role_option, parse_fallback_chain, task_model_binding + from ouroboros.loop_transport import append_unknown_recovery_input + from ouroboros.model_slots import MODEL_ACCOUNTS_KEY, model_role_option, task_model_binding from ouroboros.model_wait import current_model_wait from ouroboros.loop_llm_call import _COOLDOWN_ERROR_KINDS as _cooldown_kinds from ouroboros.provider_models import provider_for_model @@ -182,35 +244,42 @@ def _run_cross_model_fallback_chain( if str(accumulated_usage.get("_last_llm_error_kind") or "") in _cooldown_kinds: _fcd.mark_cooldown(model, use_local) + def _disclose_unknown(target: List[Dict[str, Any]], route: str) -> None: + # A NEW generation after an eligible unknown outcome names the unknown attempt it follows. + if accumulated_usage.get("_pending_transport_outcome") or str(accumulated_usage.get("_last_llm_error_kind") or "") == "provider_outcome_unknown": + append_unknown_recovery_input( + target, accumulated_usage, dict(accumulated_usage.get("_pending_transport_outcome") or {}), + lead=f"The previous attempt ended without a usable answer; the configured route {route} continues.", + continuation="configured_route") + _cooled(active_model, active_use_local) primary_context_usage = _snapshot_context_fit_usage(accumulated_usage) - fallback_use_local = runtime_setting("USE_LOCAL_FALLBACK", "").lower() in ("true", "1") attempt_cap = _fcd.attempts_per_model() waiter = current_model_wait() - configured_chain = parse_fallback_chain() - # A primary quota refusal returned here, not waited: the configured routes come + # The round's own outage or unknown outcome remains its wait if no route answers. + entry_model, entry_kind = active_model, str(accumulated_usage.get("_last_llm_error_kind") or "") + own_wait = entry_kind in _ROUND_WAIT_KINDS + entry_failure = next((dict(ref) for ref in reversed(accumulated_usage.get("llm_call_refs") or []) + if isinstance(ref, dict) and ref.get("failure_code")), {}) + # A primary resource refusal returned here, not waited: the configured routes come # first, and the owner question is that refusal's own wait, asked after them. deferred, tools._ctx._deferred_resource_refusal = getattr(tools._ctx, "_deferred_resource_refusal", None), None - owner_question = (task_type != "presence" and deferred is not None + owner_question = (task_type != "presence" and deferred is not None and not own_wait and waiter is not None and waiter.waits_allowed) deferred_candidate = None # The route that actually refused; never replay an unrelated primary. - candidates, tried = fallback_candidate_targets(active_model), [] + rows = _route_candidates(tools._ctx, active_model, active_use_local, + str(getattr(context_fit_plan, "model_role", "") or ""), waiter) + tried = [] # The notice names the model that was actually just tried. `active_model` - # stays the primary until a candidate succeeds, so a second switch would + # stays the acting route until a candidate succeeds, so a second switch would # otherwise read "primary -> B" beside B's predecessor's failure reason. previous_model, previous_tag = active_model, " (local)" if active_use_local else "" msg = None - # ABI-4: the candidate ladder arrives as typed ResolvedModelTarget values; - # `.model_id` is read once here and crosses to strings only at the LLM - # transport boundary (the chat API's model parameter). The local-vs-remote - # dispatch lane is the single global USE_LOCAL_FALLBACK flag above (the - # pre-existing chain contract): the ladder's `provider_route` stays the "" - # sentinel rather than fabricating a per-candidate fact nothing consumes. - for index, candidate in enumerate(candidates): - fallback_model = candidate.model_id - # The role belongs to the configured row, before active-model removal - # and deduplication. Those filters must not shift its account binding. - fallback_role = f"fallback:{configured_chain.index(fallback_model)}" + # ABI-4: the chain ladder arrives as typed ResolvedModelTarget values; `.model_id` + # is read once in `_route_candidates` and crosses to strings only at the LLM + # transport boundary. Every chain row uses the single global USE_LOCAL_FALLBACK + # flag (the pre-existing chain contract); the primary keeps its own locality. + for index, (fallback_model, fallback_role, fallback_use_local, is_primary) in enumerate(rows): if _fcd.is_cooling_down(fallback_model, fallback_use_local): continue deadline = _loop()._task_deadline_epoch(tools) @@ -229,9 +298,13 @@ def _run_cross_model_fallback_chain( account_note = f"; account: {fallback_account or 'Auto'}" if account_route else "" reason = str(accumulated_usage.get("_last_llm_error_kind") or "") emit_progress(f"⚡ Fallback: {previous_model}{previous_tag} → {fallback_model}{ftag}" - f"{account_note}" - f"{f'; reason: {reason}' if reason else ''}{'; pinned account: siblings were not tried' if account_route and fallback_account else ''}", + f"{' (primary)' if is_primary else ''}{account_note}" + f"{f'; reason: {reason}' if reason else ''}" + f"{'; sign-in is still needed there' if reason == 'auth_error' else ''}" + f"{'; the earlier attempt’s outcome and cost stay unknown' if reason == 'provider_outcome_unknown' else ''}" + f"{'; pinned account: siblings were not tried' if account_route and fallback_account else ''}", incident={"task_incident": "model_lane_switch", "toast_once": f"{task_id}:model_lane_switch:{round_idx}:{fallback_model}"}) + _disclose_unknown(messages, fallback_model) # Cross-FAMILY fallback must not replay the primary's # provider-private reasoning to a different family (the GLM->Claude # 400 "Invalid signature" death); the SSOT sanitizer no-ops same-family. @@ -275,7 +348,7 @@ def _run_cross_model_fallback_chain( model_role=fallback_role, emit_progress=emit_progress, defer_resource_wait=(task_type == "presence" or owner_question - or _route_follows(candidates[index + 1:], fallback_use_local)), + or _route_follows(rows[index + 1:])), ) tried.append(fallback_model) msg, _cost, candidate_mode = _loop()._call_round_model(candidate_call) @@ -286,7 +359,7 @@ def _run_cross_model_fallback_chain( deferred = getattr(tools._ctx, "_deferred_resource_refusal", None) if deferred is not None: deferred_candidate = candidate_call - owner_question = (task_type != "presence" and deferred is not None + owner_question = (task_type != "presence" and deferred is not None and not own_wait and waiter is not None and waiter.waits_allowed) if msg is not None: ( @@ -316,21 +389,28 @@ def _run_cross_model_fallback_chain( .get("tool_calls") or []) ), ) + tools._ctx._route_facts_pending = "" if is_primary else _route_facts_text( + tools._ctx, model=fallback_model, use_local=fallback_use_local, role=fallback_role, + failed_model=entry_model, failure={**entry_failure, "failure_code": entry_kind or + entry_failure.get("failure_code")}, waiter=waiter) + if not is_primary: + tools._ctx.route_wait_on_primary = False # a declared wait belonged to the primary route break tools._ctx.context_fit_plan = context_fit_plan tools._ctx.messages = messages tools._ctx.active_context_mode = active_context_mode _restore_context_fit_usage(accumulated_usage, primary_context_usage) - if str(accumulated_usage.get("_last_llm_error_kind") or "") in _CHAIN_STOP_KINDS: + if _walk_fenced(tools._ctx, accumulated_usage): break _cooled(fallback_model, fallback_use_local) previous_model, previous_tag = fallback_model, ftag + fenced = msg is None and _walk_fenced(tools._ctx, accumulated_usage) if deferred is None: deferred = getattr(tools._ctx, "_deferred_resource_refusal", None) - owner_question = (task_type != "presence" and deferred is not None + owner_question = (task_type != "presence" and deferred is not None and not own_wait and waiter is not None and waiter.waits_allowed) accumulated_usage.pop(RESOURCE_REFUSAL_KEY, None) - if msg is None and owner_question and str(accumulated_usage.get("_last_llm_error_kind") or "") not in _CHAIN_STOP_KINDS: + if msg is None and owner_question and not fenced: # Only the refused route is eligible to re-send after the owner wait; # the primary might have failed permanently before a fallback's quota refusal. deferred.ask_owner(waiter) @@ -342,11 +422,10 @@ def _run_cross_model_fallback_chain( active_use_local=active_use_local, active_context_mode=active_context_mode, drive_root=pathlib.Path(drive_logs).parent, emit_progress=emit_progress, defer_resource_wait=False) retry_call.defer_resource_wait = False + _disclose_unknown(retry_call.messages, retry_call.active_model) # An owner may select a DIFFERENT account on either the primary or a # fallback model. Rebind the retained call before measurement and physical # send; the pre-wait account's fingerprint/capacity is not evidence for B. - from ouroboros.model_slots import task_model_binding - role, account = task_model_binding( {"model_role": retry_call.model_role, "task_metadata": getattr(tools._ctx, "task_metadata", {})}, @@ -363,10 +442,13 @@ def _run_cross_model_fallback_chain( messages, retry_call.messages, retry_call.context_fit_plan, active_context_mode, tool_schemas, accumulated_usage, handover_from_model=active_model, handover_reason="owner_wait") + tools._ctx._route_facts_pending = _route_facts_text( + tools._ctx, model=active_model, use_local=active_use_local, role=role, + failed_model=entry_model, failure={**entry_failure, "failure_code": entry_kind}, waiter=waiter) else: active_model, active_use_local = retry_call.active_model, retry_call.active_use_local context_fit_plan = retry_call.context_fit_plan - elif msg is None and deferred is not None: # the refused primary buys no forced final either + elif msg is None and deferred is not None and not own_wait: # the refused primary buys no forced final either accumulated_usage[RESOURCE_REFUSAL_KEY] = deferred.terminal( fallbacks_tried=tried, owner_wait="not_asked" if owner_question else "not_allowed") return ( @@ -378,6 +460,83 @@ def _run_cross_model_fallback_chain( ) +def _recover_failed_round(limit_ctx: Any, tools: ToolRegistry, msg: Any, episode: Any, *, + context_fit_plan: Any, active_context_mode: str, emit_progress: Callable[..., None]) -> tuple: + """The recovery order of one round's outcome. + + Each observed failure can use configured alternatives, including during an existing + outage. That episode keeps its elapsed clock, backoff and prior custody; a failed + alternative never resets those bounds. Returns ``(msg, model, use_local, plan, mode, episode)``. + """ + from ouroboros.model_wait import current_model_wait + + usage, ctx = limit_ctx.accumulated_usage, tools._ctx + model, use_local = limit_ctx.active_model, limit_ctx.active_use_local + + def reconcile(current: Any) -> Any: + return _loop()._reconcile_transport_wait( + current, ctx, msg_present=msg is not None, error_kind=str(usage.get("_last_llm_error_kind") or ""), + drive_logs=limit_ctx.drive_logs, task_id=limit_ctx.task_id, model=model, emit_progress=emit_progress) + + if episode is not None: + episode = reconcile(episode) + if msg is not None: + return msg, model, use_local, context_fit_plan, active_context_mode, episode + kind, pending = str(usage.get("_last_llm_error_kind") or ""), usage.get("_pending_transport_outcome") + if (msg is None and _loop()._fallback_chain_allowed(ctx, kind, None, usage) and _route_candidates( + ctx, model, use_local, str(getattr(context_fit_plan, "model_role", "") or ""), current_model_wait())): + msg, model, use_local, context_fit_plan, active_context_mode = _loop()._run_cross_model_fallback_chain( + llm=limit_ctx.llm, ctx=ctx, tools=tools, messages=limit_ctx.messages, active_model=model, + active_use_local=use_local, tool_schemas=limit_ctx.tool_schemas, active_effort=limit_ctx.active_effort, + max_retries=limit_ctx.max_retries, drive_logs=limit_ctx.drive_logs, task_id=limit_ctx.task_id, + round_idx=limit_ctx.round_idx, event_queue=limit_ctx.event_queue, accumulated_usage=usage, + task_type=limit_ctx.task_type, emit_progress=emit_progress, context_fit_plan=context_fit_plan, + active_context_mode=active_context_mode) + if msg is None and not _walk_fenced(ctx, usage) and (kind in _ROUND_WAIT_KINDS or usage.get("_pending_transport_outcome")): + # The round's own outage or unknown outcome owns its wait: a later + # candidate's failure never re-aims it (nor the probe's expected route). + outstanding = usage.get("_pending_transport_outcome") or pending + usage["_last_llm_error_kind"] = "provider_outcome_unknown" if outstanding else kind + if outstanding: + usage["_pending_transport_outcome"] = outstanding + return msg, model, use_local, context_fit_plan, active_context_mode, reconcile(episode) + + +def _walk_fenced(tool_ctx: Any, usage: Dict[str, Any]) -> bool: + """A spent deadline, or an unknown outcome that permits no new generation, ends the walk.""" + from ouroboros.loop_transport import new_generation_after_unknown + + kind = str(usage.get("_last_llm_error_kind") or "") + return kind == "deadline_exhausted" or ( + kind == "provider_outcome_unknown" and not new_generation_after_unknown(tool_ctx, usage)) + + +def _apply_round_route_overrides(ctx: Any, tools: ToolRegistry, messages: List[Dict[str, Any]], route: tuple, + context_fit_plan: Any, active_context_mode: str, preferred_mode: str, + tool_schemas: List[Dict[str, Any]]) -> tuple: + """Apply one-shot ``switch_model`` choices at the round boundary. + + ``route`` is ``(model, use_local, effort)``. A symbolic primary return also names + the primary's role, so the rebound fit plan, account and processing binding are the + primary's own (Auto stays Auto: no observed account becomes a pin); effort intent is + untouched. Returns ``(model, use_local, effort, plan, mode)``. + """ + previous = route[:2] + model, use_local, effort = _loop()._apply_runtime_overrides(ctx, *route) + role, ctx.active_role_override = getattr(ctx, "active_role_override", None), None + if (model, use_local) != previous or role: + context_fit_plan, active_context_mode = _loop()._rebind_context_fit_plan( + context_fit_plan, tools, messages, model=model, use_local=use_local, preferred_mode=preferred_mode, + tool_schemas=tool_schemas, **({"model_role": role, "model_route": {}} if role else {})) + if model != previous[0]: + # Cross-FAMILY switch: discard provider-private reasoning + # signatures before the new route sees them (same family is a no-op). + sanitized = LLMClient.sanitize_reasoning_on_model_switch(messages, previous[0], model) + if sanitized is not messages: + messages[:] = sanitized + return model, use_local, effort, context_fit_plan, active_context_mode + + def _rebind_context_fit_plan( plan: Any, tools: ToolRegistry, @@ -533,21 +692,27 @@ class _RoundModelCallContext: defer_resource_wait: Optional[bool] = None -def _route_follows(candidates: List[Any], use_local: bool) -> bool: +def _route_follows(candidates: List[Tuple[str, str, bool, bool]]) -> bool: from ouroboros import fallback_cooldown - return any(not fallback_cooldown.is_cooling_down(item.model_id, use_local) for item in candidates) + return any(not fallback_cooldown.is_cooling_down(model, use_local) for model, _role, use_local, _primary in candidates) def _fallback_route_follows(ctx: _RoundModelCallContext) -> bool: - """Whether this primary round's fallback chain would still dial a configured route.""" - from ouroboros.config import fallback_candidate_targets + """Whether this round's configured-route walk would still dial a route. - if bool(getattr(ctx.tools._ctx, "exact_model_route", False)) or isinstance( - ctx.accumulated_usage.get(TRANSPORT_DEATHS_KEY), dict): + A declared wait on the primary keeps its resource refusal on the primary's own + visible wait instead of deferring it to paid alternatives. + """ + from ouroboros.model_wait import current_model_wait + + tool_ctx = ctx.tools._ctx + if (bool(getattr(tool_ctx, "exact_model_route", False)) or getattr(tool_ctx, "route_wait_on_primary", False) + or isinstance(ctx.accumulated_usage.get(TRANSPORT_DEATHS_KEY), dict)): return False - use_local = runtime_setting("USE_LOCAL_FALLBACK", "").lower() in ("true", "1") - return _route_follows(fallback_candidate_targets(ctx.active_model), use_local) + return _route_follows(_route_candidates( + tool_ctx, ctx.active_model, ctx.active_use_local, + str(getattr(ctx.context_fit_plan, "model_role", "") or ctx.model_role or ""), current_model_wait())) def _context_fit_round_id(ctx: _RoundModelCallContext) -> str: @@ -660,10 +825,7 @@ def _dispatch_round_model( candidate_predicate: Optional[Callable[[Any], Any]] = None, ) -> Tuple[Any, float]: from ouroboros.model_wait import current_model_wait - from ouroboros.loop_transport import ( - emit_model_substitution, - managed_transport_continuation, transport_repeat_stop_requested, - ) + from ouroboros.loop_transport import emit_model_substitution, transport_repeat_stop_requested from ouroboros.owner_mailbox import OwnerMailboxPeek from ouroboros.send_clock import main_clock_policy @@ -691,7 +853,11 @@ def _dispatch_round_model( with contextlib.ExitStack() as binding: deferral = None if waiter is not None: - binding.enter_context(waiter.register_reprepare(role, lambda kwargs: _reprepare_waiting_main(ctx, kwargs))) + from ouroboros.loop_llm_call import classify_llm_exception + resource_reason = (lambda error: {"auth_error": "auth", "quota_exhausted": "quota"}.get( + classify_llm_exception(error).kind, "")) if getattr(ctx.tools._ctx, "route_wait_on_primary", False) else None + binding.enter_context(waiter.register_reprepare( + role, lambda kwargs: _reprepare_waiting_main(ctx, kwargs), resource_reason=resource_reason)) if ((not waiter.waits_allowed or _fallback_route_follows(ctx)) if primary else ctx.defer_resource_wait): deferral = binding.enter_context(waiter.defer_resource_wait(role)) result = _loop().call_llm_with_retry( @@ -702,8 +868,10 @@ def _dispatch_round_model( deadline_ts=_loop()._task_deadline_epoch(ctx.tools), transport_reserve_sec=task_pacing.get_finalization_grace_sec(), attempt_cap=attempt_cap, + # Inline Presence keeps its bounded typed-death repeat; every other round's new + # generation after an unknown outcome is the round recovery (configured routes first). transport_death_retries=(_TRANSPORT_DEATH_RETRIES if attempt_cap is None - and not managed_transport_continuation(ctx.tools._ctx) else 0), + and ctx.task_type == "presence" else 0), stop_retry_check=(lambda: transport_repeat_stop_requested(ctx.tools._ctx, mailbox_peek=mailbox_peek)) if attempt_cap is None else None, allow_server_web_search=_loop()._server_web_allowed_by_task(ctx.tools._ctx), physical_context=(_physical_context_for_fit(disposition) if disposition is not None else None), @@ -1123,6 +1291,10 @@ def _project_wake_input(ctx: _RoundModelCallContext, *, overflowed: bool = False def _call_round_model(ctx: _RoundModelCallContext) -> Tuple[Any, float, str]: """Measure, optionally reclaim, dispatch, and recover one Main round.""" + facts = getattr(ctx.tools._ctx, "_route_facts_pending", "") + if facts and ctx.defer_resource_wait is None: # the acting route's first own round after a switch + ctx.tools._ctx._route_facts_pending = "" + _loop()._append_or_merge_user_message(ctx.messages, facts) _append_routing_receipts(ctx) _project_wake_input(ctx) disposition = _loop()._measure_round_main_fit(ctx, automatic_pass_used=False) diff --git a/ouroboros/loop_transport.py b/ouroboros/loop_transport.py index fa69c3fed..59b88421a 100644 --- a/ouroboros/loop_transport.py +++ b/ouroboros/loop_transport.py @@ -2,8 +2,10 @@ A REMOTE pre-dispatch transport failure (typed ``released`` custody: connect refused/timed out before request bytes left this host, $0 in the ledger) is not a -model failure. Instead of burning the fallback chain or terminalizing, the round -gate in ``loop.py`` latches a :class:`TransportWaitEpisode` and waits: durable +model failure, and one provider's failure does not prove the egress dead: the +round first tries every configured route (``loop_model_call._recover_failed_round``), +and only when none answered does the round gate in ``loop.py`` latch a +:class:`TransportWaitEpisode` and wait: durable ``network_wait`` events + owner progress notes, an interruptible backoff sleep, then a free redial of the SAME round (the round budget is not consumed). A managed task waits as long as its existing rails allow — owner deadline minus the @@ -22,8 +24,11 @@ the durable ``ended`` detail names the rail that expired — the bound's own ``interactive_wait_window_exhausted``, or the deadline's detail when the owner window closed first. Interactive progress notes omit cancellation promises; direct-turn Stop uses its existing typed control and wakes the same sleep. -Recovery is an owner note for every episode; local adoption and -error-kind change are notes for interactive turns only, because such a turn +An eligible unknown outcome (``new_generation_after_unknown``; its charge is disclosed) +follows the same order on every turn except inline Presence: configured routes +first, then the upstream-observed continuation inside the same wait bounds. +Recovery is an owner note for every episode; an +error-kind change is a note for interactive turns only, because such a turn has no progress row to show the closure — a managed task keeps the durable row and its ordinary progress; exhaustion is a note for an interactive turn, while a managed task's exhaustion is its terminal result. Every episode note is @@ -38,8 +43,6 @@ size-ratchet byte cap). from __future__ import annotations -from ouroboros.config import runtime_setting - import logging import pathlib import queue @@ -89,7 +92,6 @@ class TransportWaitEpisode: redials: int = 0 wait_iterations: int = 0 last_note_monotonic: float = 0.0 - local_pass_used: bool = False final_redial_done: bool = False outcome_custody: Dict[str, Any] = field(default_factory=dict) continuation_granted: bool = False @@ -144,9 +146,55 @@ def emit_network_wait_event( def managed_transport_continuation(ctx: Any) -> bool: - """Managed cognition may recover; a live delegated-leaf hold takes priority.""" + """Ordinary cognition rejoins its accepted operation under its existing controls.""" return bool(ctx is not None and getattr(ctx, "task_id", "") - and not getattr(ctx, "is_direct_chat", False)) + and not inline_presence(ctx)) + + +def inline_presence(ctx: Any) -> bool: + """An inline Presence turn: its unknown outcome ends in its own typed retry refusal.""" + from ouroboros.dialogue_provenance import is_presence_task + + metadata = getattr(ctx, "task_metadata", None) + return (getattr(ctx, "current_task_type", None) == "presence" + or is_presence_task({"metadata": metadata if isinstance(metadata, dict) else {}})) + + +def new_generation_after_unknown(ctx: Any, accumulated_usage: Optional[Dict[str, Any]] = None) -> bool: + """Whether this round may start a NEW, possibly charged, generation after its unknown outcome. + + Typed, never from prose or elapsed time: the receiving call has ended with no usable + answer and no same-operation rejoin is viable. It is refused for an accepted operation + whose READ failed (it may still finish; only it is rejoined), for a round still holding + paid repeats, and for inline Presence. It proves nothing about remote computation. + """ + usage = accumulated_usage if accumulated_usage is not None else (getattr(ctx, "_accumulated_usage", None) or {}) + pending = usage.get("_pending_transport_outcome") or {} + return bool(ctx is not None and getattr(ctx, "task_id", "") and not inline_presence(ctx) + and not isinstance(usage.get(TRANSPORT_DEATHS_KEY), dict) + and not pending.get("same_operation_recoverable")) + + +def append_unknown_recovery_input(messages: list, accumulated_usage: dict, previous: Dict[str, Any], *, + lead: str, continuation: str, connectivity: Any = None) -> bool: + """The facts-only input a NEW generation after an unknown outcome carries, once per unknown attempt. + + The old attempt keeps its own id, route and unknown cost; the new physical attempt gets its + own identity and may be another configured route. Returns False when this exact unknown + attempt was already disclosed to the transcript. + """ + noticed = (accumulated_usage.get("transport_recovery") or {}).get("previous_attempt") + accumulated_usage["transport_recovery"] = {"previous_attempt": previous, "connectivity": connectivity or {}, + "continuation": continuation, "old_outcome": "unknown"} + if previous and noticed == previous: + return False + messages.append({"role": "user", "content": "[SYSTEM NOTICE]\n" + ( + f"[Transport recovery] {lead} Continue from the recorded work in a NEW physical model attempt. " + "The previous attempt's outcome and any unreported cost remain unknown; do not treat it as failed, " + "free, completed, or an instruction to repeat completed tools. " + f"Previous physical attempt: {previous.get('physical_attempt_id') or 'unreported'}; " + f"operation: {previous.get('operation_id') or 'unreported'}.")}) + return True def continue_unknown_transport(episode: TransportWaitEpisode, *, llm: Any, tools: Any, @@ -171,22 +219,16 @@ def continue_unknown_transport(episode: TransportWaitEpisode, *, llm: Any, tools context_fit_plan=getattr(ctx, "context_fit_plan", None), overrides=waiter.overrides if waiter else None) observed = upstream_transport_reachable(llm, model, timeout=timeout, model_role=role, account_override=account, observed_after=episode.started_at, - expected_route=episode.outcome_custody.get("route")) + expected_route=(episode.outcome_custody.get("route") + if episode.outcome_custody.get("model", model) == model + and episode.outcome_custody.get("model_role", role) in ("", role) else None)) remaining = dispatch_window_remaining_sec(deadline_ts=task_deadline_epoch(tools), reserve_sec=get_finalization_grace_sec()) if (not observed or remaining == 0.0 or transport_repeat_stop_requested(ctx) or (waiter is not None and waiter.control_reason())): return False previous = dict(episode.outcome_custody) - message = ( - "[Transport recovery] Upstream connectivity is available again. Continue from the recorded work " - "in a NEW physical model attempt. The previous attempt's outcome and any unreported cost remain " - "unknown; do not treat it as failed, free, completed, or an instruction to repeat completed tools. " - f"Previous physical attempt: {previous.get('physical_attempt_id') or 'unreported'}; " - f"operation: {previous.get('operation_id') or 'unreported'}." - ) - messages.append({"role": "user", "content": "[SYSTEM NOTICE]\n" + message}) - accumulated_usage["transport_recovery"] = {"previous_attempt": previous, "connectivity": observed, - "continuation": "new_physical_attempt", "old_outcome": "unknown"} + append_unknown_recovery_input(messages, accumulated_usage, previous, connectivity=observed, + lead="Upstream connectivity is available again.", continuation="new_physical_attempt") accumulated_usage.pop(TRANSPORT_DEATHS_KEY, None) accumulated_usage.pop("_pending_transport_outcome", None) episode.continuation_granted = True @@ -198,37 +240,32 @@ def continue_unknown_transport(episode: TransportWaitEpisode, *, llm: Any, tools return True -def _use_local_fallback_configured() -> bool: - return runtime_setting("USE_LOCAL_FALLBACK", "").lower() in ("true", "1") - - def fallback_chain_allowed( ctx: Any, last_error_kind: str, episode: Optional[TransportWaitEpisode], accumulated_usage: Optional[Dict[str, Any]] = None, ) -> bool: - """Whether this round may walk the cross-model fallback chain.""" + """Whether this round may try configured alternatives for the current failure. + + Existing outage pacing bounds repeated attempts; no lifetime pass limit is added. + An eligible unknown outcome tries them first, not the same route again; a declared + wait on the primary (``switch_model(primary="wait")``) keeps its refusals and + outages on the primary's own wait instead of paid alternatives. + """ + usage = accumulated_usage or {} if bool(getattr(ctx, "exact_model_route", False)): return False - if isinstance((accumulated_usage or {}).get(TRANSPORT_DEATHS_KEY), dict): + if isinstance(usage.get(TRANSPORT_DEATHS_KEY), dict): # The round still holds an unresolved attempt (a granted transport-death # repeat with no usable response since): no paid candidate may dial over # it, whatever the last kind says. return False - if episode is not None: - # Q4: during a remote transport outage the chain runs at most ONCE per - # episode, and only when USE_LOCAL_FALLBACK makes the whole chain local — - # remote candidates never dial over a proven dead egress. - if ( - last_error_kind != "transport_unavailable" - or episode.local_pass_used - or not _use_local_fallback_configured() - ): - return False - episode.local_pass_used = True - return True - return last_error_kind not in ( - "context_overflow", "provider_outcome_unknown", "deadline_exhausted", - ) + if getattr(ctx, "route_wait_on_primary", False) and ( + last_error_kind in ("transport_unavailable", "provider_outcome_unknown") + or usage.get("_last_llm_resource_refusal")): + return False + if last_error_kind == "provider_outcome_unknown": + return new_generation_after_unknown(ctx, accumulated_usage) + return last_error_kind not in ("context_overflow", "deadline_exhausted") def reconcile_transport_wait( @@ -241,30 +278,29 @@ def reconcile_transport_wait( task_id: str, model: str, emit_progress: Callable[..., None], - after_local_pass: bool = False, ) -> Optional[TransportWaitEpisode]: """Reconcile the episode latch with one dispatch outcome. - Enters a new episode on a fresh ``transport_unavailable`` failure (durable - ``entered`` event; the first owner note fires immediately). An interactive + Enters a new episode on a fresh ``transport_unavailable`` failure, or an + eligible unknown outcome (``new_generation_after_unknown``), that the + configured routes did not answer (durable ``entered`` event; the first owner + note fires immediately). An interactive turn's episode gets the idle-timeout bound at entry, because the bound is measured from entry and the turn has no other rail. ``emit_progress`` honors the ``incident=`` keyword (``OuroborosAgent._emit_progress``). Recovery - is a note for every episode; local adoption and error-kind change are notes for + is a note for every episode; an error-kind change is a note for interactive turns only (a managed episode keeps its durable ``ended`` row and its ordinary progress), and exhaustion is separately noted only for interactive turns (``transport_wait_step``). - The round gate reconciles twice per failed dispatch: once with the - pre-chain kind, and once after the fallback chain with the FRESH kind — so - an outage first observed MID-chain (a remote candidate dying pre-dispatch - while the primary failed generically) still latches an episode instead of - falling through to a generic terminal that would dial a forced-final call - over the proven-dead egress. On a redial outcome: a response ends the - episode as ``recovered`` (mandatory owner note), a NON-transport failure - ends it as evidence the transport is passable again, while - ``transport_unavailable`` and a pre-dispatch deadline refusal keep the - latch for the wait/terminal step. A failed local fallback pass - (``after_local_pass``) never clears the latched remote cause. + With no active episode the round gate reconciles once, after the configured + routes, with the round's own failure (the chain keeps it) — or with a + candidate's pre-dispatch failure, so an outage first observed MID-chain (a + candidate dying pre-dispatch while the primary failed generically) still + latches an episode instead of a generic terminal. On a redial outcome: a + response ends the episode as ``recovered`` (mandatory owner note), a + NON-transport failure ends it as evidence the transport is passable again, + while ``transport_unavailable`` and a pre-dispatch deadline refusal keep the + latch for the wait/terminal step. A granted continuation that fails again — unknown after dispatch, or released before it — returns to the SAME episode (work-order §B9 "repeated unknown creates no burst"), and so does a free redial that crosses dispatch @@ -277,10 +313,13 @@ def reconcile_transport_wait( exactly as before. """ pending = dict((getattr(ctx, "_accumulated_usage", {}) or {}).get("_pending_transport_outcome") or {}) - unknown_again = error_kind == "provider_outcome_unknown" and managed_transport_continuation(ctx) + unknown_again = error_kind == "provider_outcome_unknown" and new_generation_after_unknown(ctx) if episode is not None and episode.continuation_granted: - if msg_present or error_kind not in ("provider_outcome_unknown", "transport_unavailable"): - episode = None # This new physical outcome ends the episode (an answer) or owns a fresh one. + if (msg_present or error_kind not in ("provider_outcome_unknown", "transport_unavailable") + or (error_kind == "provider_outcome_unknown" and pending and not unknown_again)): + # This new physical outcome ends the episode (an answer), owns a fresh one, or is + # an accepted operation that may still finish (only it is rejoined, never regenerated). + episode = None elif error_kind == "provider_outcome_unknown" and not pending: return episode # No new attempt exists yet (a failed probe, a yielded call): the grant stands. else: @@ -310,7 +349,7 @@ def reconcile_transport_wait( emit_progress("🌐 The new attempt could not reach the provider — waiting and redialing " "automatically (that attempt was $0).", incident=None) return episode - if (episode is not None and not msg_present and not after_local_pass + if (episode is not None and not msg_present and episode.wait_cause != "provider_outcome_unknown" and unknown_again): # A formerly free redial crossed dispatch and died unknown: the same # episode now needs upstream proof before its next attempt; the clock, @@ -331,7 +370,7 @@ def reconcile_transport_wait( ) return episode if episode is None: - unknown = error_kind == "provider_outcome_unknown" and managed_transport_continuation(ctx) + unknown = unknown_again if msg_present or (error_kind != "transport_unavailable" and not unknown): return None interactive = bool(getattr(ctx, "is_direct_chat", False)) @@ -360,30 +399,17 @@ def reconcile_transport_wait( return episode elapsed = time.monotonic() - episode.started_monotonic if msg_present: - if after_local_pass: - emit_network_wait_event( - drive_logs, task_id=task_id, phase="ended", elapsed_sec=elapsed, - redials=episode.redials, model=model, detail="local_fallback_adopted", - ) - if episode.interactive: # a managed task keeps its durable row and ordinary progress - emit_progress( - f"🌐 Provider connection still unavailable after {elapsed / 60.0:.1f} min " - "— continuing on the local fallback model.", - incident=None, - ) - else: - emit_network_wait_event( - drive_logs, task_id=task_id, phase="recovered", elapsed_sec=elapsed, - redials=episode.redials, model=model, - ) - emit_progress( - f"🌐 Provider connection restored after {elapsed / 60.0:.1f} min — resuming.", - incident=None, - ) + emit_network_wait_event( + drive_logs, task_id=task_id, phase="recovered", elapsed_sec=elapsed, + redials=episode.redials, model=model, + ) + emit_progress( + f"🌐 Provider connection restored after {elapsed / 60.0:.1f} min — resuming.", + incident=None, + ) return None if ( - not after_local_pass - and error_kind not in ("transport_unavailable", "deadline_exhausted") + error_kind not in ("transport_unavailable", "deadline_exhausted") and not (error_kind == "provider_outcome_unknown" and episode.wait_cause == "provider_outcome_unknown") ): # The redial got past the connect phase and failed differently: the @@ -924,7 +950,11 @@ def provider_recovery_hint(accumulated_usage: Dict[str, Any]) -> str: "fallback was sent while the earlier request has no terminal outcome, since " "either could duplicate live work." ) - if kind == "provider_outcome_unknown": + if kind == "provider_outcome_unknown" or accumulated_usage.get("_pending_transport_outcome"): + if accumulated_usage.get("transport_recovery"): + return (" An earlier dispatched attempt still has no confirmed provider outcome. " + "Recovery used new task requests; their separate outcomes and any unknown costs remain recorded. " + "No additional summary request was sent.") return ( " The dispatched request has no terminal provider outcome, so no " "retry or paid fallback was sent; either could duplicate live work." @@ -935,6 +965,11 @@ def provider_recovery_hint(accumulated_usage: Dict[str, Any]) -> str: "failure, $0 spent); the exact exception class is in the durable " "llm_api_error event. Retrying when connectivity returns will help." ) + if (accumulated_usage.get("resource_refusal") or {}).get("reason") == "unavailable" or ( + accumulated_usage.get("_last_llm_resource_refusal") == "unavailable"): + reset = str(accumulated_usage.get("_last_llm_reset_at") or "") + return (" No configured account could serve this request; quota exhaustion was not established." + + (f" The engine reported a reset time of {reset}; availability remains unconfirmed." if reset else "")) if kind == "subscription_window_exhausted": reset_at = str(accumulated_usage.get("_last_llm_reset_at") or "").strip() when = f" It resets at {reset_at}." if reset_at else "" @@ -942,16 +977,18 @@ def provider_recovery_hint(accumulated_usage: Dict[str, Any]) -> str: if not refusal: return ( " The subscription window for the delegated route is spent. This is " - f"TRANSIENT, not a billing refusal — waiting cures it.{when} Retrying is " - "scheduled against that reset time, not the ordinary short backoff." + f"TRANSIENT, not a billing refusal — waiting cures it.{when} Nothing sleeps to that reset." ) rotation, tried = refusal.get("account_rotation") or {}, ", ".join(refusal.get("fallbacks_tried") or []) # Only the engine's own pool verdict proves every account; any other stop claims nothing. accounts = (" The engine reports every compatible account blocked." if rotation.get("pool_exhausted") else f" Account rotation stopped ({rotation.get('stop')}); other accounts are unproven." if rotation else "") + subject = ("No managed account can currently serve this model route (an engine-dated pool refusal, " + "not quota)" if refusal.get("reason") == "unavailable" + else "The subscription quota for this model route is spent") return ( - " The subscription quota for this model route is spent. This is " + f" {subject}. This is " f"TRANSIENT, not a billing refusal — waiting cures it.{when}{accounts}" f"{f' Configured fallbacks tried without an answer: {tried}.' if tried else ''} Nothing " "more was sent, and nothing sleeps to that reset." diff --git a/ouroboros/model_slots.py b/ouroboros/model_slots.py index ef67d414c..6f884f17c 100644 --- a/ouroboros/model_slots.py +++ b/ouroboros/model_slots.py @@ -143,6 +143,27 @@ def task_model_binding(task: dict, *, context_fit_plan: object = None, return role, pin +def route_binding(model: str, use_local: bool, role: str, *, + overrides: dict | None = None) -> tuple[str, bool, str]: + """The complete identity a send on ``role`` uses: model, locality and account. + + A live owner choice for the role (a wait-card switch) replaces the configured + route exactly as the send applies it; the account is the configured pin or + empty Auto, never an observed Auto account, and routes without accounts have + none. Equal model strings with different accounts are different routes. + """ + from ouroboros.provider_models import provider_for_model + + chosen = (overrides or {}).get(role) or {} + model, use_local = str(chosen.get("model", model) or ""), bool(chosen.get("use_local", use_local)) + account = chosen.get("model_account_override") + if account is None: + account = model_role_option(MODEL_ACCOUNTS_KEY, role) if role else "" + if use_local or provider_for_model(model) != "claudexor": + account = "" + return model, use_local, str(account or "").strip() + + def task_processing_preference(task: dict, *, model_role: str = "", override: str | None = None) -> str: """Capture the actor's intent; an unconfigured fallback preserves that intent.""" diff --git a/ouroboros/model_wait.py b/ouroboros/model_wait.py index 2694c8b69..bad5c07b5 100644 --- a/ouroboros/model_wait.py +++ b/ouroboros/model_wait.py @@ -5,11 +5,12 @@ This module keeps only the live call's wait and role overrides. It neither schedules work nor records physical attempts: every resumed call still goes through LLMClient and the ordinary physical-attempt ledger. -Quota recovery order: the transport's Auto account rotation, then every -configured route of the round, and only then the owner. A round whose later -route exists returns its refusal instead of waiting (``ResourceDeferral``); the -owner question is then opened from that retained refusal, never from another -generation. An inline Presence turn never waits for quota or for the owner. +Resource recovery order (quota, sign-in, an engine-dated unavailable pool): the +transport's Auto account rotation, then every configured route of the round, +and only then the owner. A round whose later route exists returns its refusal +instead of waiting (``ResourceDeferral``); the owner question is then opened +from that retained refusal, never from another generation, and nothing sleeps +to a reset. An inline Presence turn never waits for a resource or the owner. """ from __future__ import annotations @@ -62,9 +63,14 @@ def model_wait_reason(error: Exception) -> str: """Only confirmed resource causes authorize this live waiting contract. The engine's typed mixed pool means auth plus quota, excluding unknown - readiness, disabled accounts and model incompatibility. Generic pool failure - proves none of those facts and must keep its ordinary error path. + readiness, disabled accounts and model incompatibility. A dated generic pool + refusal is ``unavailable`` for this request; its reset forecast proves neither + quota exhaustion nor unavailability until that instant. An undated pool failure + proves none of those facts and keeps its ordinary error path. """ + authored = getattr(error, "task_resource_wait_reason", "") + if authored in {"auth", "quota"}: + return authored # confirmed by the registered caller's refusal classifier code = getattr(error, "code", "") if code in {"auth_required", "subscription_window_exhausted"}: return "auth" if code == "auth_required" else "quota" @@ -72,6 +78,8 @@ def model_wait_reason(error: Exception) -> str: context = problem.get("context") if isinstance(problem, dict) else None if code == "credential_pool_exhausted" and isinstance(context, dict) and context.get("poolCause") == "mixed": return "auth_quota" + if code == "credential_pool_exhausted" and str(getattr(error, "reset_at", "") or "").strip(): + return "unavailable" return "" @@ -160,7 +168,7 @@ def execution_elapsed_seconds(meta: dict, now: float) -> float: _CURRENT: contextvars.ContextVar[TaskModelWait | None] = contextvars.ContextVar( "ouroboros_model_wait", default=None) -_REPREPARE: contextvars.ContextVar[dict[str, Callable] | None] = contextvars.ContextVar( +_REPREPARE: contextvars.ContextVar[dict[str, tuple[Callable, Callable | None]] | None] = contextvars.ContextVar( "ouroboros_model_wait_reprepare", default=None) _CALENDAR: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar( "ouroboros_model_wait_calendar", default=()) @@ -417,10 +425,11 @@ class TaskModelWait: self.started_monotonic = time.monotonic() - max(0.0, time.time() - float(started_at)) @contextlib.contextmanager - def register_reprepare(self, role: str, callback: Callable[[dict], dict]) -> Iterator[None]: - """Bind one call's Main-context preparation without a cross-thread registry.""" + def register_reprepare(self, role: str, callback: Callable[[dict], dict], *, + resource_reason: Callable[[Exception], str] | None = None) -> Iterator[None]: + """Bind one caller's reprepare and optional confirmed-refusal classification.""" bindings = dict(_REPREPARE.get() or {}) - bindings[role] = callback + bindings[role] = (callback, resource_reason) token = _REPREPARE.set(bindings) try: yield @@ -458,7 +467,8 @@ class TaskModelWait: def reprepare(self, role: str, kwargs: dict) -> dict | PreparedModelCall: """A route change must rebind any already-prepared Main fit authority.""" - callback = (_REPREPARE.get() or {}).get(role) + binding = (_REPREPARE.get() or {}).get(role) + callback = binding[0] if binding else None if callback is not None: return callback(copy.deepcopy(kwargs)) from ouroboros.usage_accounting import current_physical_attempt_context @@ -600,7 +610,11 @@ class TaskModelWait: from ouroboros.provider_models import parse_claudexor_model role = kwargs["model_role"] - source, native_model = parse_claudexor_model(kwargs["model"]) + try: + source, native_model = parse_claudexor_model(kwargs["model"]) + except ValueError: + source, native_model = "", kwargs["model"] # no configured non-generating access observer + observes_access = bool(source) route = getattr(error, "route", {}) or {} problem_context = (getattr(error, "problem", {}) or {}).get("context") or {} account_intent = kwargs.get("model_account_override") @@ -610,13 +624,14 @@ class TaskModelWait: scope = current_usage_scope() slot_id = str(getattr(scope, "review_slot_id", "") or "") reason = model_wait_reason(error) - quota_wait = reason in {"quota", "auth_quota"} + quota_wait = reason in {"quota", "auth_quota"} # unavailable does not prove a quota window row = {"wait_id": wait_id, "task_attempt": self.attempt, "role": role, "_slot_id": slot_id, "model": kwargs["model"], "source": source, "credential_profile_id": "" if reason == "auth_quota" else str(route.get("credentialProfileId") or problem_context.get("credentialProfileId") or account_intent or ""), "credential_harness": "", "reason": reason, "reset_at": str(getattr(error, "reset_at", "") or ""), - "auto_continue": self.auto_continue.get(role, True), "state": "waiting", + "auto_continue": self.auto_continue.get(role, True) if observes_access else False, "state": "waiting", + **({"availability_observation": "unavailable"} if not observes_access else {}), "worker_slot_held": self.worker_slot_held, "started_at": utc_now_iso()} if self.owner_id: row["model_wait_owner_id"] = self.owner_id @@ -662,7 +677,7 @@ class TaskModelWait: now = time.monotonic() if row.pop("_check_now", False): next_check = 0.0 - if now >= next_check: + if observes_access and now >= next_check: try: if not row["credential_harness"]: sources = llm.claudexor_model_sources() @@ -791,15 +806,19 @@ def model_waitable(function: Callable | None = None, *, client_parameter: str = from ouroboros.llm_claudexor import ClaudexorModelError capture = getattr(error, "physical_attempt_capture", None) + binding = (_REPREPARE.get() or {}).get(values.get("model_role")) + classified = binding[1](error) if binding and binding[1] is not None else "" + if classified: + error.task_resource_wait_reason = classified reason = model_wait_reason(error) if not (context and not context.closed and values.get("model_role") and reason - and isinstance(error, ClaudexorModelError) - and getattr(capture, "state", None) in {"released", "settled"}): + and (classified or (isinstance(error, ClaudexorModelError) + and getattr(capture, "state", None) in {"released", "settled"}))): return False deferral = _deferral(values["model_role"]) - # Quota: account rotation, then every configured route, then the owner. Sign-in - # keeps its own owner wait wherever waiting is possible. - if deferral is not None and (reason != "auth" or not context.waits_allowed): + # Account rotation, then every configured route, then the owner: sign-in too, so a + # healthy configured route keeps working while the refused account awaits repair. + if deferral is not None: deferral.retain(receiver, error, values) return False return bool(context.waits_allowed and values.get("wait_for_resources", True)) diff --git a/ouroboros/provider_models.py b/ouroboros/provider_models.py index a039f5f28..1dbd18937 100644 --- a/ouroboros/provider_models.py +++ b/ouroboros/provider_models.py @@ -242,11 +242,13 @@ def resolve_model_target( ) -def fallback_candidate_targets(active_model: str = "") -> tuple[ResolvedModelTarget, ...]: +def fallback_candidate_targets(active_model: str = "", *, preserve_slots: bool = False) -> tuple[ResolvedModelTarget, ...]: """The cross-model fallback candidate ladder as typed targets (ABI-4). Same membership and order as ``model_slots.get_fallback_models`` — a typed - view over the ONE chain SSOT, not a second resolver. Effort stays the "" + view over the ONE chain SSOT, not a second resolver. ``preserve_slots`` keeps + repeated model ids so callers can compare their distinct configured account bindings. + Effort stays the "" sentinel: the ladder resolves destinations, the dispatching round owns the active effort. ``provider_route`` stays the ``""`` sentinel DELIBERATELY: the chain's local-vs-remote dispatch lane is the loop's single global @@ -254,11 +256,11 @@ def fallback_candidate_targets(active_model: str = "") -> tuple[ResolvedModelTar through the ABI-4 sweep), so a per-candidate route here would be a fabricated fact no dispatcher consumes. """ - from ouroboros.model_slots import get_fallback_models + from ouroboros.model_slots import get_fallback_models, parse_fallback_chain return tuple( ResolvedModelTarget(model_id=model, provider_route="") - for model in get_fallback_models(active_model) + for model in (parse_fallback_chain() if preserve_slots else get_fallback_models(active_model)) ) diff --git a/ouroboros/size_ratchet_manifest.py b/ouroboros/size_ratchet_manifest.py index fc65b31d5..5fa3b69e8 100644 --- a/ouroboros/size_ratchet_manifest.py +++ b/ouroboros/size_ratchet_manifest.py @@ -128,6 +128,7 @@ BAND_PATHS = { "ouroboros/loop_delivery.py": "F6 upstream sync: the delivery-protocol upstream deltas (hold-control literals, trailing-object/fence-aware protocol parsers) folded into the campaign delivery leaf (upstream leaf delivery_protocol.py retired)", "ouroboros/loop_forced_finalization.py": "Forced-finalization rail of the v7 L-B loop split: one cohesive owner for the forced/orphan/absorption path, moved byte-preserving from loop.py (D01 lane).", "ouroboros/loop_model_call.py": "Fallback waits retain the exact quota-refusing route across failures without inventing a second router.", + "ouroboros/loop_transport.py": "Owns the existing wait episode and terminal evidence across direct and queued recovery, retaining distinct Presence behavior.", "ouroboros/marketplace/ouroboroshub.py": "Entered the band from 373 lines: the hubflow sprint added the adopt transaction (eligibility prelude, CAS re-verification, move-aside + state-quintet snapshot, verified rollback with per-step error collection, retention finalize) beside the existing install/update flows (hubflow sprint, adopt-in-ouroboroshub owner decision D4).", "ouroboros/mcp_client.py": "F3.1 typed-organ producer cutover (D05 entry 7b) plus E5+s2r2 (#447): the MCP transport keeps the SDK-owned error bit as a typed ToolResult, follows nextCursor pagination with injective 12-hex slugs, and discloses collision/pagination omissions; grew into the band from 984 lines, shrink-only otherwise.", "ouroboros/memory.py": "ibl-2b09abdadd25: scratchpad content-size cap added alongside the existing block-count cap in append_scratchpad_block's eviction loop", diff --git a/ouroboros/tools/control.py b/ouroboros/tools/control.py index 24d748ade..3cd7c257b 100644 --- a/ouroboros/tools/control.py +++ b/ouroboros/tools/control.py @@ -409,11 +409,20 @@ def get_tools() -> List[ToolEntry]: "name": "switch_model", "description": "Switch to a different LLM model or reasoning effort level. " "Use when you need more power (complex code, deep reasoning) " - "or want to save budget (simple tasks). Takes effect on next round.", + "or want to save budget (simple tasks). Takes effect on next round. " + "After the host moved this turn to a configured fallback, primary='return' " + "or 'wait' goes back to the turn's primary route.", "parameters": {"type": "object", "properties": { "model": {"type": "string", "description": "Model name (e.g. anthropic/claude-sonnet-4). Leave empty to keep current."}, "effort": {"type": "string", "enum": list(EFFORT_SCALE), "description": "Reasoning effort level (adapted down per route when a model tops out lower). Leave empty to keep current."}, + "primary": {"type": "string", "enum": ["return", "wait"], + "description": ("Omit to keep the current route. Return to this turn's primary route: its model, role and account policy " + "(Auto stays Auto) plus the owner's wait-card choice; effort stays. 'return': " + "if the primary refuses, configured routes are tried again. 'wait': if it " + "refuses or is unreachable, wait for it where this turn may wait instead of " + "paid alternatives. The next real request tests it; no timer or probe does. " + "Not with model.")}, }, "required": []}, }, _switch_model), ToolEntry("get_task_result", { diff --git a/ouroboros/tools/control_runtime.py b/ouroboros/tools/control_runtime.py index 840e220ee..32eedfed5 100644 --- a/ouroboros/tools/control_runtime.py +++ b/ouroboros/tools/control_runtime.py @@ -481,10 +481,14 @@ def _set_next_wakeup(ctx: ToolContext, seconds: int) -> str: "next wake-up (a wake-up already pending keeps its time; a wake-up calling this sets its own next one).") -def _switch_model(ctx: ToolContext, model: str = "", effort: str = "") -> str: +def _switch_model(ctx: ToolContext, model: str = "", effort: str = "", primary: str = "") -> str: """LLM-driven model/effort switch (Constitution P5: LLM-first). - Stored in ToolContext, applied on the next LLM call in the loop. + Stored in ToolContext, applied on the next LLM call in the loop. ``primary`` + returns to this turn's primary binding (the acting model's choice): its model, role, + locality and account policy with the owner's live wait-card choice for that + role; "wait" also keeps a refusal there on the primary's own wait instead of + paid alternatives. Effort intent is untouched. """ from ouroboros.config import EFFORT_SCALE from ouroboros.llm import LLMClient @@ -496,8 +500,26 @@ def _switch_model(ctx: ToolContext, model: str = "", effort: str = "") -> str: requested_effort = str(effort or "").strip().lower() if requested_effort and requested_effort not in EFFORT_SCALE: return _publish_tool_result(ctx, ToolResult(status="error", code="TOOL_ARG_ERROR", text=(f"⚠️ Unknown effort: {effort}. Valid: {', '.join(EFFORT_SCALE)}"))) + requested_primary = str(primary or "").strip().lower() + route = getattr(ctx, "primary_route", None) + if requested_primary and (requested_primary not in ("return", "wait") or model): + return _publish_tool_result(ctx, ToolResult(status="error", code="TOOL_ARG_ERROR", text=( + "⚠️ primary must be 'return' or 'wait', without model."))) + if requested_primary and not (isinstance(route, dict) and route.get("model")): + return _publish_tool_result(ctx, ToolResult(status="unavailable", code="CAPABILITY_UNAVAILABLE", text=( + "⚠️ This turn has no recorded primary route to return to."))) - if model: + if requested_primary: + waiter = getattr(ctx, "model_wait_context", None) + chosen = ((getattr(waiter, "overrides", None) or {}).get(route["role"]) or {}) + ctx.active_model_override = str(chosen.get("model") or route["model"]) + ctx.active_use_local_override = bool(chosen.get("use_local", route["use_local"])) + ctx.active_role_override = route["role"] + ctx.route_wait_on_primary = requested_primary == "wait" + changes.append(f"primary route {ctx.active_model_override}{' (local)' if ctx.active_use_local_override else ''}" + f" (role {route['role']}; if it refuses: " + f"{'wait for it' if requested_primary == 'wait' else 'configured routes again'})") + elif model: if model not in available: return _publish_tool_result(ctx, ToolResult(status="error", code="TOOL_ARG_ERROR", text=(f"⚠️ Unknown model: {model}. Available: {', '.join(available)}"))) @@ -513,6 +535,7 @@ def _switch_model(ctx: ToolContext, model: str = "", effort: str = "") -> str: ctx.active_model_override = model ctx.active_use_local_override = use_local + ctx.route_wait_on_primary, ctx.active_role_override = False, None # an explicit route ends a declared wait changes.append(f"model={model}{' (local)' if use_local else ''}") if requested_effort: @@ -520,6 +543,7 @@ def _switch_model(ctx: ToolContext, model: str = "", effort: str = "") -> str: changes.append(f"effort={requested_effort}") if not changes: - return f"Current available models: {', '.join(available)}. Pass model and/or effort to switch." + return (f"Current available models: {', '.join(available)}. Pass model and/or effort to switch, " + "or primary='return'/'wait' to go back to this turn's primary route.") return f"OK: switching to {', '.join(changes)} on next round." diff --git a/tests/test_llm_claudexor.py b/tests/test_llm_claudexor.py index 96a0c7bc4..6fc1ab022 100644 --- a/tests/test_llm_claudexor.py +++ b/tests/test_llm_claudexor.py @@ -77,7 +77,7 @@ def test_display_redacts_typed_details_without_mutating_custody(): ("invalid_request", 400, False, "bad_request", False, "", 42), ("invalid_request", 400, False, "bad_request", False, "", " "), ("auth_required", 401, False, "auth_error", False, "auth", "context_length_exceeded"), - ("subscription_window_exhausted", 429, False, "subscription_window_exhausted", True, "quota", "context_length_exceeded"), + ("subscription_window_exhausted", 429, False, "subscription_window_exhausted", False, "quota", "context_length_exceeded"), ("unsupported_parameter", 400, False, "bad_request", False, "", "context_length_exceeded"), ("invalid_request", 400, True, "provider_outcome_unknown", False, "", "context_length_exceeded"), ]) diff --git a/tests/test_loop_compaction.py b/tests/test_loop_compaction.py index a25bd5973..df2bc4343 100644 --- a/tests/test_loop_compaction.py +++ b/tests/test_loop_compaction.py @@ -726,6 +726,7 @@ def test_overflow_retry_is_skipped_while_the_round_holds_an_unresolved_attempt(t context = _ctx(tmp_path) context.llm = _DeathThenOverflow() + context.task_type = "presence" # inline Presence alone keeps the paid transport-death repeat fits = iter([_fit(), _fit(profile="task_local_low", mode="low")]) reclaims = [] diff --git a/tests/test_loop_transport_wait.py b/tests/test_loop_transport_wait.py index 0c4519768..9549b6520 100644 --- a/tests/test_loop_transport_wait.py +++ b/tests/test_loop_transport_wait.py @@ -3,7 +3,8 @@ Covers: typed classification of released pre-dispatch transport failures (remote vs local provider), the one-physical-attempt-per-call contract, the round-level wait episode (free redials, recovery, deterministic no-resend -terminal, the interactive turns' idle-timeout bound, local-only fallback pass), +terminal, the interactive turns' idle-timeout bound, the one configured-route +walk tried BEFORE the wait opens and never inside it), the owner-signal-interruptible sleep, and durable ``network_wait`` evidence. Interactive-episode contracts continue in ``test_loop_transport_wait_interactive.py``. """ @@ -208,6 +209,27 @@ def _transport_failing_call(fail_times: int, final_content: str = "done"): return fake_call, calls +def _counting_chain(kind=None, answer_on=None): + """A configured-route walk that answers on call ``answer_on`` or fails, optionally + overwriting the mutable kind with a candidate's own failure class.""" + calls = {"n": 0, "kinds": []} + + def chain(**kwargs): + calls["n"] += 1 + usage = kwargs["accumulated_usage"] + calls["kinds"].append(usage.get("_last_llm_error_kind")) + if answer_on is not None and calls["n"] == answer_on: + usage.pop("_last_llm_error_kind", None) + return ({"role": "assistant", "content": "fallback-ok"}, "other/model", False, + kwargs["context_fit_plan"], kwargs["active_context_mode"]) + if kind is not None: + usage["_last_llm_error_kind"] = kind + return (None, kwargs["active_model"], kwargs["active_use_local"], + kwargs["context_fit_plan"], kwargs["active_context_mode"]) + + return chain, calls + + def test_transport_outage_waits_redials_free_rounds_and_recovers(tmp_path, monkeypatch): fake_call, calls = _transport_failing_call(fail_times=3) sleeps = [] @@ -272,14 +294,13 @@ def test_interactive_turns_wait_redial_free_and_terminalize_at_the_idle_bound(tm def test_deadline_bounds_wait_with_one_last_free_redial_then_no_resend(tmp_path, monkeypatch): + """One provider's outage proves no dead egress: the configured routes are tried + once BEFORE the wait (remote included), never again inside it.""" fake_call, calls = _transport_failing_call(fail_times=99) clock = _FakeClock(monkeypatch) monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call) - - def _chain_must_not_run(**_kwargs): - raise AssertionError("remote fallback chain must not dial during a transport outage") - - monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", _chain_must_not_run) + chain, chain_calls = _counting_chain() + monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", chain) monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "other/model") monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) @@ -302,6 +323,7 @@ def test_deadline_bounds_wait_with_one_last_free_redial_then_no_resend(tmp_path, assert "waited and redialed" in result # honest waited-out terminal text phases = [row["phase"] for row in _read_network_wait_events(tmp_path)] assert phases[-1] == "ended" + assert chain_calls["kinds"] == ["transport_unavailable"] * calls["n"] # alternatives follow every paced failure def test_deadline_refusal_during_episode_takes_transport_no_resend_terminal(tmp_path, monkeypatch): @@ -405,6 +427,7 @@ def test_outage_first_observed_mid_chain_latches_episode_and_recovers(tmp_path, monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call) monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", chain_breaks_on_transport) monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "other/model") # the walk needs a configured route monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) registry = ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path) if turn_flag: @@ -453,6 +476,7 @@ def test_mid_chain_latch_that_never_recovers_takes_the_no_resend_terminal(tmp_pa monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call) monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", chain_breaks_on_transport) monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "other/model") # the walk needs a configured route monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) registry = ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path) if turn_flag: @@ -465,7 +489,7 @@ def test_mid_chain_latch_that_never_recovers_takes_the_no_resend_terminal(tmp_pa notes = [] _result, usage, trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, notes)) - assert chain_calls["n"] == 1 # the chain never re-dials over the dead egress + assert chain_calls["n"] == calls["n"] # one failed provider does not exclude the other routes assert calls["n"] >= 2 assert calls["n"] == len(clock.sleeps) + 1 # every dispatch after the first followed a wait assert usage.get("execution_status") == "infra_failed" @@ -507,118 +531,67 @@ def test_exact_model_route_waits_and_redials_its_own_pin(tmp_path, monkeypatch): assert {route[0] for route in calls["routes"]} == {"test-model"} # only the pin dialed -def test_local_fallback_pass_adopts_local_route_when_configured(tmp_path, monkeypatch): +@pytest.mark.parametrize("local", [False, True]) +def test_configured_route_answers_before_any_wait_opens(tmp_path, monkeypatch, local): + """#1409 case 4: a provider that cannot be reached is not a dead network. Any + configured route (remote or local) answers before a wait opens: no episode rows.""" fake_call, calls = _transport_failing_call(fail_times=99) - chain_calls = {"n": 0} - - def fake_chain(**kwargs): - chain_calls["n"] += 1 - return ( - {"role": "assistant", "content": "local-ok"}, "local/candidate", True, - kwargs["context_fit_plan"], kwargs["active_context_mode"], - ) - + chain, chain_calls = _counting_chain(answer_on=1) monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call) - monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", fake_chain) + monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", chain) monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") - monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "local/candidate") - monkeypatch.setenv("USE_LOCAL_FALLBACK", "true") + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "other/model") + monkeypatch.setenv("USE_LOCAL_FALLBACK", "true" if local else "false") notes = [] result, _usage, _trace = run_llm_loop(**_loop_kwargs(tmp_path, ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path), notes)) - assert result == "local-ok" - assert chain_calls["n"] == 1 # Q4: the LOCAL chain dialed, exactly once - events = _read_network_wait_events(tmp_path) - assert [row["phase"] for row in events] == ["entered", "ended"] - assert events[-1].get("detail") == "local_fallback_adopted" + assert result == "fallback-ok" + assert calls["n"] == 1 and chain_calls["n"] == 1 + assert _read_network_wait_events(tmp_path) == [] + assert not any("provider connection" in note.lower() for note in notes) -def test_failed_local_pass_keeps_remote_cause_and_runs_at_most_once(tmp_path, monkeypatch): - """A failed local walk overwrites the mutable error kind, but the latched - remote cause keeps the episode waiting — and the local pass never re-dials.""" +@pytest.mark.parametrize("candidate_kind", ["provider_error", "provider_outcome_unknown"]) +def test_failed_route_walk_keeps_the_rounds_outage_across_paced_recovery( + tmp_path, monkeypatch, candidate_kind): + """A failed walk overwrites the mutable kind with a candidate's class (even an + eligible unknown one), but the round's own outage keeps the episode waiting on its + own route and clock while configured alternatives remain eligible.""" fake_call, calls = _transport_failing_call(fail_times=3) - chain_calls = {"n": 0} - - def failing_chain(**kwargs): - chain_calls["n"] += 1 - kwargs["accumulated_usage"]["_last_llm_error_kind"] = "provider_error" - return ( - None, kwargs["active_model"], kwargs["active_use_local"], - kwargs["context_fit_plan"], kwargs["active_context_mode"], - ) - + chain, chain_calls = _counting_chain(kind=candidate_kind) sleeps = [] monkeypatch.setattr(loop_transport, "interruptible_wait_sleep", lambda sec, _wake: (sleeps.append(sec), False)[1]) monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call) - monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", failing_chain) + monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", chain) + monkeypatch.setattr(loop_transport, "upstream_transport_reachable", + lambda *_a, **_kw: pytest.fail("an outage episode redials; it probes nothing")) monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") - monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "local/candidate") - monkeypatch.setenv("USE_LOCAL_FALLBACK", "true") + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "other/model") notes = [] result, _usage, _trace = run_llm_loop(**_loop_kwargs(tmp_path, ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path), notes)) assert result == "done" # the episode kept waiting and recovered - assert chain_calls["n"] == 1 # one local pass per episode + assert chain_calls["n"] == 3 # each failed primary round offers the alternatives assert calls["n"] == 4 assert len(sleeps) == 3 phases = [row["phase"] for row in _read_network_wait_events(tmp_path)] - assert phases[-1] == "recovered" + assert phases[0] == "entered" and phases[-1] == "recovered" -def _unknown_outcome_chain(): - chain_calls = {"n": 0} - - def failing_chain(**kwargs): - chain_calls["n"] += 1 - kwargs["accumulated_usage"]["_last_llm_error_kind"] = "provider_outcome_unknown" - return ( - None, kwargs["active_model"], kwargs["active_use_local"], - kwargs["context_fit_plan"], kwargs["active_context_mode"], - ) - - return failing_chain, chain_calls - - -def test_failed_local_pass_with_unknown_outcome_keeps_episode_waiting(tmp_path, monkeypatch): - """Hardening for the refuted local-pass finding: a local pass that dies - with provider_outcome_unknown overwrites the mutable kind, but the latched - remote cause keeps the episode waiting until the egress recovers.""" - fake_call, calls = _transport_failing_call(fail_times=3) - failing_chain, chain_calls = _unknown_outcome_chain() - sleeps = [] - monkeypatch.setattr(loop_transport, "interruptible_wait_sleep", - lambda sec, _wake: (sleeps.append(sec), False)[1]) - monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call) - monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", failing_chain) - monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") - monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "local/candidate") - monkeypatch.setenv("USE_LOCAL_FALLBACK", "true") - notes = [] - result, _usage, _trace = run_llm_loop(**_loop_kwargs(tmp_path, ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path), notes)) - - assert result == "done" # the episode kept waiting and recovered - assert chain_calls["n"] == 1 # one local pass per episode - assert calls["n"] == 4 - assert len(sleeps) == 3 - phases = [row["phase"] for row in _read_network_wait_events(tmp_path)] - assert phases[-1] == "recovered" - - -def test_unknown_outcome_local_pass_then_deadline_takes_transport_no_resend(tmp_path, monkeypatch): - """Same shape, but the deadline expires while the egress stays dead: the - terminal must key on the episode's latched cause — transport no-resend — - not on the local pass's provider_outcome_unknown overwrite.""" +def test_unknown_candidate_then_deadline_takes_the_rounds_transport_no_resend(tmp_path, monkeypatch): + """Same shape, but the deadline expires while the provider stays unreachable: the + terminal keys on the episode's latched cause — transport no-resend — not on the + candidate's provider_outcome_unknown overwrite.""" fake_call, calls = _transport_failing_call(fail_times=99) - failing_chain, chain_calls = _unknown_outcome_chain() + chain, chain_calls = _counting_chain(kind="provider_outcome_unknown") sleeps = [] monkeypatch.setattr(loop_transport, "interruptible_wait_sleep", lambda sec, _wake: (sleeps.append(sec), False)[1]) monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call) - monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", failing_chain) + monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", chain) monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") - monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "local/candidate") - monkeypatch.setenv("USE_LOCAL_FALLBACK", "true") + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "other/model") from ouroboros.config import get_finalization_grace_sec registry = ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path) @@ -627,7 +600,7 @@ def test_unknown_outcome_local_pass_then_deadline_takes_transport_no_resend(tmp_ notes = [] _result, usage, trace = run_llm_loop(**_loop_kwargs(tmp_path, registry, notes)) - assert chain_calls["n"] == 1 + assert chain_calls["n"] == calls["n"] assert calls["n"] >= 2 assert usage.get("execution_status") == "infra_failed" assert usage.get("reason_code") == "provider_unavailable" @@ -831,27 +804,22 @@ def test_redial_failing_with_different_kind_ends_episode_and_resumes_fallback(tm ) return None, 0.0 - chain_calls = {"n": 0} - - def fake_chain(**kwargs): - chain_calls["n"] += 1 - kwargs["accumulated_usage"].pop("_last_llm_error_kind", None) - return ( - {"role": "assistant", "content": "fallback-ok"}, "other/model", False, - kwargs["context_fit_plan"], kwargs["active_context_mode"], - ) + # The walk before the wait finds no route; after the episode ends the ordinary + # walk runs again for the redial's own failure and a route answers. + fake_chain, chain_calls = _counting_chain(answer_on=2) monkeypatch.setattr(loop_transport, "interruptible_wait_sleep", lambda _sec, _wake: False) monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call) monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", fake_chain) monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "other/model") monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) notes = [] result, usage, _trace = run_llm_loop(**_loop_kwargs(tmp_path, ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path), notes)) assert result == "fallback-ok" assert calls["n"] == 2 # primary, then the one free redial - assert chain_calls["n"] == 1 # ordinary policy resumed after the episode ended + assert chain_calls["kinds"] == ["transport_unavailable", "provider_transient"] # before the wait, then ordinary policy assert usage.get("reason_code") is None events = _read_network_wait_events(tmp_path) assert events[-1]["phase"] == "ended" @@ -873,12 +841,11 @@ def test_redial_unknown_waits_without_unproved_paid_continuation(tmp_path, monke accumulated_usage.update(execution_status="infra_failed", reason_code="llm_api_error") return None, 0.0 - def _chain_must_not_run(**_kwargs): - raise AssertionError("no fallback chain after an unknown-outcome redial") + chain, chain_calls = _counting_chain() monkeypatch.setattr(loop_transport, "interruptible_wait_sleep", lambda _sec, _wake: False) monkeypatch.setattr(loop_mod, "call_llm_with_retry", fake_call) - monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", _chain_must_not_run) + monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", chain) monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "other/model") monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) @@ -894,6 +861,8 @@ def test_redial_unknown_waits_without_unproved_paid_continuation(tmp_path, monke assert probes == [1] assert calls["n"] == 2 # zero dials after the unknown outcome + assert chain_calls["kinds"][0] == "transport_unavailable" + assert set(chain_calls["kinds"][1:]) == {"provider_outcome_unknown"} # configured recovery stays available assert usage.get("execution_status") == "infra_failed" assert trace.get("forced_finalization", {}).get("source") == "provider_outcome_unknown_no_resend" events = _read_network_wait_events(tmp_path) diff --git a/tests/test_loop_transport_wait_interactive.py b/tests/test_loop_transport_wait_interactive.py index 98b343b92..6c22c8989 100644 --- a/tests/test_loop_transport_wait_interactive.py +++ b/tests/test_loop_transport_wait_interactive.py @@ -442,36 +442,11 @@ def test_error_kind_change_closure_is_an_interactive_note(tmp_path, monkeypatch, assert incident is None -@pytest.mark.parametrize("flags", [{}, {"is_direct_chat": True}]) -def test_local_fallback_adoption_closure_is_an_interactive_note(tmp_path, monkeypatch, flags): - """Adopting the local fallback route closes the episode with a durable row - for every episode; the owner note saying the remote connection is still - down is an interactive turn's only closure surface, so a managed episode - gets none.""" - _FakeClock(monkeypatch) - ctx = _ctx(**flags) - notes = _NoteRecorder() - episode = _enter(tmp_path, ctx, notes, task_id="t-local") - assert loop_transport.reconcile_transport_wait( - episode, ctx, msg_present=True, error_kind="", drive_logs=tmp_path, - task_id="t-local", model="m", emit_progress=notes, after_local_pass=True, - ) is None - - assert _read_network_wait_events(tmp_path)[-1]["detail"] == "local_fallback_adopted" - if not flags: - assert notes.texts == [BASE_MANAGED_ENTRY] # the managed closure is its row, not a note - return - assert "still unavailable" in notes.texts[-1] - assert "local fallback model" in notes.texts[-1] - incident = notes.incidents[-1] - assert incident is None - - -@pytest.mark.parametrize("closure", ["recovered", "local_fallback_adopted", "error_kind_changed"]) +@pytest.mark.parametrize("closure", ["recovered", "error_kind_changed"]) def test_managed_episode_owner_texts_are_byte_identical_to_base(tmp_path, monkeypatch, closure): """The managed wordings are a frozen contract: entry still ends with the Stop promise its cancel authority honors, recovery still says resuming, - and the two other closures write their durable row and no note at all — + and the other closure writes its durable row and no note at all — the literal texts the base emits, nothing more.""" clock = _FakeClock(monkeypatch) ctx = _ctx() @@ -480,7 +455,6 @@ def test_managed_episode_owner_texts_are_byte_identical_to_base(tmp_path, monkey clock.now += 90.0 outcome = { "recovered": dict(msg_present=True, error_kind=""), - "local_fallback_adopted": dict(msg_present=True, error_kind="", after_local_pass=True), "error_kind_changed": dict(msg_present=False, error_kind="provider_transient"), }[closure] assert loop_transport.reconcile_transport_wait( @@ -493,7 +467,6 @@ def test_managed_episode_owner_texts_are_byte_identical_to_base(tmp_path, monkey last = _read_network_wait_events(tmp_path)[-1] assert (last["phase"], last.get("detail")) == { "recovered": ("recovered", None), - "local_fallback_adopted": ("ended", "local_fallback_adopted"), "error_kind_changed": ("ended", "error_kind_changed:provider_transient"), }[closure] diff --git a/tests/test_model_wait.py b/tests/test_model_wait.py index f32e69483..1e260cec6 100644 --- a/tests/test_model_wait.py +++ b/tests/test_model_wait.py @@ -288,13 +288,20 @@ def test_wait_projection_refuses_corrupt_existing_task_result(tmp_path): assert path.read_bytes() == b'{"broken":' -def test_unknown_or_generic_empty_pool_never_becomes_quota_wait(live_wait): - _root, transport, client, _controller, events, _decide = live_wait - transport.results = [_refusal("credential_pool_exhausted")] - transport.dispatch = ["not_started"] - with pytest.raises(Exception) as raised: - client.chat([{"role": "user", "content": "hi"}], MODEL, model_role="main") - assert raised.value.code == "credential_pool_exhausted" and events.empty() +def test_dated_pool_of_unknown_cause_waits_visibly_as_unavailable_never_as_quota(elapsed_quota_wait): + """#1409 case 1: an engine-dated pool refusal is neither quota nor sign-in. With no later + route it takes the visible, interruptible live wait under its own reason and heals on the + SAME call; it never sleeps to the reset and never reads as quota.""" + root, transport, client, controller, events, _decide = elapsed_quota_wait + transport.results = [_refusal("credential_pool_exhausted"), result()] + transport.dispatch = ["not_started", "response_received"] + answer, usage = client.chat([{"role": "user", "content": "hi"}], MODEL, model_role="main") + assert answer == result()["message"] and len(usage["ledger_attempt_ids"]) == 2 + rows = list(events.queue) + assert rows[0]["reason"] == rows[-1]["reason"] == "unavailable" + assert rows[0]["reset_at"] == "2099-01-01T00:00:00Z" and rows[0]["quota_clock"]["active"] is False + assert rows[-1]["resolution"] == "resource_available" and controller.paused_seconds() == 0 + assert load_task_result(root, "task-one")["model_waits"][rows[0]["wait_id"]]["reason"] == "unavailable" def test_async_cancellation_resolves_only_its_wait_and_keeps_shared_task(live_wait, monkeypatch): diff --git a/tests/test_model_wait_browser.py b/tests/test_model_wait_browser.py index ae285553e..b8817d2d0 100644 --- a/tests/test_model_wait_browser.py +++ b/tests/test_model_wait_browser.py @@ -312,6 +312,54 @@ def test_mixed_access_wait_replays_and_opens_accounts_without_automatic_login(wa assert ui['logins'] == [] +def test_dated_unavailable_pool_waits_visibly_without_a_sign_in_prompt(waiting_ui): + """An engine-dated pool refusal is neither quota nor sign-in: the card says so, names the + engine's reset, keeps automatic continuation and never offers a login.""" + ui, page = waiting_ui, waiting_ui['page'] + ui['rows']['light-wait'].update(revision=2, reason='unavailable', credential_profile_id='', + reset_at='2026-09-07T01:00:00Z') + ui['emit'](ui['rows']['light-wait']) + pool = page.locator('[data-wait-id="light-wait"]') + page.wait_for_function("() => document.querySelector('[data-wait-id=light-wait] [data-wait-reason]').textContent === 'Waiting for an account'") + assert pool.locator('[data-wait-login]').is_hidden() + assert pool.locator('[data-wait-auto]').is_checked() + assert pool.locator('[data-wait-auto-label]').inner_text() == 'Continue automatically when access is restored' + reset = pool.locator('[data-wait-reset]').inner_text() + assert 'No account can serve this model now.' in reset and 'availability is unconfirmed' in reset + assert 'Quota' not in reset and 'sign-in' not in pool.inner_text().lower() + capture(page, 'waiting-unavailable-pool') + page.set_viewport_size({'width': 390, 'height': 844}) + assert page.evaluate('document.documentElement.scrollWidth <= window.innerWidth') + capture(page, 'waiting-unavailable-pool-narrow') + ui['rows']['light-wait'].update(revision=3, reset_at=None) + ui['emit'](ui['rows']['light-wait']) + page.wait_for_function("() => document.querySelector('[data-wait-id=light-wait] [data-wait-reset]').textContent.includes('The reset time is not known.')") + page.reload() + page.wait_for_selector('[data-wait-id="light-wait"]') + assert pool.locator('[data-wait-reason]').inner_text() == 'Waiting for an account' + assert ui['logins'] == [] + + +def test_direct_api_wait_never_offers_subscription_login_or_auto_rotation(waiting_ui): + ui, page = waiting_ui, waiting_ui['page'] + ui['rows']['light-wait'].update(revision=2, reason='auth', model='openai::primary-test', source='', + credential_profile_id='', credential_harness='', auto_continue=False, availability_observation='unavailable') + ui['emit'](ui['rows']['light-wait']) + card = page.locator('[data-wait-id="light-wait"]') + page.wait_for_function("() => document.querySelector('[data-wait-id=light-wait] [data-wait-reason]').textContent === 'Waiting for provider access'") + assert card.locator('[data-wait-login]').is_hidden() and card.locator('[data-wait-auto]').is_hidden() + assert 'Configured API access' in card.inner_text() and 'Auto rotation' not in card.inner_text() + assert 'Automatic access checks are unavailable.' in card.inner_text() + capture(page, 'waiting-direct-api-access') + page.set_viewport_size({'width': 390, 'height': 844}) + assert page.evaluate('document.documentElement.scrollWidth <= window.innerWidth') + capture(page, 'waiting-direct-api-access-narrow') + page.reload() + page.wait_for_selector('[data-wait-id="light-wait"]') + assert card.locator('[data-wait-login]').is_hidden() + assert ui['logins'] == [] + + def test_wait_reconnect_preserves_unsubmitted_form_without_context_or_animation(waiting_ui): ui, page = waiting_ui, waiting_ui["page"] row = page.locator('[data-wait-id="main-wait"]') diff --git a/tests/test_model_wait_controls.py b/tests/test_model_wait_controls.py index b0eed52f2..8558a41a9 100644 --- a/tests/test_model_wait_controls.py +++ b/tests/test_model_wait_controls.py @@ -362,6 +362,8 @@ def test_outage_wrap_keeps_older_wire_death_custody_without_summary(tmp_path, mo monkeypatch.setattr(loop_transport, "interruptible_wait_sleep", release_wait_after_control_check) kwargs = _loop_kwargs(tmp_path, ControlledLLM(), []) kwargs["tools"]._ctx.is_direct_chat = interactive + if interactive: # inline Presence, the one direct caller that keeps the paid-repeat rail + kwargs["task_type"] = kwargs["tools"]._ctx.current_task_type = "presence" with model_wait.task_model_wait_scope(task={"id": "t-death"}, drive_root=tmp_path, event_queue=None, worker_slot_held=not interactive) as owner: owner.tool_context = kwargs["tools"]._ctx diff --git a/tests/test_module_handle_extraction.py b/tests/test_module_handle_extraction.py index 08a45f648..3242f6368 100644 --- a/tests/test_module_handle_extraction.py +++ b/tests/test_module_handle_extraction.py @@ -378,7 +378,9 @@ LEAVES: dict[str, tuple[str, str, frozenset[str]]] = { "_record_owner_directive", })), "ouroboros/loop_model_call.py": ("ouroboros/loop.py", "_loop", frozenset({ - "_RoundModelCallContext", "_account_compaction_usage", "_call_round_model", + "_RoundModelCallContext", "_account_compaction_usage", "_append_or_merge_user_message", + "_apply_runtime_overrides", "_call_round_model", "_fallback_chain_allowed", + "_reconcile_transport_wait", "_run_cross_model_fallback_chain", "_context_overflow_retries", "_context_reclaim_materializations", "_context_reclaim_passes", "_dispatch_round_model", "_emit_checkpoint_event", "_measure_round_main_fit", "_rebind_context_fit_plan", "_run_main_reclaim", diff --git a/tests/test_native_fallback_binding_acceptance.py b/tests/test_native_fallback_binding_acceptance.py new file mode 100644 index 000000000..af8d1672b --- /dev/null +++ b/tests/test_native_fallback_binding_acceptance.py @@ -0,0 +1,309 @@ +"""Independent #1407/#1409 native request proofs, proposed for the frozen candidate. + +Uses existing main_call/setup fixtures and the real loop, rebind, LLMClient, +Claudexor serializer and physical ledger. No provider calls. Not run by the scout. +""" + +from copy import deepcopy +from dataclasses import replace +import json +import queue +from types import SimpleNamespace + +import pytest +import httpx + +from ouroboros import context, fallback_cooldown, llm_claudexor, loop +from ouroboros import loop_transport +from ouroboros.gateways.claudexor import ClaudexorUnavailable +from ouroboros.model_slots import MODEL_ACCOUNTS_KEY +from ouroboros.model_wait import ModelWaitInterrupted +from ouroboros.tools.control_runtime import _switch_model +from tests.test_llm_claudexor import MODEL, ROUTE, ledger, result +from tests.test_llm_claudexor import setup as gateway_fixture +from tests.test_model_wait import live_wait as wait_fixture +from tests.test_model_wait_controls import _loop_tools +from tests.test_subscription_main_wait import main_call as main_call_fixture + +setup = gateway_fixture +live_wait = wait_fixture +main_call = main_call_fixture +ROUTE_B = {**ROUTE, "credentialProfileId": "account-b", "accountFingerprint": "fingerprint-b"} +ROUTE_C = {**ROUTE, "credentialProfileId": "account-c", "accountFingerprint": "fingerprint-c"} + + +def _reply(route, message): + row = result(route=route) + row["message"] = message + return row + + +def _bounded_creates(monkeypatch, gateway, maximum): + create = gateway.create_model_operation + + def checked(*args, **kwargs): + if len(gateway.creates) >= maximum: + pytest.fail("unexpected extra model operation; the test must not retry into success") + return create(*args, **kwargs) + + monkeypatch.setattr(gateway, "create_model_operation", checked) + + +@pytest.mark.parametrize("role,direct", [("main", False), ("main", True), ("consciousness", True)]) +@pytest.mark.parametrize("primary_account", ["account-a", ""], ids=["pinned-A", "Auto"]) +def test_native_loop_fallback_then_symbolic_return_uses_real_role_account_and_fit( + main_call, monkeypatch, role, direct, primary_account): + """A -> B -> primary: equal model names do not make equal account/role bindings. + + The fallback MODEL is deliberately identical to the primary MODEL. A wrong + fallback:0 role on return would send pin B, and a global-Main return from + consciousness would send the decoy model/account. Both fail physical assertions. + """ + ctx, gateway, owner, events, _decide, _observations = main_call + monkeypatch.setenv("OUROBOROS_CONTEXT_MODE", "max") + monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") + monkeypatch.setenv("OUROBOROS_SAFETY_MODE", "off") + monkeypatch.setenv("OUROBOROS_MAX_ROUNDS", "4") + monkeypatch.setenv("MCP_ENABLED", "false") + monkeypatch.setenv("OUROBOROS_IMAGE_INPUT_MODE", "inline") + monkeypatch.setenv("OUROBOROS_MODEL", MODEL if role == "main" else "claudexor::codex=decoy-main") + monkeypatch.setenv("OUROBOROS_MODEL_CONSCIOUSNESS", MODEL) + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", MODEL) + monkeypatch.delenv("USE_LOCAL_MAIN", raising=False) + monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) + pins = {"main": "decoy-main-pin", "consciousness": "decoy-consciousness-pin", "fallback": ["account-b"]} + pins[role] = primary_account + monkeypatch.setenv(MODEL_ACCOUNTS_KEY, json.dumps(pins)) + monkeypatch.setattr(fallback_cooldown, "is_cooling_down", lambda *_args: False) + monkeypatch.setattr(loop_transport, "upstream_transport_reachable", + lambda *_args, **_kwargs: pytest.fail("healthy configured routes require no recovery probe")) + + # Metadata is the only route observation stub. _rebind_context_fit_plan, + # task_model_binding and every physical preparation/send stay real. + def route_evidence(task, **_kwargs): + actual_role = task["model_role"] + assert actual_role in (role, "fallback:0") + assert task["model"] == MODEL and not task["use_local_model"] + observed = task.get("model_route") or {} + explicit = task.get("credential_profile_id") + configured = "account-b" if actual_role == "fallback:0" else primary_account + profile = (explicit if explicit is not None else configured) or observed.get("credentialProfileId") or "account-c" + window = 240_000 if profile == "account-b" else 900_000 + return ({"model": MODEL, "provider": "claudexor"}, SimpleNamespace( + route_fp=f"capacity-{profile}", status="confirmed", stale=False, window_tokens=window, + source_id="codex", source="advertised", credential_profile_id=profile, + account_fingerprint=f"fingerprint-{profile.removeprefix('account-')}")) + + monkeypatch.setattr(context, "_context_fit_route", route_evidence) + ctx.context_fit_plan = replace(ctx.context_fit_plan, model_role=role, model_route=deepcopy(ROUTE)) + tools = _loop_tools(ctx, owner) + tools._ctx.task_metadata = {"model_role": role} + tools._ctx.is_direct_chat = direct + tools._ctx.model_wait_context = owner + owner.worker_slot_held = not direct + original_tools = deepcopy([row for row in ctx.messages if row.get("role") == "tool"]) + image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,eA=="}} + ctx.messages.append({"role": "user", "content": [{"type": "text", "text": "Keep the diagram"}, image]}) + return_call = {"id": "return-primary", "type": "function", "function": { + "name": "switch_model", "arguments": json.dumps({"primary": "return"})}} + answered_route = ROUTE if primary_account else ROUTE_C + gateway.results = [ + result(outcome="unknown", route=ROUTE), + _reply(ROUTE_B, {"role": "assistant", "content": "Returning to the original role.", + "tool_calls": [return_call]}), + _reply(answered_route, {"role": "assistant", "content": "Primary answered after return."}), + ] + gateway.dispatch = ["unknown", "response_received", "response_received"] + _bounded_creates(monkeypatch, gateway, 3) + + text, usage, trace = loop.run_llm_loop( + ctx.messages, tools, ctx.llm, ctx.drive_logs, lambda *_args, **_kwargs: None, + queue.Queue(), task_id="task-one", drive_root=ctx.drive_root, event_queue=events, + initial_effort="high", + ) + + assert text == "Primary answered after return." + assert len(gateway.uploads) == len(gateway.creates) == len(gateway.accepted_operations) == 3 + payloads = [payload for payload, _key in gateway.uploads] + assert all(payload["model"] == "exact-model" for payload in payloads) + assert payloads[1]["account"] == {"mode": "pin", "profileId": "account-b"} + for index in (0, 2): + account = payloads[index]["account"] + if primary_account: + assert account == {"mode": "pin", "profileId": "account-a"} + else: + assert account["mode"] == "auto" and "profileId" not in account + # A returned Auto account is observation, not permission to convert Auto to pin C. + assert tools._ctx.context_fit_plan.model_role == role + assert tools._ctx.context_fit_plan.model_route["credentialProfileId"] == answered_route["credentialProfileId"] + assert tools._ctx.primary_route == {"model": MODEL, "use_local": False, "role": role} + request_manifests = [json.loads((ctx.drive_root / "observability" / "calls" / "task-one" / + f"{attempt}_model_request.json").read_text(encoding="utf-8")) + for attempt in gateway.creates] + assert [manifest["model_role"] for manifest in request_manifests] == [role, "fallback:0", role] + rows = ledger(ctx.drive_root) + assert [row["state"] for row in rows] == [ + "reserved", "dispatched", "unresolved", "reserved", "dispatched", "settled", + "reserved", "dispatched", "settled"] + dispatched = [row for row in rows if row["state"] == "dispatched"] + assert len({row["attempt_id"] for row in dispatched}) == 3 + assert [row["physical_context"]["capacity_total_tokens"] for row in dispatched] == [900_000, 240_000, 900_000] + assert [row["physical_context"]["route_fp"] for row in dispatched] == [ + "capacity-account-a", "capacity-account-b", f"capacity-{answered_route['credentialProfileId']}"] + assert usage["transport_recovery"]["previous_attempt"]["physical_attempt_id"] == dispatched[0]["attempt_id"] + for payload in payloads: + sent = payload["messages"] + for completed in original_tools: + matches = [row for row in sent if row.get("role") == "tool" and + row.get("tool_call_id") == completed["tool_call_id"]] + assert len(matches) == 1 and matches[0]["content"] == completed["content"] + assert any(image in row.get("content", []) for row in sent if isinstance(row.get("content"), list)) + returned_tool = [row for row in payloads[2]["messages"] if row.get("role") == "tool" and + row.get("tool_call_id") == "return-primary"] + assert len(returned_tool) == 1 and "OK: switching to primary route" in returned_tool[0]["content"] + assert [call["tool"] for call in trace["tool_calls"]] == ["switch_model"] + + +@pytest.mark.parametrize("direct", [False, True], ids=["queued", "direct"]) +def test_accepted_get_404_recovers_same_native_operation_for_both_turns(live_wait, monkeypatch, direct): + root, gateway, client, owner, _events, _decide = live_wait + owner.tool_context = SimpleNamespace(task_id="task-one", is_direct_chat=direct, task_metadata={}) + owner.worker_slot_held = not direct + gateway.pending = True + original_get = gateway.get_model_operation + observed_reads = [] + + def read(operation_id, **kwargs): + observed_reads.append(operation_id) + if len(observed_reads) <= 2: + raise ClaudexorUnavailable("http_404", "accepted operation temporarily not readable", status_code=404) + gateway.pending = False + return original_get(operation_id, **kwargs) + + monkeypatch.setattr(gateway, "get_model_operation", read) + monkeypatch.setattr(llm_claudexor, "read_owned_gateway", lambda: gateway) + now = [0.0] + monkeypatch.setattr(llm_claudexor, "time", SimpleNamespace( + monotonic=lambda: now[0], sleep=lambda seconds: now.__setitem__(0, now[0] + seconds))) + _bounded_creates(monkeypatch, gateway, 1) + message, usage = client.chat([{"role": "user", "content": "recover this response"}], MODEL, + model_role="main", timeout=1) + assert message == result()["message"] + assert observed_reads == ["op-0"] * 3 + assert len(gateway.uploads) == len(gateway.creates) == len(gateway.accepted_operations) == 1 + assert [row["state"] for row in ledger(root)] == ["reserved", "dispatched", "settled"] + assert len(usage["ledger_attempt_ids"]) == 1 and not gateway.cancels + events = [json.loads(line) for line in (root / "logs/events.jsonl").read_text(encoding="utf-8").splitlines()] + assert any(event.get("detail") == "same_model_operation_rejoined" for event in events) + + +def test_lost_create_reply_without_operation_id_cannot_authorize_new_generation(main_call, monkeypatch): + """A generic create 4xx is not model_request_invalid: accepted custody may exist. + + Same-key rereads are allowed. After the second lost reply the scripted owner + stops observation; neither another physical id nor a fallback send is allowed. + An immediate honest recoverable-unknown terminal is also no-new-generation. + """ + ctx, gateway, owner, events, _decide, _observations = main_call + monkeypatch.setenv("OUROBOROS_CONTEXT_MODE", "max") + monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") + monkeypatch.setenv("OUROBOROS_SAFETY_MODE", "off") + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", MODEL) + monkeypatch.setenv(MODEL_ACCOUNTS_KEY, json.dumps({"main": "account-a", "fallback": ["account-b"]})) + monkeypatch.setattr(fallback_cooldown, "is_cooling_down", lambda *_args: False) + monkeypatch.setattr(loop_transport, "upstream_transport_reachable", + lambda *_args, **_kwargs: pytest.fail("metadata cannot authorize a new generation")) + tools = _loop_tools(ctx, owner) + tools._ctx.is_direct_chat = True + owner.worker_slot_held = False + create = gateway.create_model_operation + original_key = [] + + def lost_reply(ref, *, idempotency_key, **kwargs): + if original_key and idempotency_key != original_key[0]: + pytest.fail("lost create reply caused a NEW physical id/fallback generation") + original_key[:] = [idempotency_key] + create(ref, idempotency_key=idempotency_key, **kwargs) # engine accepted; host never sees its id + raise ClaudexorUnavailable("http_404", "create reply lost after admission", status_code=404) + + monkeypatch.setattr(gateway, "create_model_operation", lost_reply) + monkeypatch.setattr(llm_claudexor, "read_owned_gateway", lambda: gateway) + owner.owner_control = lambda: "cancelled" if len(gateway.creates) >= 2 else None + now = [0.0] + monkeypatch.setattr(llm_claudexor, "time", SimpleNamespace( + monotonic=lambda: now[0], sleep=lambda seconds: now.__setitem__(0, now[0] + seconds))) + with pytest.raises(ModelWaitInterrupted, match="cancelled"): + loop.run_llm_loop( + ctx.messages, tools, ctx.llm, ctx.drive_logs, lambda *_args, **_kwargs: None, + queue.Queue(), task_id="task-one", drive_root=ctx.drive_root, event_queue=events, initial_effort="high", + ) + assert len(gateway.accepted_operations) == len(gateway.uploads) == 1 + assert set(gateway.creates) == {original_key[0]} + assert gateway.uploads[0][0]["account"] == {"mode": "pin", "profileId": "account-a"} + rows = ledger(ctx.drive_root) + assert [row["state"] for row in rows] == ["reserved", "dispatched", "unresolved"] + assert len({row["attempt_id"] for row in rows}) == 1 + assert not gateway.acks + + +def test_duplicate_model_slots_keep_their_distinct_native_accounts(main_call, monkeypatch): + ctx, gateway, owner, events, _decide, _observations = main_call + monkeypatch.setenv("OUROBOROS_CONTEXT_MODE", "max") + monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", f"{MODEL},{MODEL}") + monkeypatch.setenv(MODEL_ACCOUNTS_KEY, json.dumps({"main": "account-a", "fallback": ["account-b", "account-c"]})) + monkeypatch.setattr(fallback_cooldown, "is_cooling_down", lambda *_args: False) + tools = _loop_tools(ctx, owner) + gateway.results = [result(outcome="unknown", route=ROUTE), result(outcome="unknown", route=ROUTE_B), + _reply(ROUTE_C, {"role": "assistant", "content": "Third account answered."})] + gateway.dispatch = ["unknown", "unknown", "response_received"] + _bounded_creates(monkeypatch, gateway, 3) + text, _usage, _trace = loop.run_llm_loop( + ctx.messages, tools, ctx.llm, ctx.drive_logs, lambda *_args, **_kwargs: None, + queue.Queue(), task_id="task-one", drive_root=ctx.drive_root, event_queue=events) + assert text == "Third account answered." + assert [payload["account"] for payload, _ in gateway.uploads] == [ + {"mode": "pin", "profileId": pin} for pin in ("account-a", "account-b", "account-c")] + assert tools._ctx.context_fit_plan.model_role == "fallback:1" + assert [row["state"] for row in ledger(ctx.drive_root)].count("unresolved") == 2 + + +@pytest.mark.parametrize("status,reason", [(401, "auth"), (402, "quota")]) +def test_primary_wait_honors_a_direct_api_access_refusal(main_call, monkeypatch, status, reason): + """A declared wait must not be undone by a paid fallback merely because access uses an API key.""" + from ouroboros import usage_accounting as ua + from ouroboros.llm_attempt import _attempt_request, _candidate_before_dispatch + + ctx, gateway, owner, events, _decide, _observations = main_call + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", MODEL) + monkeypatch.setenv("OPENAI_API_KEY", "fixture-key") + ctx.active_model = "openai::primary-test" + ctx.context_fit_plan = replace(ctx.context_fit_plan, model=ctx.active_model, provider="openai", model_route={}) + ctx.tools._ctx.context_fit_plan = ctx.context_fit_plan + ctx.tools._ctx.primary_route = {"model": ctx.active_model, "use_local": False, "role": "main"} + ctx.tools._ctx.model_wait_context = owner + _switch_model(ctx.tools._ctx, primary="wait") + sent = [] + + def send(target, messages, schemas, *args, **kwargs): + assert target["provider"] == "openai" + sent.append(target["provider"]) + body = {"messages": deepcopy(messages), "tools": schemas or [], "model": "primary-test"} + request = _attempt_request(target, body) + + def refused(): + raise httpx.HTTPStatusError("Access refused", request=httpx.Request("POST", "https://fixture.invalid"), + response=httpx.Response(status)) + + return ua.execute_physical_attempt(request, refused, before_dispatch=_candidate_before_dispatch(body, request)) + + monkeypatch.setattr(ctx.llm, "_chat_remote", send) + monkeypatch.setattr(ctx.llm, "claudexor_model_sources", lambda: pytest.fail("API wait has no subscription observer")) + owner.owner_control = lambda: "cancelled" if any(row.get("state") == "waiting" for row in owner.waits.values()) else None + with pytest.raises(ModelWaitInterrupted, match="cancelled"): + loop._call_round_model(ctx) + waits = [row for row in list(events.queue) if row.get("type") == "task_model_wait"] + assert waits[0]["state"] == "waiting" and waits[0]["reason"] == reason + assert waits[0]["availability_observation"] == "unavailable" and waits[0]["auto_continue"] is False + assert waits[0]["credential_harness"] == "" and waits[-1]["resolution"] == "cancelled" + assert sent == ["openai"] and gateway.creates == [] diff --git a/tests/test_quota_rotation_fallback.py b/tests/test_quota_rotation_fallback.py index ac4e468df..faf51cbcb 100644 --- a/tests/test_quota_rotation_fallback.py +++ b/tests/test_quota_rotation_fallback.py @@ -24,7 +24,7 @@ from ouroboros.send_clock import CLOCK_NOTE_PREFIX from ouroboros import fallback_cooldown, loop, loop_llm_call, model_wait from ouroboros import llm_claudexor as transport from ouroboros import usage_accounting as ua -from ouroboros.loop_llm_call import RETRY_WALL_EXHAUSTED_KEY, call_llm_with_retry, provider_no_call_source +from ouroboros.loop_llm_call import call_llm_with_retry, provider_no_call_source from ouroboros.loop_model_call import RESOURCE_REFUSAL_KEY from ouroboros.loop_transport import provider_recovery_hint from ouroboros.model_slots import MODEL_ACCOUNTS_KEY @@ -236,18 +236,60 @@ def test_a_fallback_answer_leaves_no_owner_question_and_no_refusal(main_call, on assert not _waits(events) and RESOURCE_REFUSAL_KEY not in usage and len(gateway.accepted_operations) == 1 -def test_an_unknown_fallback_outcome_asks_no_owner_and_sends_nothing_more(main_call, one_fallback, monkeypatch): +def test_an_eligible_unknown_fallback_outcome_still_opens_the_primary_refusal_wait(main_call, one_fallback, monkeypatch): + """Owner decision 1A: a candidate's eligible unknown outcome does not fence the round. With no + configured route left, the primary's retained refusal opens its own visible owner wait (never a + sleep to its reset), and the send after that wait is a NEW generation carrying one facts-only + input naming the unknown candidate attempt; the old attempt keeps its own identity.""" ctx, gateway, owner, _events, _decide, _observations = main_call gateway.results, gateway.dispatch = [_pool_quota()], ["not_started"] assert loop._call_round_model(ctx)[0] is None and ctx.tools._ctx._deferred_resource_refusal is not None original = loop._call_round_model + sent, asked = [], [] def candidate(round_call): - assert round_call.defer_resource_wait is True # the primary's owner question would follow - round_call.accumulated_usage["_last_llm_error_kind"] = "provider_outcome_unknown" + sent.append((round_call.active_model, round_call.defer_resource_wait, [dict(row) for row in round_call.messages])) + if round_call.defer_resource_wait is False: # the retained refusal's own send after the owner wait + return {"role": "assistant", "content": "after the wait"}, 0.0, "max" + round_call.accumulated_usage.update(_last_llm_error_kind="provider_outcome_unknown", _pending_transport_outcome={ + "physical_attempt_id": "fallback-unknown", "operation_id": "", "outcome": "unknown"}) return None, 0.0, "max" monkeypatch.setattr(loop, "_call_round_model", candidate) + monkeypatch.setattr(model_wait.ResourceDeferral, "ask_owner", + lambda deferral, _waiter: asked.append(deferral.fact["reason"])) + message, *_rest = loop._run_cross_model_fallback_chain( + llm=ctx.llm, ctx=ctx.tools._ctx, tools=ctx.tools, messages=ctx.messages, active_model=ctx.active_model, + active_use_local=False, tool_schemas=[], active_effort="medium", max_retries=3, drive_logs=ctx.drive_logs, + task_id=ctx.task_id, round_idx=1, event_queue=_events, accumulated_usage=ctx.accumulated_usage, + task_type="task", emit_progress=lambda *_a, **_kw: None, context_fit_plan=ctx.context_fit_plan, + active_context_mode="max") + monkeypatch.setattr(loop, "_call_round_model", original) + assert message["content"] == "after the wait" and asked == ["quota"] + assert [(model, defer) for model, defer, _rows in sent] == [(FALLBACK, True), (MODEL, False)] + assert not any("[Transport recovery]" in str(row.get("content")) for row in sent[0][2]) + notices = [row for row in sent[-1][2] if "[Transport recovery]" in str(row.get("content"))] + assert len(notices) == 1 and "fallback-unknown" in notices[0]["content"] + assert RESOURCE_REFUSAL_KEY not in ctx.accumulated_usage and len(gateway.accepted_operations) == 1 + + +def test_a_readable_unknown_fallback_operation_fences_the_round(main_call, one_fallback, monkeypatch): + """Disallowed counterpart: an unknown whose accepted operation could not be READ may still + finish, so nothing further is sent — not the owner question's send, not a forced final.""" + ctx, gateway, owner, _events, _decide, _observations = main_call + gateway.results, gateway.dispatch = [_pool_quota()], ["not_started"] + assert loop._call_round_model(ctx)[0] is None + original = loop._call_round_model + + def candidate(round_call): + assert round_call.defer_resource_wait is True + round_call.accumulated_usage.update(_last_llm_error_kind="provider_outcome_unknown", _pending_transport_outcome={ + "physical_attempt_id": "still-readable", "outcome": "unknown", "same_operation_recoverable": True}) + return None, 0.0, "max" + + monkeypatch.setattr(loop, "_call_round_model", candidate) + monkeypatch.setattr(model_wait.ResourceDeferral, "ask_owner", + lambda *_a: pytest.fail("no owner wait may send over a readable operation")) message, *_rest = loop._run_cross_model_fallback_chain( llm=ctx.llm, ctx=ctx.tools._ctx, tools=ctx.tools, messages=ctx.messages, active_model=ctx.active_model, active_use_local=False, tool_schemas=[], active_effort="medium", max_retries=3, drive_logs=ctx.drive_logs, @@ -256,11 +298,7 @@ def test_an_unknown_fallback_outcome_asks_no_owner_and_sends_nothing_more(main_c active_context_mode="max") monkeypatch.setattr(loop, "_call_round_model", original) assert message is None and not _waits(_events) and len(gateway.accepted_operations) == 1 - refusal = ctx.accumulated_usage[RESOURCE_REFUSAL_KEY] - assert (refusal["owner_wait"], refusal["fallbacks_tried"]) == ("not_asked", [FALLBACK]) - # The unknown-outcome fence still outranks: nothing is resent, not even a forced final. assert provider_no_call_source(ctx.accumulated_usage, False)[0] == "provider_outcome_unknown_no_resend" - assert provider_no_call_source({RESOURCE_REFUSAL_KEY: refusal}, True) == ("resource_refusal_no_resend", True) def test_a_candidate_defers_only_while_a_later_route_or_the_owner_question_follows(main_call, monkeypatch): @@ -604,27 +642,49 @@ def test_main_retry_never_sleeps_to_a_reset_nor_resends_a_spent_window(tmp_path, usage = {} message, _cost = call_llm_with_retry(Client(), [{"role": "user", "content": "go"}], MODEL, None, "medium", 3, tmp_path / "logs", "task-one", 1, None, usage, deadline_ts=None) - assert message is None and len(calls) == 1 and usage[RETRY_WALL_EXHAUSTED_KEY] is True + assert message is None and len(calls) == 1 + assert usage["_last_llm_retry_same_request"] is False and usage["_last_llm_resource_refusal"] == "quota" + + +def test_dated_pool_classifier_retains_forecast_without_inventing_quota(): + error = transport.ClaudexorModelNotDispatched({"code": "credential_pool_exhausted", + "message": "No usable route", "context": {"poolCause": "unavailable", "resetsAt": RESET}}, route=ROUTE) + fact = loop_llm_call.classify_llm_exception(error) + assert fact.kind == "provider_error" and fact.provider_code == "credential_pool_exhausted" + assert fact.reset_at == RESET and fact.retry_same_request is False and fact.retry_after_sec is None + quota = loop_llm_call.classify_llm_exception(transport.ClaudexorModelError(_vendor_quota()["problem"], route=ROUTE)) + assert quota.kind == "subscription_window_exhausted" # a proved quota window retains its own meaning @pytest.mark.parametrize("presence", [False, True]) -def test_an_unproven_dated_pool_keeps_its_reset_timer_only_where_waiting_is_allowed(tmp_path, monkeypatch, presence): - slept = [] - monkeypatch.setattr(loop_llm_call, "_sleep_within_deadline", lambda seconds, *_a, **_kw: slept.append(seconds) and False) +def test_a_dated_unavailable_pool_never_sleeps_to_its_reset_anywhere(tmp_path, monkeypatch, presence): + """#1376 overlap / #1409 case 1: a dated pool refusal is a typed resource refusal of its own + (``unavailable``, never quota) that is neither resent nor slept on, where waiting is allowed and + where it is not; the round's configured routes and the visible wait own what follows.""" + monkeypatch.setattr(loop_llm_call, "_sleep_within_deadline", lambda *_a, **_kw: pytest.fail("slept to a reset")) dated = transport.ClaudexorModelNotDispatched({"code": "credential_pool_exhausted", "message": "no account", "context": {"poolCause": "unavailable", "resetsAt": RESET}}) dated.physical_attempt_capture = SimpleNamespace(state="released", attempt_id="a-1", provider="claudexor", route_is_loopback=False) + calls = [] class Client: def chat(self, **_kwargs): + calls.append(1) raise dated + usage = {} with model_wait.task_model_wait_scope(task={"id": "task-one", "_presence_turn": presence}, drive_root=tmp_path, event_queue=None, worker_slot_held=False): - call_llm_with_retry(Client(), [{"role": "user", "content": "go"}], MODEL, None, "medium", 3, - tmp_path / "logs", "task-one", 1, None, {}, deadline_ts=None) - assert (slept == []) is presence and all(seconds > 3600 for seconds in slept) + message, _cost = call_llm_with_retry(Client(), [{"role": "user", "content": "go"}], MODEL, None, "medium", 3, + tmp_path / "logs", "task-one", 1, None, usage, deadline_ts=None) + assert message is None and calls == [1] + assert usage["_last_llm_resource_refusal"] == "unavailable" and usage["_last_llm_reset_at"] == RESET + assert "_last_llm_retry_after_sec" not in usage + assert model_wait.model_wait_reason(dated) == "unavailable" + undated = transport.ClaudexorModelNotDispatched({"code": "credential_pool_exhausted", "message": "no account", + "context": {"poolCause": "unavailable"}}) + assert model_wait.model_wait_reason(undated) == "" # an undated pool proves no resource fact @pytest.mark.parametrize("code,context,outage", [ diff --git a/tests/test_transport_death_retry.py b/tests/test_transport_death_retry.py index 930000a15..730fee1a6 100644 --- a/tests/test_transport_death_retry.py +++ b/tests/test_transport_death_retry.py @@ -2,8 +2,10 @@ A DISPATCHED request whose socket died with a typed transport death (httpx ReadError / WriteError / RemoteProtocolError, or the requests ProtocolError / -RemoteDisconnected shape) is `provider_outcome_unknown`; the PRIMARY main-loop -round dispatch alone may repeat it at most twice per round, each repeat a NEW +RemoteDisconnected shape) is `provider_outcome_unknown`; an inline Presence +turn's PRIMARY round dispatch alone may repeat it at most twice per round (every +other turn's new generation after an unknown outcome is the round recovery, +``tests/test_unknown_fallback_first.py``), each repeat a NEW physical attempt with its own ledger lifecycle (the earlier rows stay unresolved at their upper bound). Every other surface keeps the no-resend doctrine, the classifier is unchanged, and every durable `llm_api_error` row tells the truth @@ -325,7 +327,7 @@ def test_admission_refusal_after_a_granted_repeat_keeps_the_unknown_fence(tmp_pa llm = _ScriptedLLM(_death, _death, _death) notes = [] kwargs = _loop_kwargs(tmp_path, llm, notes) - kwargs["tools"]._ctx.is_direct_chat = True # This caller retains the bounded repeat rail. + _presence_turn(kwargs) # Inline Presence alone retains the bounded repeat rail. _result, usage, trace = run_llm_loop(**kwargs) assert llm.calls == 1 # the refused repeat and the forced-final rail both dispatched nothing @@ -503,6 +505,14 @@ def _no_chain(**_kwargs): raise AssertionError("unknown physical work must stop the paid fallback chain") +def _presence_turn(kwargs): + """Inline Presence, the one caller that keeps the bounded paid-repeat rail.""" + kwargs["task_type"] = "presence" + kwargs["tools"]._ctx.is_direct_chat = True + kwargs["tools"]._ctx.current_task_type = "presence" + return kwargs + + def test_primary_round_dispatch_exhaustion_takes_the_unknown_no_resend_terminal(tmp_path, monkeypatch, no_sleep): """End to end through the real round dispatcher: three deaths, then the forced-final rail ships the salvage WITHOUT a provider call and the terminal @@ -514,7 +524,7 @@ def test_primary_round_dispatch_exhaustion_takes_the_unknown_no_resend_terminal( llm = _ScriptedLLM(_death, _death, _death, _death) notes = [] kwargs = _loop_kwargs(tmp_path, llm, notes) - kwargs["tools"]._ctx.is_direct_chat = True # This caller retains the bounded repeat rail. + _presence_turn(kwargs) # Inline Presence alone retains the bounded repeat rail. result, usage, trace = run_llm_loop(**kwargs) assert llm.calls == 3 # zero further dials of any kind: no forced-final resend @@ -596,7 +606,7 @@ def test_unexpected_loop_error_carries_accumulated_evidence_to_owner_projection( assert trace["tool_calls"][0]["trace_ref"]["call_id"] -@pytest.mark.parametrize("turn_flag", ["is_direct_chat"]) +@pytest.mark.parametrize("turn_flag", ["presence"]) def test_counter_survives_the_wait_episodes_free_redial_of_the_same_round(tmp_path, monkeypatch, no_sleep, turn_flag): """death → released ConnectError → wait episode → free redial → death → death: the round stays bounded by two paid repeats in total (sol s1). The @@ -612,7 +622,7 @@ def test_counter_survives_the_wait_episodes_free_redial_of_the_same_round(tmp_pa notes = [] kwargs = _loop_kwargs(tmp_path, llm, notes) if turn_flag: - setattr(kwargs["tools"]._ctx, turn_flag, True) + _presence_turn(kwargs) _result, usage, trace = run_llm_loop(**kwargs) assert llm.calls == 4 # death, connect (free), death, death — the fifth script step never runs @@ -653,13 +663,13 @@ def test_forced_final_call_never_repeats_a_transport_death(tmp_path, no_sleep): @pytest.mark.parametrize("attempt_cap,expected_calls", [(None, 3), (2, 1)]) def test_round_dispatcher_opts_in_only_the_primary(tmp_path, no_sleep, attempt_cap, expected_calls): """attempt_cap is set only for fallback-chain candidates (the primary passes - None): candidates get zero paid repeats, the primary its bounded two.""" + None): candidates get zero paid repeats, an inline Presence primary its bounded two.""" llm = _ScriptedLLM(_death, _death) registry = ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path) ctx = SimpleNamespace( llm=llm, messages=MESSAGES, tools=registry, active_model="test-model", tool_schemas=None, active_effort="low", max_retries=3, drive_logs=tmp_path, task_id="t-cand", round_idx=1, - event_queue=None, accumulated_usage={}, task_type="task", active_use_local=False, + event_queue=None, accumulated_usage={}, task_type="presence", active_use_local=False, ) msg, _cost = loop_mod._dispatch_round_model(ctx, None, attempt_cap=attempt_cap) @@ -761,7 +771,7 @@ def test_repeat_failing_with_another_class_ends_the_round_on_the_unknown_termina llm = _ScriptedLLM(_death, lambda: _status_failure(status), _death, _death) notes = [] kwargs = _loop_kwargs(tmp_path, llm, notes) - kwargs["tools"]._ctx.is_direct_chat = True # This caller retains the bounded repeat rail. + _presence_turn(kwargs) # Inline Presence alone retains the bounded repeat rail. result, usage, trace = run_llm_loop(**kwargs) assert llm.calls == 2 # no burst, no forced-final dial, no chain candidate @@ -791,7 +801,7 @@ def test_repeat_returning_an_empty_response_ends_the_round_and_keeps_the_record( llm = _ScriptedLLM(_death, EMPTY_RESPONSE, _death, _death, _death) notes = [] kwargs = _loop_kwargs(tmp_path, llm, notes) - kwargs["tools"]._ctx.is_direct_chat = True # This caller retains the bounded repeat rail. + _presence_turn(kwargs) # Inline Presence alone retains the bounded repeat rail. _result, usage, trace = run_llm_loop(**kwargs) assert llm.calls == 2 # the empty repeat is not retried; the third death never happens @@ -910,7 +920,7 @@ def test_repeat_failing_as_context_overflow_takes_the_unknown_terminal(tmp_path, llm = _ScriptedLLM(_death, _overflow_failure, _death, _death) notes = [] kwargs = _loop_kwargs(tmp_path, llm, notes) - kwargs["tools"]._ctx.is_direct_chat = True # This caller retains the bounded repeat rail. + _presence_turn(kwargs) # Inline Presence alone retains the bounded repeat rail. result, usage, trace = run_llm_loop(**kwargs) assert llm.calls == 2 @@ -1008,8 +1018,7 @@ def test_proxy_tunnel_failure_keeps_the_base_unknown_terminal(data_root, tmp_pat return exc loop_llm = _ScriptedLLM(scripted, scripted) - kwargs = _loop_kwargs(tmp_path, loop_llm, []) - kwargs["tools"]._ctx.is_direct_chat = True + kwargs = _presence_turn(_loop_kwargs(tmp_path, loop_llm, [])) _result, loop_usage, trace = run_llm_loop(**kwargs) assert loop_llm.calls == 1 assert _events(tmp_path, "network_wait") == [] diff --git a/tests/test_transport_repeat_owner_stop.py b/tests/test_transport_repeat_owner_stop.py index 6c7fed06c..d8ecd2c56 100644 --- a/tests/test_transport_repeat_owner_stop.py +++ b/tests/test_transport_repeat_owner_stop.py @@ -39,6 +39,8 @@ def test_stop_arriving_in_real_backoff_keeps_only_actual_physical_attempts( ctx = kwargs["tools"]._ctx ctx.task_id, ctx.task_attempt, ctx.budget_drive_root = "t-death", 1, canonical ctx.is_direct_chat = not owner_grace + if not owner_grace: # inline Presence, the one direct caller that keeps the paid-repeat rail + kwargs["task_type"] = ctx.current_task_type = "presence" ctx.task_metadata = {"root_task_id": "t-death"} first_empty_check = threading.Event() real_wait = loop_transport.interruptible_wait_sleep diff --git a/tests/test_transport_unknown_continuation.py b/tests/test_transport_unknown_continuation.py index 2e24a7463..fcbaf2f23 100644 --- a/tests/test_transport_unknown_continuation.py +++ b/tests/test_transport_unknown_continuation.py @@ -167,12 +167,24 @@ def test_grant_without_a_new_attempt_is_not_a_phantom_repeat(tmp_path): ("continued", "continuation_outcome_unknown", 1)] -@pytest.mark.parametrize("flag", ["is_direct_chat"]) -def test_unknown_policy_does_not_expand_other_execution_classes(tmp_path, flag): - ctx = SimpleNamespace(task_id="t", **{flag: True}) - assert transport.reconcile_transport_wait(None, ctx, msg_present=False, +def test_unknown_policy_admits_direct_turns_but_not_presence_or_a_readable_operation(tmp_path): + """Owner decision 1A: a direct turn's eligible unknown outcome enters the same wait and + continuation as a queued one, inside its interactive bound. Inline Presence keeps its own + no-resend terminal, and an accepted operation whose read failed is only rejoined.""" + notes = [] + direct = SimpleNamespace(task_id="t", is_direct_chat=True, _accumulated_usage={}) + episode = transport.reconcile_transport_wait(None, direct, msg_present=False, error_kind="provider_outcome_unknown", drive_logs=tmp_path, task_id="t", model="m", - emit_progress=lambda *a, **kw: pytest.fail("unexpected automatic continuation")) is None + emit_progress=lambda text, **_kw: notes.append(text)) + assert episode is not None and episode.interactive and episode.wait_cause == "provider_outcome_unknown" + assert episode.wait_bound_sec is not None and notes and "Stop cancels" not in notes[0] + readable = {"_pending_transport_outcome": {"physical_attempt_id": "a", "same_operation_recoverable": True}} + for ctx in (SimpleNamespace(task_id="t", is_direct_chat=True, current_task_type="presence", _accumulated_usage={}), + SimpleNamespace(task_id="t", _accumulated_usage=readable), + SimpleNamespace(task_id="t", task_metadata={"presence": {"binding": "b"}}, _accumulated_usage={})): + assert transport.reconcile_transport_wait(None, ctx, msg_present=False, + error_kind="provider_outcome_unknown", drive_logs=tmp_path, task_id="t", model="m", + emit_progress=lambda *a, **kw: pytest.fail("unexpected automatic continuation")) is None def test_deadline_closes_unknown_wait_without_a_probe_or_send(tmp_path, monkeypatch): diff --git a/tests/test_transport_wait_repeat_interaction.py b/tests/test_transport_wait_repeat_interaction.py index 8d78df125..2993b1387 100644 --- a/tests/test_transport_wait_repeat_interaction.py +++ b/tests/test_transport_wait_repeat_interaction.py @@ -5,10 +5,10 @@ and the paid repeat rail (a dispatched typed transport death, ``provider_outcome never overlap, and while the round holds a repeat record that record outranks the wait terminal: -- a dispatched death on an interactive turn takes the repeat rail and never opens a - wait episode (unknown is not the episode's kind); -- inside an episode the chain's local-only pass exists for ``transport_unavailable`` - alone, and a round record blocks even that pass; +- a dispatched death on an inline Presence turn (the one caller that keeps the + repeat rail) takes the repeat rail and never opens a wait episode; +- the configured-route walk runs once BEFORE a wait episode opens and never inside + one, and a round record blocks it; - an episode exhausted on a round that still holds a repeat record ends on the record's source, worded as both the wait and the unresolved attempt — and that wording names the class the repeat was RELEASED with (it rides the round record, @@ -16,8 +16,8 @@ wait terminal: closed the window or the later free redial's own class; - the terminal precedence is decided once, at the provider-death rail: a round record outranks the latched wait cause, which outranks the overflow salvage — - so the ``context_overflow`` a failed local-only pass leaves in the mutable kind - never turns a waited-out outage into the overflow terminal. + so the ``context_overflow`` a failed route walk leaves behind never turns a + waited-out outage into the overflow terminal. """ from __future__ import annotations @@ -41,11 +41,21 @@ from tests.test_transport_death_retry import ( _loop_kwargs, _no_chain, _overflow_failure, + _presence_turn, _released_connect, _status_failure, ) +def _failing_walk(walks): + """A configured-route walk that finds no answer and records the kind it was entered with.""" + def chain(**kwargs): + walks.append(kwargs["accumulated_usage"].get("_last_llm_error_kind")) + return (None, kwargs["active_model"], kwargs["active_use_local"], + kwargs["context_fit_plan"], kwargs["active_context_mode"]) + return chain + + @pytest.fixture def no_sleep(monkeypatch): """The repeat rail's backoff sleeps, recorded instead of slept (as in the @@ -55,12 +65,12 @@ def no_sleep(monkeypatch): return sleeps -@pytest.mark.parametrize("turn_flag", ["is_direct_chat"]) +@pytest.mark.parametrize("turn_flag", ["presence"]) @pytest.mark.parametrize("deaths", [2, 3]) def test_interactive_turn_death_takes_the_repeat_rail_and_never_enters_a_wait_episode( tmp_path, monkeypatch, no_sleep, turn_flag, deaths, ): - """A direct-chat turn whose DISPATCHED request died with a typed + """An inline Presence turn whose DISPATCHED request died with a typed transport death is on the paid repeat rail (its round dispatch is primary), never in the free wait episode: `provider_outcome_unknown` is not the episode's `transport_unavailable`, so no `network_wait` event exists, the @@ -76,11 +86,10 @@ def test_interactive_turn_death_takes_the_repeat_rail_and_never_enters_a_wait_ep monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) llm = _ScriptedLLM(*([_death] * deaths)) notes = [] - kwargs = _loop_kwargs(tmp_path, llm, notes) - setattr(kwargs["tools"]._ctx, turn_flag, True) + kwargs = _presence_turn(_loop_kwargs(tmp_path, llm, notes)) result, usage, trace = run_llm_loop(**kwargs) - assert llm.calls == 3 # the primary send plus two paid repeats, whatever the turn kind + assert llm.calls == 3 # the primary send plus two paid repeats assert no_sleep == [4.0, 8.0] # the repeat rail's backoffs, never the episode's wait assert _events(tmp_path, "network_wait") == [] assert not any("provider connection" in note.lower() for note in notes) @@ -97,7 +106,7 @@ def test_interactive_turn_death_takes_the_repeat_rail_and_never_enters_a_wait_ep @pytest.mark.parametrize("turn,with_record", [ - ("managed", False), ("is_direct_chat", True), ("is_direct_chat", False), + ("managed", False), ("presence", True), ("presence", False), ]) def test_wait_episode_exhausted_on_a_round_holding_a_repeat_record_takes_the_unknown_source( tmp_path, monkeypatch, no_sleep, turn, with_record, @@ -114,7 +123,8 @@ def test_wait_episode_exhausted_on_a_round_holding_a_repeat_record_takes_the_unk clock = _FakeClock(monkeypatch) monkeypatch.setattr(loop_transport, "get_task_idle_timeout_sec", lambda: 60) - monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", _no_chain) + walks = [] + monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", _failing_walk(walks)) monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "other/model") monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) @@ -126,8 +136,10 @@ def test_wait_episode_exhausted_on_a_round_holding_a_repeat_record_takes_the_unk deadline = datetime.now(timezone.utc) + timedelta(seconds=get_finalization_grace_sec() + 8) kwargs["tools"]._ctx.task_metadata = {"deadline_at": deadline.isoformat()} else: - setattr(kwargs["tools"]._ctx, turn, True) + _presence_turn(kwargs) result, usage, trace = run_llm_loop(**kwargs) + # One walk before the wait; a round record (an unresolved paid repeat) blocks even that one. + assert walks == ([] if with_record else ["transport_unavailable"]) assert (3 if with_record else 2) <= llm.calls <= len(script) # the episode ended; the turn never recovered assert no_sleep == ([4.0] if with_record else []) # one repeat backoff; released redials never re-arm it @@ -175,8 +187,7 @@ def test_deadline_refused_redial_still_names_the_class_the_repeat_was_released_w monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) llm = _ScriptedLLM(_death, _released_connect, _released_connect) notes = [] - kwargs = _loop_kwargs(tmp_path, llm, notes) - kwargs["tools"]._ctx.is_direct_chat = True # This caller retains the bounded paid-repeat rail. + kwargs = _presence_turn(_loop_kwargs(tmp_path, llm, notes)) # Inline Presence alone keeps the paid-repeat rail. # Room for the grant (backoff 4 s + the admission reserve) and for one wait. metadata = {"deadline_at": ( datetime.now(timezone.utc) + timedelta(seconds=get_finalization_grace_sec() + 8) @@ -220,8 +231,7 @@ def test_generic_terminal_names_the_class_the_repeat_was_released_with(tmp_path, monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) llm = _ScriptedLLM(_death, _released_connect, lambda: _status_failure(400)) notes = [] - kwargs = _loop_kwargs(tmp_path, llm, notes) - kwargs["tools"]._ctx.is_direct_chat = True + kwargs = _presence_turn(_loop_kwargs(tmp_path, llm, notes)) result, usage, trace = run_llm_loop(**kwargs) assert llm.calls == 3 # the primary send, its released repeat, one free redial @@ -240,31 +250,32 @@ def test_generic_terminal_names_the_class_the_repeat_was_released_with(tmp_path, def test_fallback_chain_fence_holds_inside_a_wait_episode_too(monkeypatch): - """`fallback_chain_allowed` on the combined tree: inside a wait episode the - one local-only chain pass exists for `transport_unavailable` alone (never - for the unknown kind), and a round record — an unresolved attempt of this - round — blocks even that pass whatever the kind says, because no candidate - may dial over a request that may still be live.""" - monkeypatch.setenv("USE_LOCAL_FALLBACK", "1") - routable = SimpleNamespace(exact_model_route=False) + """`fallback_chain_allowed` on the combined tree: the configured-route walk runs + BEFORE a wait episode opens (remote routes included) and never inside one; a round + record — an unresolved paid repeat of this round — blocks it whatever the kind says; + an unknown outcome walks only when it is eligible for a new generation.""" + monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) + routable = SimpleNamespace(exact_model_route=False, task_id="t") record = {"round_id": "r", "count": 1, "backoff_sec": 4.0} episode = loop_transport.TransportWaitEpisode(started_monotonic=1.0, interactive=True, wait_bound_sec=900.0) - assert loop_transport.fallback_chain_allowed(routable, "provider_outcome_unknown", episode) is False - assert loop_transport.fallback_chain_allowed( - routable, "transport_unavailable", episode, {TRANSPORT_DEATHS_KEY: record}, - ) is False - assert episode.local_pass_used is False # neither refusal spent the episode's single local pass - assert loop_transport.fallback_chain_allowed(routable, "transport_unavailable", episode) is True - assert episode.local_pass_used is True - assert loop_transport.fallback_chain_allowed(routable, "transport_unavailable", episode) is False # once per episode - assert loop_transport.fallback_chain_allowed(routable, "provider_outcome_unknown", None) is False + assert loop_transport.fallback_chain_allowed(routable, "transport_unavailable", None, {}) is True + assert loop_transport.fallback_chain_allowed(routable, "provider_outcome_unknown", None, {}) is True + for kind in ("transport_unavailable", "provider_outcome_unknown", "bad_request"): + assert loop_transport.fallback_chain_allowed(routable, kind, episode, {}) is False + assert loop_transport.fallback_chain_allowed(routable, kind, None, {TRANSPORT_DEATHS_KEY: record}) is False + rejoinable = {"_pending_transport_outcome": {"same_operation_recoverable": True}} + assert loop_transport.fallback_chain_allowed(routable, "provider_outcome_unknown", None, rejoinable) is False + presence = SimpleNamespace(exact_model_route=False, task_id="t", current_task_type="presence") + assert loop_transport.fallback_chain_allowed(presence, "provider_outcome_unknown", None, {}) is False + assert loop_transport.fallback_chain_allowed(presence, "transport_unavailable", None, {}) is True + assert not hasattr(episode, "local_pass_used") def _overflowing_local_pass(spend_window): - """The episode's one local-only chain pass, failing with a context overflow - exactly as ``call_llm_with_retry`` stamps it, and slow enough that the wait - window is already spent when the round gate reads it (``spend_window``).""" + """The one configured-route walk before the wait, failing with a context overflow + exactly as ``call_llm_with_retry`` stamps it, and slow enough that the owner + deadline is already spent when the round gate reads it (``spend_window``).""" chain_calls = {"n": 0} def failing_chain(**kwargs): @@ -283,13 +294,14 @@ def _overflowing_local_pass(spend_window): return failing_chain, chain_calls -@pytest.mark.parametrize("turn", ["managed", "is_direct_chat"]) +@pytest.mark.parametrize("turn", ["managed"]) def test_latched_wait_cause_outranks_the_overflow_a_failed_local_pass_left(tmp_path, monkeypatch, turn): - """outage latched → the episode's one local-only pass fails with a context - overflow → the binding window (a managed task's deadline, an interactive - turn's idle bound) is spent before any redial: the round now holds BOTH - facts (``wait_cause == "transport_unavailable"`` and the pass's - ``context_overflow`` in the mutable kind), and the latched wait cause wins — + """outage → the one route walk before the wait fails with a context overflow → + the owner deadline is spent before any redial: the round's own outage owns its + wait (the walk's overflow never re-aims it), and the latched wait cause wins — + (An interactive turn's idle bound measures waiting from episode entry, which now + follows the walk, so a slow walk cannot spend it; the rail-level precedence is + pinned below.) durable source ``transport_unavailable_no_resend``, the wait terminal's ``provider_unavailable``/``infra_failed`` stamps, the outage wording and no forced-final dial — never the overflow salvage's source, ``llm_api_error`` @@ -321,9 +333,9 @@ def test_latched_wait_cause_outranks_the_overflow_a_failed_local_pass_left(tmp_p monkeypatch.setattr(loop_mod, "_run_cross_model_fallback_chain", failing_chain) result, usage, trace = run_llm_loop(**kwargs) - assert llm.calls == 1 and chain_calls["n"] == 1 # the primary send and the local pass; no redial, no forced dial - assert clock.sleeps == [] # the window was spent inside the pass, so the episode never slept - assert usage["_last_llm_error_kind"] == "context_overflow" # the pass's overwrite is still the sticky kind + assert llm.calls == 1 and chain_calls["n"] == 1 # the primary send and the walk; no redial, no forced dial + assert clock.sleeps == [] # the window was spent inside the walk, so the episode never slept + assert usage["_last_llm_error_kind"] == "transport_unavailable" # the round's own outage, not the walk's overflow assert usage["execution_status"] == "infra_failed" assert usage["reason_code"] == "provider_unavailable" assert trace["forced_finalization"]["source"] == "transport_unavailable_no_resend" diff --git a/tests/test_unknown_fallback_first.py b/tests/test_unknown_fallback_first.py new file mode 100644 index 000000000..366015cb5 --- /dev/null +++ b/tests/test_unknown_fallback_first.py @@ -0,0 +1,445 @@ +"""Owner decisions 1A/2A (#1407/#1409): configured routes before any wait, one recovery for +direct and queued turns, typed unknown eligibility, and model-owned return to the primary. + +Real round loop, round dispatcher and physical-attempt ledger; only each model's socket is +scripted, so the ledger proves which physical attempts existed and under which identity. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import httpx +import pytest + +import ouroboros.loop as loop_mod +import ouroboros.loop_transport as loop_transport +from ouroboros import owner_mailbox +from ouroboros import usage_accounting as ua +from ouroboros.loop import run_llm_loop +from ouroboros.loop_model_call import _route_candidates, _route_facts_text +from ouroboros.model_slots import MODEL_ACCOUNTS_KEY +from ouroboros.tools.control_runtime import _switch_model +from ouroboros.tools.registry import ToolRegistry +from supervisor.owner_stop import REASON_OWNER_STOPPED_DIRECT_TURN +from tests.test_llm_claudexor import ROUTE, result +from tests.test_llm_claudexor import setup as gateway_fixture +from tests.test_transport_death_retry import _events, _ledger + +setup = gateway_fixture +PRIMARY = "primary/model" + + +def _death(): + return httpx.ReadError("socket died after dispatch") + + +def _unreadable(): + """An accepted operation that could not be READ: it may still finish.""" + error = RuntimeError("model control link lost") + error.same_operation_recoverable = True + return error + + +class _RouteLLM: + """Every chat() is a REAL physical attempt; its send follows the script of the model it targets.""" + + def __init__(self, root, **scripts): + self.root, self.sent = root, [] + self.scripts = {model: list(steps) for model, steps in scripts.items()} + + def default_model(self): + return PRIMARY + + def chat(self, **kwargs): + model = kwargs["model"] + self.sent.append((model, [dict(row) for row in kwargs["messages"]])) + steps = self.scripts.get(model) or [] + step = steps.pop(0) if steps else None + + def send(): + if step is not None: + error = step() + if isinstance(error, httpx.HTTPError): + raise RuntimeError("Connection error.") from error + raise error + return {"content": "ok"} + + request = ua.AttemptRequest(model=model, provider="openrouter", reservation_usd=1.0, drive_root=self.root, + task_id="t-1a", root_task_id="t-1a", source="test.unknown_fallback_first") + ua.execute_physical_attempt(request, send, extractor=lambda _resp: ({"prompt_tokens": 1, "completion_tokens": 1}, 0.01, True)) + return {"role": "assistant", "content": f"answer from {model}"}, {"prompt_tokens": 1, "completion_tokens": 1} + + +@pytest.fixture +def data_root(tmp_path, monkeypatch): + root = tmp_path / "data" + monkeypatch.setenv("OUROBOROS_DATA_DIR", str(root)) + monkeypatch.setenv("OUROBOROS_SETTINGS_PATH", str(root / "settings.json")) + monkeypatch.setenv("TOTAL_BUDGET", "100") + monkeypatch.setenv("OUROBOROS_TASK_REVIEW_MODE", "off") + monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) + monkeypatch.delenv(MODEL_ACCOUNTS_KEY, raising=False) + monkeypatch.setattr(loop_mod, "_rebind_context_fit_plan", lambda plan, *_a, **_kw: (plan, "max")) + from ouroboros import fallback_cooldown + monkeypatch.setattr(fallback_cooldown, "is_cooling_down", lambda *_a: False) + (root / "state").mkdir(parents=True) + return root + + +def _run(tmp_path, llm, *, direct=False, notes=None): + registry = ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path) + registry._ctx.is_direct_chat = direct + notes = [] if notes is None else notes + with ua.usage_scope(ua.UsageScope(drive_root=llm.root, task_id="t-1a", root_task_id="t-1a", global_limit_usd=100)): + text, usage, trace = run_llm_loop( + messages=[{"role": "user", "content": "go"}], tools=registry, llm=llm, drive_logs=tmp_path, + emit_progress=lambda text, *, incident=None, **_kw: notes.append(text), incoming_messages=__import__("queue").Queue(), + task_id="t-1a", drive_root=tmp_path) + return text, usage, trace, registry + + +def _notices(messages): + return [row["content"] for row in messages if "[Transport recovery]" in str(row.get("content"))] + + +# -- 1A: a configured route answers first, for direct and queued turns alike -------------------- + + +@pytest.mark.parametrize("direct", [False, True]) +def test_unknown_outcome_tries_the_configured_route_first_with_a_new_identity(data_root, tmp_path, monkeypatch, direct): + """No legacy paid primary repeat runs ahead of the fallback (the direct turn's former rail), + no upstream probe or wait precedes it, and the old attempt keeps its own row and cost.""" + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "fb/one") + monkeypatch.setattr(loop_transport, "upstream_transport_reachable", + lambda *_a, **_kw: pytest.fail("a configured route answers before any wait")) + llm = _RouteLLM(data_root, **{PRIMARY: [_death]}) + text, usage, _trace, _registry = _run(tmp_path, llm, direct=direct) + + assert text == "answer from fb/one" + assert [model for model, _messages in llm.sent] == [PRIMARY, "fb/one"] + rows = _ledger(data_root) + assert [row["state"] for row in rows] == ["reserved", "dispatched", "unresolved", "reserved", "dispatched", "settled"] + old, new = rows[0]["attempt_id"], rows[3]["attempt_id"] + assert old != new + assert ua.usage_projection(data_root)["unresolved_upper_bound_usd"] == 1.0 + notices = _notices(llm.sent[-1][1]) + assert len(notices) == 1 and old in notices[0] and "fb/one" in notices[0] + assert usage["transport_recovery"]["previous_attempt"]["physical_attempt_id"] == old + assert _events(tmp_path, "network_wait") == [] + + +def test_repeated_unknowns_move_on_through_the_configured_routes(data_root, tmp_path, monkeypatch): + """An eligible unknown on a candidate moves on too; each new generation names the attempt it follows.""" + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "fb/one,fb/two") + llm = _RouteLLM(data_root, **{PRIMARY: [_death], "fb/one": [_death]}) + text, _usage, _trace, _registry = _run(tmp_path, llm, direct=True) + + assert text == "answer from fb/two" + assert [model for model, _messages in llm.sent] == [PRIMARY, "fb/one", "fb/two"] + first, second, _third = (row["attempt_id"] for row in _ledger(data_root) if row["state"] == "dispatched") + notices = _notices(llm.sent[-1][1]) + assert len(notices) == 2 and first in notices[0] and second in notices[1] + + +def test_fallback_can_recover_during_an_existing_primary_outage(data_root, tmp_path, monkeypatch): + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "fb/one") + probes, sleeps = [], [] + monkeypatch.setattr(loop_transport, "upstream_transport_reachable", + lambda *_a, **kw: probes.append(kw) or {}) + monkeypatch.setattr(loop_transport, "interruptible_wait_sleep", + lambda seconds, _wake: sleeps.append(seconds) or False) + llm = _RouteLLM(data_root, **{PRIMARY: [_death], "fb/one": [_death]}) + text, usage, _trace, _ctx = _run(tmp_path, llm, direct=True) + assert text == "answer from fb/one" + assert [model for model, _ in llm.sent] == [PRIMARY, "fb/one", "fb/one"] + assert len(probes) == len(sleeps) == 1 # the existing episode paces recovery + assert probes[0]["expected_route"] is None # the old failed fallback is not the primary probe's identity + assert usage["transport_recovery"]["old_outcome"] == "unknown" + assert [row["state"] for row in _ledger(data_root)].count("unresolved") == 2 + + +def test_non_unknown_candidate_failure_cannot_buy_a_forced_summary_after_unknown(): + from ouroboros.loop_llm_call import provider_no_call_source + + usage = {"_last_llm_error_kind": "bad_request", + "_pending_transport_outcome": {"physical_attempt_id": "earlier-unknown"}} + assert provider_no_call_source(usage, False)[0] == "provider_outcome_unknown_no_resend" + usage.pop("_pending_transport_outcome") + assert provider_no_call_source(usage, False)[0] == "" # normal finalization remains available + + +def test_without_a_configured_route_a_direct_turn_waits_and_continues_inside_its_bound(data_root, tmp_path, monkeypatch): + """The existing managed recovery, extended to a direct turn: upstream observation, then a + NEW attempt with facts-only input; never a paid same-request repeat.""" + monkeypatch.delenv("OUROBOROS_MODEL_FALLBACKS", raising=False) + observed = [] + monkeypatch.setattr(loop_transport, "upstream_transport_reachable", + lambda *_a, **_kw: observed.append(1) or {"kind": "upstream_http", "status_code": 200}) + monkeypatch.setattr(loop_transport, "interruptible_wait_sleep", lambda *_a: False) + llm = _RouteLLM(data_root, **{PRIMARY: [_death]}) + notes = [] + text, usage, _trace, _registry = _run(tmp_path, llm, direct=True, notes=notes) + + assert text == f"answer from {PRIMARY}" and observed == [1] + assert [model for model, _messages in llm.sent] == [PRIMARY, PRIMARY] + assert len(_notices(llm.sent[-1][1])) == 1 + rows = _events(tmp_path, "network_wait") + assert rows[0]["phase"] == "entered" and rows[-1]["detail"] == "new_attempt_after_unknown_outcome" + assert all("Stop cancels" not in note for note in notes) # an interactive episode's wording + assert usage["transport_recovery"]["old_outcome"] == "unknown" + + +def test_direct_stop_during_the_unknown_wait_sends_nothing_further(data_root, tmp_path, monkeypatch): + monkeypatch.delenv("OUROBOROS_MODEL_FALLBACKS", raising=False) + monkeypatch.setattr(loop_transport, "upstream_transport_reachable", lambda *_a, **_kw: {}) + + def stop_arrives(_seconds, _wake): + owner_mailbox.write_owner_message(tmp_path, REASON_OWNER_STOPPED_DIRECT_TURN, "t-1a", + msg_id="direct-stop", kind=owner_mailbox.KIND_FINALIZE_NOW) + return True + + monkeypatch.setattr(loop_transport, "interruptible_wait_sleep", stop_arrives) + llm = _RouteLLM(data_root, **{PRIMARY: [_death]}) + _text, usage, _trace, _registry = _run(tmp_path, llm, direct=True) + + assert [model for model, _messages in llm.sent] == [PRIMARY] + assert [row["state"] for row in _ledger(data_root)] == ["reserved", "dispatched", "unresolved"] + assert usage["_last_llm_error_kind"] == "provider_outcome_unknown" + + +def test_an_unreadable_accepted_operation_is_never_regenerated(data_root, tmp_path, monkeypatch): + """Disallowed counterpart: the same operation may still finish, so neither a configured + route nor a continuation runs; the turn ends on the honest no-resend terminal.""" + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "fb/one") + monkeypatch.setattr(loop_transport, "upstream_transport_reachable", lambda *_a, **_kw: pytest.fail("no continuation")) + llm = _RouteLLM(data_root, **{PRIMARY: [_unreadable]}) + _text, usage, trace, _registry = _run(tmp_path, llm, direct=True) + + assert [model for model, _messages in llm.sent] == [PRIMARY] + assert usage["_pending_transport_outcome"]["same_operation_recoverable"] is True + assert trace["forced_finalization"]["source"] == "provider_outcome_unknown_no_resend" + assert _events(tmp_path, "network_wait") == [] + + +def test_an_adopted_fallback_that_fails_retains_the_primary_as_a_route(data_root, tmp_path, monkeypatch): + """Nothing in Settings repeats Main, yet the primary binding is tried when the acting fallback fails.""" + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "fb/one") + tool_call = {"id": "call-1", "type": "function", "function": {"name": "chat_history", "arguments": "{}"}} + replies = iter([({"role": "assistant", "content": "", "tool_calls": [tool_call]}, {"prompt_tokens": 1, "completion_tokens": 1})]) + llm = _RouteLLM(data_root, **{PRIMARY: [_death], "fb/one": [None, lambda: RuntimeError("HTTP 400 bad request")]}) + original = llm.chat + + def chat(**kwargs): + if kwargs["model"] == "fb/one" and llm.scripts["fb/one"] and llm.scripts["fb/one"][0] is None: + llm.scripts["fb/one"].pop(0) + llm.sent.append(("fb/one", [dict(row) for row in kwargs["messages"]])) + return next(replies) + return original(**kwargs) + + llm.chat = chat + text, _usage, _trace, registry = _run(tmp_path, llm) + + sent = [model for model, _messages in llm.sent] + # r1: primary unknown -> fb/one answers with a tool call; r2: fb/one refuses -> the primary answers + # (a later primary round may follow: the host-driven handover's own recovery nudge). + assert sent[:4] == [PRIMARY, "fb/one", "fb/one", PRIMARY] and set(sent[4:]) <= {PRIMARY} + assert text == f"answer from {PRIMARY}" + facts = [row["content"] for row in llm.sent[2][1] if "[ROUTE FACTS]" in str(row.get("content"))] + assert len(facts) == 1 and PRIMARY in facts[0] and 'switch_model(primary="wait")' in facts[0] + assert registry._ctx.primary_route == {"model": PRIMARY, "use_local": False, "role": "main"} + + +# -- binding identity ----------------------------------------------------------------------------- + + +def test_route_candidates_compare_complete_bindings_not_model_strings(monkeypatch): + main = "claudexor::codex=gpt-test" + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", main) + monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) + tool_ctx = SimpleNamespace(primary_route={"model": main, "use_local": False, "role": "main"}) + monkeypatch.setenv(MODEL_ACCOUNTS_KEY, json.dumps({"fallback": ["account-b"]})) + assert _route_candidates(tool_ctx, main, False, "main", None) == [(main, "fallback:0", False, False)] + monkeypatch.setenv(MODEL_ACCOUNTS_KEY, json.dumps({"fallback": [""]})) + assert _route_candidates(tool_ctx, main, False, "main", None) == [] # the same binding is no alternative + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", "") + assert _route_candidates(tool_ctx, main, False, "main", None) == [] # an empty chain stays empty + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", f"{main},other/model") + monkeypatch.setenv(MODEL_ACCOUNTS_KEY, json.dumps({"fallback": ["account-b", ""]})) + acting = _route_candidates(tool_ctx, main, False, "fallback:0", None) + assert acting == [(main, "main", False, True), ("other/model", "fallback:1", False, False)] + owner_choice = SimpleNamespace(overrides={"main": {"model_account_override": "account-b"}}) + assert _route_candidates(tool_ctx, main, False, "fallback:0", owner_choice) == [ + ("other/model", "fallback:1", False, False)] # the owner's wait-card choice made them one binding + + +# -- 2A: the acting model decides whether to return ---------------------------------------------- + + +def test_switch_model_returns_to_the_exact_primary_binding_or_declares_a_wait(tmp_path): + ctx = SimpleNamespace(primary_route={"model": "consc/model", "use_local": False, "role": "consciousness"}, + model_wait_context=None, active_model_override=None, active_use_local_override=None, + active_effort_override=None) + assert "primary route consc/model" in _switch_model(ctx, primary="return") + assert (ctx.active_model_override, ctx.active_use_local_override, ctx.active_role_override) == ( + "consc/model", False, "consciousness") + assert ctx.route_wait_on_primary is False and ctx.active_effort_override is None # effort intent untouched + ctx.model_wait_context = SimpleNamespace(overrides={"consciousness": {"model": "owner/choice", "use_local": True}}) + assert "wait for it" in _switch_model(ctx, primary="wait") + assert (ctx.active_model_override, ctx.active_use_local_override, ctx.route_wait_on_primary) == ( + "owner/choice", True, True) + assert "TOOL_ARG_ERROR" in _switch_model(ctx, primary="wait", model="x") or "without model" in _switch_model( + ctx, primary="wait", model="x") + assert "primary must be" in _switch_model(ctx, primary="later") + assert "no recorded primary" in _switch_model(SimpleNamespace(primary_route=None), primary="return") + + +def test_a_declared_wait_keeps_refusals_and_outages_on_the_primary(): + ctx = SimpleNamespace(task_id="t", exact_model_route=False, route_wait_on_primary=True) + refusal = {"_last_llm_resource_refusal": "quota"} + for kind, usage in (("subscription_window_exhausted", refusal), ("transport_unavailable", {}), + ("provider_outcome_unknown", {})): + assert loop_transport.fallback_chain_allowed(ctx, kind, None, usage) is False + assert loop_transport.fallback_chain_allowed(ctx, "bad_request", None, {}) is True # ordinary recovery stays + ctx.route_wait_on_primary = False + assert loop_transport.fallback_chain_allowed(ctx, "subscription_window_exhausted", None, refusal) is True + + +def test_primary_return_rebinds_role_and_leaves_auto_unpinned(monkeypatch): + from ouroboros.loop_model_call import _apply_round_route_overrides + + calls = [] + monkeypatch.setattr(loop_mod, "_rebind_context_fit_plan", + lambda plan, *_a, **kwargs: calls.append(kwargs) or (plan, "low")) + ctx = SimpleNamespace(primary_route={"model": "consc/model", "use_local": False, "role": "consciousness"}, + model_wait_context=None, active_model_override=None, active_use_local_override=None, + active_effort_override=None) + _switch_model(ctx, primary="return") + messages = [{"role": "user", "content": "go"}] + route = _apply_round_route_overrides(ctx, SimpleNamespace(_ctx=ctx), messages, ("fb/one", False, "high"), + "plan", "max", "max", []) + assert route == ("consc/model", False, "high", "plan", "low") + assert calls[-1]["model_role"] == "consciousness" and calls[-1]["model_route"] == {} + assert ctx.active_role_override is None # one-shot + + +def test_initial_route_records_a_role_slot_primary(tmp_path): + ctx = SimpleNamespace(task_model_override="consc/model", task_use_local_override=False, + task_metadata={"model_role": "consciousness"}, context_fit_plan=None) + model, _effort, local, *_rest = loop_mod._initial_round_route(ctx, SimpleNamespace(default_model=lambda: "main/model"), "high") + assert (model, local) == ("consc/model", False) + assert ctx.primary_route == {"model": "consc/model", "use_local": False, "role": "consciousness"} + + +def test_route_facts_name_both_bindings_the_dated_failure_and_the_choices(): + ctx = SimpleNamespace(primary_route={"model": "claudexor::codex=gpt-test", "use_local": False, "role": "main"}) + text = _route_facts_text(ctx, model="fb/one", use_local=False, role="fallback:0", + failed_model="claudexor::codex=gpt-test", + failure={"failure_code": "subscription_window_exhausted", "ts": "2026-09-30T12:00:00Z", + "reset_at": "2026-09-30T18:00:00Z"}, waiter=None) + assert text.startswith("[ROUTE FACTS]") and "(role main; account Auto)" in text and "(role fallback:0)" in text + assert "subscription_window_exhausted at 2026-09-30T12:00:00Z" in text and "2026-09-30T18:00:00Z" in text + assert 'switch_model(primary="return")' in text and "No catalog check or timer" in text + assert _route_facts_text(SimpleNamespace(), model="m", use_local=False, role="r", failed_model="p", + failure={}, waiter=None) == "" + + +def test_route_facts_reach_only_the_acting_routes_own_next_round(monkeypatch): + from ouroboros import loop_model_call + + tool_ctx = SimpleNamespace(_route_facts_pending="[ROUTE FACTS] facts") + monkeypatch.setattr(loop_mod, "_dispatch_round_model", lambda *_a, **_kw: ({"content": "x"}, 0.0)) + monkeypatch.setattr(loop_model_call, "_append_routing_receipts", lambda _ctx: False) + monkeypatch.setattr(loop_model_call, "_project_wake_input", lambda *_a, **_kw: False) + monkeypatch.setattr(loop_mod, "_measure_round_main_fit", lambda *_a, **_kw: None) + + def call(defer): + messages = [{"role": "user", "content": "go"}, {"role": "assistant", "content": "tool round"}] + loop_model_call._call_round_model(SimpleNamespace( + tools=SimpleNamespace(_ctx=tool_ctx), messages=messages, defer_resource_wait=defer, + attempt_cap=None, accumulated_usage={}, active_context_mode="max")) + return messages + + assert call(True)[-1]["content"] == "tool round" # a candidate's copy never takes the note + assert call(None)[-1]["content"] == "[ROUTE FACTS] facts" + assert tool_ctx._route_facts_pending == "" and call(None)[-1]["content"] == "tool round" # exactly once + + +# -- Claudexor custody: engine-terminal unknown vs. an operation that could not be read ---------- + + +def test_engine_terminal_unknown_is_eligible_and_a_lost_control_link_is_not(setup): + from ouroboros import llm_claudexor as transport + + _root, gateway, client = setup + gateway.results, gateway.dispatch = [result(outcome="unknown"), result()], ["unknown", "response_received"] + with pytest.raises(transport.ClaudexorModelError) as terminal: + client.chat([{"role": "user", "content": "hi"}], "claudexor::codex=exact-model") + assert terminal.value.code == "model_outcome_unknown" and terminal.value.same_operation_recoverable is False + gateway.pending = gateway.read_error = True + with pytest.raises(transport.ClaudexorModelError) as lost: + client.chat([{"role": "user", "content": "hi"}], "claudexor::codex=exact-model", timeout=0.01) + assert lost.value.code == "model_outcome_unknown" and lost.value.same_operation_recoverable is True + + +@pytest.mark.parametrize("managed", [True, False]) +def test_a_refused_read_of_an_accepted_operation_keeps_its_custody(tmp_path, monkeypatch, managed): + """A 404 is no proof of non-dispatch: ordinary direct and queued calls rejoin the same operation.""" + from ouroboros import llm_claudexor + from ouroboros.gateways.claudexor import ClaudexorUnavailable + + inv = llm_claudexor._ModelInvocation({"usage_model": "claudexor::codex=test"}, {}, {"timeout": 1}) + inv.operation_id, inv.invocation_id, inv.task_id, inv.root = "same-op", "same-attempt", "t", tmp_path + inv.create_attempted = True + reads = [] + ctx = SimpleNamespace(task_id="t", is_direct_chat=not managed) + monkeypatch.setattr(llm_claudexor, "current_model_wait", + lambda: SimpleNamespace(tool_context=ctx, control_reason=lambda: None)) + now = [0.0] + monkeypatch.setattr(llm_claudexor, "time", SimpleNamespace( + monotonic=lambda: now[0], sleep=lambda seconds: now.__setitem__(0, now[0] + seconds))) + + class Gateway: + def get_model_operation(self, operation, **_kwargs): + reads.append(operation) + if len(reads) <= 2: + raise ClaudexorUnavailable("http_404", "operation not found", status_code=404) + return {"id": operation, "state": "succeeded", "dispatch": {"state": "response_received", "route": ROUTE}, + "response": {"state": "ready", "ref": {"sha256": "exact"}}} + + def get_model_result(self, operation, **_kwargs): + return b'{"outcome":"completed","message":{"content":"same operation answer"}}' + + def create_model_operation(self, *_a, **_kw): + pytest.fail("a second model operation") + + def close(self): + pass + + gateway = inv.gateway = Gateway() + monkeypatch.setattr(llm_claudexor, "read_owned_gateway", lambda: gateway) + monkeypatch.setattr(inv, "retain", lambda _raw: None) + assert inv.receive()["message"]["content"] == "same operation answer" + assert reads == ["same-op"] * 3 + + +def test_a_readable_unknown_after_a_granted_continuation_ends_the_episode(tmp_path): + """Allowed vs disallowed after a grant: a new eligible unknown keeps the same episode + (same clock and backoff), while one whose accepted operation is still readable ends it, + so no further generation can be granted over an operation that may still finish.""" + def granted(): + return loop_transport.TransportWaitEpisode( + wait_cause="provider_outcome_unknown", started_monotonic=0.0, continuation_granted=True, + outcome_custody={"physical_attempt_id": "old"}, wait_iterations=1, redials=1) + + kwargs = dict(msg_present=False, error_kind="provider_outcome_unknown", drive_logs=tmp_path, task_id="t", + model="m", emit_progress=lambda *_a, **_kw: None) + eligible = SimpleNamespace(task_id="t", _accumulated_usage={ + "_pending_transport_outcome": {"physical_attempt_id": "new", "outcome": "unknown"}}) + episode = granted() + assert loop_transport.reconcile_transport_wait(episode, eligible, **kwargs) is episode + assert not episode.continuation_granted and episode.outcome_custody["physical_attempt_id"] == "new" + readable = SimpleNamespace(task_id="t", _accumulated_usage={"_pending_transport_outcome": { + "physical_attempt_id": "new", "outcome": "unknown", "same_operation_recoverable": True}}) + assert loop_transport.reconcile_transport_wait(granted(), readable, **kwargs) is None diff --git a/web/modules/model_roles.js b/web/modules/model_roles.js index 4f00982a0..4f3a79199 100644 --- a/web/modules/model_roles.js +++ b/web/modules/model_roles.js @@ -217,7 +217,7 @@ export function createModelRolesEditor({ hostId, store = claudexorStatus, ${slot.settingsToggleId ? `` : ''} ${slot.slot === 'fallback' ? '' : ''} - ${slot.slot === 'fallback' ? '

Tried in this order. Subscription quota waits for your choice before using API.

' : ''} + ${slot.slot === 'fallback' ? '

Tried in this order when the acting model refuses, cannot be reached or loses its answer. A wait starts only when none of them answers.

' : ''} ${matching.map((row, index) => rowHtml(row, index, matching.length)).join('')} `; }).join(''); diff --git a/web/modules/model_wait.js b/web/modules/model_wait.js index cc1630b62..3ee77675c 100644 --- a/web/modules/model_wait.js +++ b/web/modules/model_wait.js @@ -26,13 +26,16 @@ export function modelWaitRoleLabel(role = '') { return String(role || 'Model'); } +/** Host wait reasons: `unavailable` is an engine-dated pool refusal, neither quota nor sign-in. */ +export const MODEL_WAIT_REASONS = ['quota', 'auth', 'auth_quota', 'unavailable']; + /** A late snapshot cannot rewind a row or reopen a resolved wait episode. */ export function mergeModelWaits(previous = {}, incoming = {}) { const result = { ...previous }; for (const [id, row] of Object.entries(incoming || {})) { if (!row || row.wait_id !== id || !Number.isInteger(row.revision) || row.revision < 1 || !Number.isInteger(row.task_attempt) || row.task_attempt < 1 - || !['waiting', 'resolved'].includes(row.state) || !['quota', 'auth', 'auth_quota'].includes(row.reason)) continue; + || !['waiting', 'resolved'].includes(row.state) || !MODEL_WAIT_REASONS.includes(row.reason)) continue; const prior = result[id]; if (prior && (prior.revision > row.revision || prior.state === 'resolved')) continue; if (prior?.revision === row.revision) { @@ -130,25 +133,31 @@ export function createModelWaitController({ getRecord, onDomWrite = (fn) => fn() const node = view.node; const pending = view.sending || isPending(row); const mixed = row.reason === 'auth_quota'; + const pool = row.reason === 'unavailable'; const quota = row.reason === 'quota' || mixed; + const observesAccess = row.availability_observation !== 'unavailable'; + const timed = (quota || pool) && observesAccess; node.querySelector('[data-wait-role]').textContent = modelWaitRoleLabel(row.role); - node.querySelector('[data-wait-reason]').textContent = mixed ? 'Waiting for access' - : row.reason === 'auth' ? 'Sign-in required' : 'Waiting for quota'; + node.querySelector('[data-wait-reason]').textContent = !observesAccess ? 'Waiting for provider access' : mixed ? 'Waiting for access' + : pool ? 'Waiting for an account' : row.reason === 'auth' ? 'Sign-in required' : 'Waiting for quota'; const model = parseModelSource(row.model).model || row.model; - node.querySelector('[data-wait-model]').textContent = `${model || 'Model'} · ${row.credential_profile_id ? `Account: ${row.credential_profile_id}` : 'Auto rotation'}`; + node.querySelector('[data-wait-model]').textContent = `${model || 'Model'} · ${!observesAccess ? 'Configured API access' : row.credential_profile_id ? `Account: ${row.credential_profile_id}` : 'Auto rotation'}`; const reset = row.reset_at ? new Date(row.reset_at) : null; - node.querySelector('[data-wait-reset]').textContent = (mixed - ? 'Some accounts need sign-in; others are waiting for quota. ' : '') + (quota - ? (reset && Number.isFinite(reset.getTime()) ? `Quota resets ${reset.toLocaleString()}.` : 'Quota reset time is not known.') : ''); + const known = reset && Number.isFinite(reset.getTime()); + node.querySelector('[data-wait-reset]').textContent = !observesAccess + ? 'Automatic access checks are unavailable. Retry after access is restored, choose another model, or stop.' : (mixed + ? 'Some accounts need sign-in; others are waiting for quota. ' : '') + (pool + ? `No account can serve this model now. ${known ? `The engine reported a reset at ${reset.toLocaleString()}; availability is unconfirmed.` : 'The reset time is not known.'}` + : quota ? (known ? `Quota resets ${reset.toLocaleString()}.` : 'Quota reset time is not known.') : ''); const automatic = node.querySelector('[data-wait-auto]'); - automatic.parentElement.hidden = !quota; - node.querySelector('[data-wait-auto-label]').textContent = mixed + automatic.parentElement.hidden = !timed; + node.querySelector('[data-wait-auto-label]').textContent = mixed || pool ? 'Continue automatically when access is restored' : 'Continue automatically when quota resets'; const requested = view.sending ? view.lastRequest : row.pending_action; automatic.checked = pending && requested?.action === 'auto_continue' ? requested.auto_continue : row.auto_continue !== false; const auth = node.querySelector('[data-wait-login]'); - auth.hidden = row.reason !== 'auth'; + auth.hidden = row.reason !== 'auth' || !observesAccess; node.querySelector('[data-wait-settings]').textContent = mixed ? 'Accounts' : 'Settings'; node.querySelector('[data-wait-notice]').textContent = view.error || (view.sending ? 'Sending your choice…' : isPending(row) diff --git a/web/tests/model_wait.test.js b/web/tests/model_wait.test.js index 11ea58512..1900e487a 100644 --- a/web/tests/model_wait.test.js +++ b/web/tests/model_wait.test.js @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { activeModelWaits, createModelWaitController, isModelWaitReference, mergeModelWaits, modelWaitAction, modelWaitRoleLabel } from '../modules/model_wait.js'; +import { activeModelWaits, createModelWaitController, isModelWaitReference, MODEL_WAIT_REASONS, mergeModelWaits, modelWaitAction, modelWaitRoleLabel } from '../modules/model_wait.js'; import { desiredLiveCardPhase } from '../modules/task_phase_chip.js'; import { computeDerivedChatStatus } from '../modules/chat_activity.js'; @@ -74,7 +74,13 @@ test('confirmed mixed access keeps an actionable replay row without inventing an const state = mergeModelWaits({}, { [mixed.wait_id]: mixed }); assert.deepEqual(activeModelWaits(state), [mixed]); assert.equal(state[mixed.wait_id].credential_profile_id, ''); - assert.deepEqual(mergeModelWaits({}, { [mixed.wait_id]: row({ reason: 'unavailable' }) }), {}); + assert.deepEqual(mergeModelWaits({}, { [mixed.wait_id]: row({ reason: 'pool' }) }), {}); +}); + +test('an engine-dated unavailable pool is its own actionable reason, not quota or sign-in', () => { + const pool = row({ reason: 'unavailable', credential_profile_id: '', reset_at: '2099-01-01T00:00:00Z' }); + assert.deepEqual(activeModelWaits(mergeModelWaits({}, { [pool.wait_id]: pool })), [pool]); + assert.ok(MODEL_WAIT_REASONS.includes('unavailable') && !MODEL_WAIT_REASONS.includes('transport')); }); test('switch payload binds the exact wait revision and defaults to temporary role change', () => { From 14360dedf2241bcd22d63cf59b99b57a092177a5 Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:41:48 +0300 Subject: [PATCH 02/12] fix: preserve fallback account and primary intent across recovery --- docs/architecture/06-agent-core.md | 2 +- ouroboros/fallback_cooldown.py | 19 ++--- ouroboros/loop_model_call.py | 23 ++++-- ouroboros/owner_wait.py | 1 + tests/test_model_wait_browser.py | 9 ++- ...test_native_fallback_binding_acceptance.py | 76 +++++++++++++++++++ .../test_transport_wait_repeat_interaction.py | 15 ++-- 7 files changed, 120 insertions(+), 25 deletions(-) diff --git a/docs/architecture/06-agent-core.md b/docs/architecture/06-agent-core.md index 2b87eede7..34841c769 100644 --- a/docs/architecture/06-agent-core.md +++ b/docs/architecture/06-agent-core.md @@ -171,7 +171,7 @@ Retry rails nest, each with its own owner and bound; the table exists so the mul | SDK `max_retries` | client construction (`net_transport.py`, `llm.py`) | 0 everywhere | nothing: every physical send is visible to the ledger | | request-wire recovery ladder | `request_wire_recovery.py` inside one `LLMClient.chat` | at most eight typed compatibility actions per call | a rejected request in a corrected wire shape | | transport-death repeat | `call_llm_with_retry`, inline Presence primary round only | ≤2 per round, 4 s then 8 s deadline-aware backoff | a dispatched request that died with a typed transport death, as a new attempt | -| configured-route walk | `loop_model_call._run_cross_model_fallback_chain` | one attempt per route per failed round, none inside a wait episode | the round on each other configured binding, the primary first when a fallback acts | +| configured-route walk | `loop_model_call._run_cross_model_fallback_chain` | bounded attempts per binding per walk; retried with existing wait pacing | the round on each other configured binding, the primary first when a fallback acts | | transient burst | `call_llm_with_retry` | `OUROBOROS_TRANSIENT_RETRY_MAX` attempts per call (the outer ceiling of the same loop) | typed 408/429/5xx and empty/incomplete responses | | round redial | `loop_transport.py` wait episode | a managed task: owner deadline, budget, Stop, absolute ceiling; an interactive turn: the task idle timeout measured from episode entry (`OUROBOROS_TASK_IDLE_TIMEOUT_SEC`), or an explicit deadline when that is shorter | a released ($0) pre-dispatch failure of the same round, or an eligible unknown as a marked new attempt | | served-model redo | `llm_substitution.py` inside one `chat_claudexor` | `OUROBOROS_SERVED_MODEL_REDOS` per call, and none under a pin, an admitted candidate, a spent send budget or a spent owner window | a round the engine says another model answered, as a new operation. It nests inside the rails above, so a repeat granted by one of them starts a call whose redo budget begins again | diff --git a/ouroboros/fallback_cooldown.py b/ouroboros/fallback_cooldown.py index 5d9bc436c..89e4a5215 100644 --- a/ouroboros/fallback_cooldown.py +++ b/ouroboros/fallback_cooldown.py @@ -1,9 +1,10 @@ """Process-local 429-aware cooldown for the cross-model fallback chain (F1, v6.39). -A model that just failed transiently (429 / 5xx / overloaded) is parked on a short +A route that just failed transiently (429 / 5xx / overloaded) is parked on a short cooldown so the fallback chain skips it for a window instead of immediately hammering it again. Scope is PER-PROCESS: it bounds re-hits WITHIN one worker — a single task's own -fallback walk and its repeated rounds stop re-trying a just-rate-limited model. It does +fallback walk and its repeated rounds stop re-trying a just-rate-limited binding. +Account pins and Auto are distinct: one account does not cool another. It does NOT coordinate across sibling worker processes (each worker has its own map), so it is not a swarm-WIDE rate-limit governor; durable cross-worker coordination is the project journal / task-tree blackboard's job (Phase 3), not this advisory in-process guard. The @@ -20,7 +21,7 @@ import time from typing import Dict, Tuple from ouroboros.config import runtime_setting -_cooldown: Dict[Tuple[str, bool], float] = {} +_cooldown: Dict[Tuple[str, bool, str], float] = {} _lock = threading.Lock() @@ -50,21 +51,21 @@ def attempts_per_model() -> int: return 1 -def mark_cooldown(model: str, use_local: bool = False) -> None: - """Put a model on cooldown after a TRANSIENT failure (caller classifies).""" +def mark_cooldown(model: str, use_local: bool = False, account: str = "") -> None: + """Cool the failed binding (configured pin or Auto, never the observed Auto account).""" if not cooldown_enabled(): return - key = (str(model or ""), bool(use_local)) + key = (str(model or ""), bool(use_local), str(account or "")) with _lock: _cooldown[key] = time.time() + _cooldown_sec() -def is_cooling_down(model: str, use_local: bool = False) -> bool: - """True while the model is inside its cooldown window. Passive heal-back: an expired +def is_cooling_down(model: str, use_local: bool = False, account: str = "") -> bool: + """True while the binding is inside its cooldown window. Passive heal-back: an expired entry is dropped and reads as available again.""" if not cooldown_enabled(): return False - key = (str(model or ""), bool(use_local)) + key = (str(model or ""), bool(use_local), str(account or "")) now = time.time() with _lock: until = _cooldown.get(key, 0.0) diff --git a/ouroboros/loop_model_call.py b/ouroboros/loop_model_call.py index 140a25fa1..a2407e6ac 100644 --- a/ouroboros/loop_model_call.py +++ b/ouroboros/loop_model_call.py @@ -235,14 +235,14 @@ def _run_cross_model_fallback_chain( """ from ouroboros import fallback_cooldown as _fcd from ouroboros.loop_transport import append_unknown_recovery_input - from ouroboros.model_slots import MODEL_ACCOUNTS_KEY, model_role_option, task_model_binding + from ouroboros.model_slots import MODEL_ACCOUNTS_KEY, model_role_option, task_model_binding, route_binding from ouroboros.model_wait import current_model_wait from ouroboros.loop_llm_call import _COOLDOWN_ERROR_KINDS as _cooldown_kinds from ouroboros.provider_models import provider_for_model - def _cooled(model: str, use_local: bool) -> None: + def _cooled(model: str, use_local: bool, role: str) -> None: if str(accumulated_usage.get("_last_llm_error_kind") or "") in _cooldown_kinds: - _fcd.mark_cooldown(model, use_local) + _fcd.mark_cooldown(*route_binding(model, use_local, role, overrides=waiter.overrides if waiter else None)) def _disclose_unknown(target: List[Dict[str, Any]], route: str) -> None: # A NEW generation after an eligible unknown outcome names the unknown attempt it follows. @@ -252,10 +252,10 @@ def _run_cross_model_fallback_chain( lead=f"The previous attempt ended without a usable answer; the configured route {route} continues.", continuation="configured_route") - _cooled(active_model, active_use_local) + waiter = current_model_wait() + _cooled(active_model, active_use_local, str(getattr(context_fit_plan, "model_role", "") or "main")) primary_context_usage = _snapshot_context_fit_usage(accumulated_usage) attempt_cap = _fcd.attempts_per_model() - waiter = current_model_wait() # The round's own outage or unknown outcome remains its wait if no route answers. entry_model, entry_kind = active_model, str(accumulated_usage.get("_last_llm_error_kind") or "") own_wait = entry_kind in _ROUND_WAIT_KINDS @@ -280,7 +280,8 @@ def _run_cross_model_fallback_chain( # transport boundary. Every chain row uses the single global USE_LOCAL_FALLBACK # flag (the pre-existing chain contract); the primary keeps its own locality. for index, (fallback_model, fallback_role, fallback_use_local, is_primary) in enumerate(rows): - if _fcd.is_cooling_down(fallback_model, fallback_use_local): + if _fcd.is_cooling_down(*route_binding(fallback_model, fallback_use_local, fallback_role, + overrides=waiter.overrides if waiter else None)): continue deadline = _loop()._task_deadline_epoch(tools) if deadline and time.time() >= deadline: @@ -402,7 +403,7 @@ def _run_cross_model_fallback_chain( _restore_context_fit_usage(accumulated_usage, primary_context_usage) if _walk_fenced(tools._ctx, accumulated_usage): break - _cooled(fallback_model, fallback_use_local) + _cooled(fallback_model, fallback_use_local, fallback_role) previous_model, previous_tag = fallback_model, ftag fenced = msg is None and _walk_fenced(tools._ctx, accumulated_usage) if deferred is None: @@ -695,7 +696,13 @@ class _RoundModelCallContext: def _route_follows(candidates: List[Tuple[str, str, bool, bool]]) -> bool: from ouroboros import fallback_cooldown - return any(not fallback_cooldown.is_cooling_down(model, use_local) for model, _role, use_local, _primary in candidates) + from ouroboros.model_slots import route_binding + from ouroboros.model_wait import current_model_wait + + waiter = current_model_wait() + return any(not fallback_cooldown.is_cooling_down(*route_binding( + model, use_local, role, overrides=waiter.overrides if waiter else None)) + for model, role, use_local, _primary in candidates) def _fallback_route_follows(ctx: _RoundModelCallContext) -> bool: diff --git a/ouroboros/owner_wait.py b/ouroboros/owner_wait.py index 60b39b06a..28449dbc0 100644 --- a/ouroboros/owner_wait.py +++ b/ouroboros/owner_wait.py @@ -150,6 +150,7 @@ def continuation_state(ctx: Any, messages: list, trace: dict, usage: dict, "route": {key: getattr(ctx, key, None) for key in ( "active_model", "active_effort", "active_use_local", "active_context_mode", "active_model_override", "active_effort_override", "active_use_local_override", + "active_role_override", "route_wait_on_primary", "primary_route", "_route_facts_pending", )}, "delivery_candidate": asdict(candidate) if candidate is not None else None, "delivery": {key: getattr(ctx, key, None) for key in ( diff --git a/tests/test_model_wait_browser.py b/tests/test_model_wait_browser.py index b8817d2d0..b502c6b07 100644 --- a/tests/test_model_wait_browser.py +++ b/tests/test_model_wait_browser.py @@ -273,8 +273,15 @@ def test_wait_updates_preserve_reading_position_away_from_the_live_edge(waiting_ page.reload() page.wait_for_selector('.chat-bubble.user') page.wait_for_selector('[data-wait-id="light-wait"]') + page.wait_for_selector('#chat-messages[data-history-hydrated="true"]') scroll = page.locator('#chat-messages') - scroll.evaluate('(el) => { el.style.overflowAnchor = "none"; el.scrollTop = 300; }') + scroll.evaluate('(el) => { el.style.overflowAnchor = "none"; }') + # A real upward gesture cancels the initial history-follow frames and + # establishes reading intent; assigning scrollTop alone does neither. + page.wait_for_function("() => document.querySelector('#chat-messages').scrollTop > 0") + scroll.hover() + page.mouse.wheel(0, 300 - scroll.evaluate('(el) => el.scrollTop')) + page.wait_for_function("() => Math.abs(document.querySelector('#chat-messages').scrollTop - 300) <= 1") assert scroll.evaluate('(el) => el.scrollHeight - el.clientHeight - el.scrollTop > 48') anchor = page.locator('.chat-bubble.user').nth(2) before = anchor.bounding_box()['y'] diff --git a/tests/test_native_fallback_binding_acceptance.py b/tests/test_native_fallback_binding_acceptance.py index af8d1672b..cca530d6a 100644 --- a/tests/test_native_fallback_binding_acceptance.py +++ b/tests/test_native_fallback_binding_acceptance.py @@ -307,3 +307,79 @@ def test_primary_wait_honors_a_direct_api_access_refusal(main_call, monkeypatch, assert waits[0]["availability_observation"] == "unavailable" and waits[0]["auto_continue"] is False assert waits[0]["credential_harness"] == "" and waits[-1]["resolution"] == "cancelled" assert sent == ["openai"] and gateway.creates == [] + + +@pytest.mark.parametrize("primary_account,fallback_account", [("account-a", "account-b"), ("", "account-b"), ("account-a", "")]) +def test_transient_cooldown_keeps_other_native_account_available(main_call, monkeypatch, primary_account, fallback_account): + from ouroboros import loop_model_call + + ctx, gateway, owner, _events, _decide, _observations = main_call + monkeypatch.setenv("OUROBOROS_MODEL_FALLBACKS", MODEL) + monkeypatch.setenv("OUROBOROS_FALLBACK_COOLDOWN_ENABLED", "true") + monkeypatch.setenv(MODEL_ACCOUNTS_KEY, json.dumps({"main": primary_account, "fallback": ["decoy"]})) + owner.overrides["fallback:0"] = {"model_account_override": fallback_account} + inner = ctx.tools._ctx + inner.primary_route = {"model": MODEL, "use_local": False, "role": "main"} + ctx.accumulated_usage["_last_llm_error_kind"] = "provider_transient" + gateway.results = [_reply(ROUTE_B, {"role": "assistant", "content": "Other account answered."})] + gateway.dispatch = ["response_received"] + _bounded_creates(monkeypatch, gateway, 1) + fallback_cooldown.reset_for_tests() + try: + reply, *_ = loop_model_call._run_cross_model_fallback_chain( + llm=ctx.llm, ctx=inner, tools=ctx.tools, messages=ctx.messages, active_model=MODEL, + active_use_local=False, tool_schemas=[], active_effort="high", max_retries=1, + drive_logs=ctx.drive_logs, task_id=inner.task_id, round_idx=1, event_queue=ctx.event_queue, + accumulated_usage=ctx.accumulated_usage, task_type="task", emit_progress=lambda *_a, **_k: None, + context_fit_plan=ctx.context_fit_plan, active_context_mode="max") + assert reply["content"] == "Other account answered." + sent = gateway.uploads[0][0]["account"] + if fallback_account: + assert sent == {"mode": "pin", "profileId": fallback_account} + else: + assert sent["mode"] == "auto" and "profileId" not in sent + assert fallback_cooldown.is_cooling_down(MODEL, False, primary_account) + assert not fallback_cooldown.is_cooling_down(MODEL, False, fallback_account) + # Resource-refusal deferral must use the same account key as the actual walk. + assert loop_model_call._route_follows([(MODEL, "fallback:0", False, False)]) + assert not loop_model_call._route_follows([(MODEL, "main", False, True)]) + finally: + fallback_cooldown.reset_for_tests() + + +@pytest.mark.parametrize("choice", ["return", "wait"]) +@pytest.mark.parametrize("primary_account", ["account-a", ""]) +def test_primary_choice_survives_shared_cold_continuation(main_call, monkeypatch, choice, primary_account): + from ouroboros import budget_pause, loop_model_call, owner_wait + from ouroboros.model_slots import route_binding + + ctx, _gateway, owner, _events, _decide, _observations = main_call + monkeypatch.setenv(MODEL_ACCOUNTS_KEY, json.dumps({"main": "decoy", "fallback": ["account-b"]})) + owner.overrides["main"] = {"model_account_override": primary_account} + inner = ctx.tools._ctx + inner.model_wait_context = owner + inner.active_effort, inner.active_use_local = "high", False + inner.context_fit_plan = replace(ctx.context_fit_plan, model_role="fallback:0") + inner.primary_route = {"model": MODEL, "use_local": False, "role": "main"} + loop._reset_turn_state(inner) + assert "OK:" in _switch_model(inner, primary=choice) + inner._route_facts_pending = "dated refusal fact" + state = owner_wait.continuation_state(inner, ctx.messages, {}, {}, 1, [], set()) + # JSON storage and a fresh turn can replace the initial role before restore. + state = json.loads(json.dumps(state)) + loop._reset_turn_state(inner) + inner.primary_route = {"model": "decoy", "use_local": False, "role": "fallback:0"} + inner.budget_drive_root = inner.drive_root = ctx.drive_root + state["_pause_row"] = {"pause_id": "primary-pause", "state": budget_pause.STATE_RESUME_GRANTED, + "rail": budget_pause.RAIL_GLOBAL_EXHAUSTED, "grant": {"grant_id": "owner-resume"}} + budget_pause.set_budget_pause(ctx.drive_root, inner.task_id, state["_pause_row"]) + model, effort, local, mode, _round, restored_plan = budget_pause.resume_paused_loop( + ctx.tools, state, ctx.messages, {}, {}, set(), budget_remaining_usd=5.0) + assert budget_pause.budget_pause_row(ctx.drive_root, inner.task_id)["state"] == budget_pause.STATE_RESUMED + assert any("explicit owner Resume" in str(row.get("content")) for row in ctx.messages) + model, local, effort, plan, _mode = loop_model_call._apply_round_route_overrides( + inner, ctx.tools, ctx.messages, (model, local, effort), restored_plan, mode, "max", []) + assert plan.model_role == "main" and inner.route_wait_on_primary is (choice == "wait") + assert route_binding(model, local, plan.model_role, overrides=owner.overrides) == (MODEL, False, primary_account) + assert inner.primary_route == {"model": MODEL, "use_local": False, "role": "main"} + assert inner._route_facts_pending == "dated refusal fact" and effort == "high" diff --git a/tests/test_transport_wait_repeat_interaction.py b/tests/test_transport_wait_repeat_interaction.py index 2993b1387..28c5a924d 100644 --- a/tests/test_transport_wait_repeat_interaction.py +++ b/tests/test_transport_wait_repeat_interaction.py @@ -7,8 +7,8 @@ wait terminal: - a dispatched death on an inline Presence turn (the one caller that keeps the repeat rail) takes the repeat rail and never opens a wait episode; -- the configured-route walk runs once BEFORE a wait episode opens and never inside - one, and a round record blocks it; +- configured alternatives run before waiting and after paced failed redials; + an unresolved Presence repeat record blocks them throughout; - an episode exhausted on a round that still holds a repeat record ends on the record's source, worded as both the wait and the unresolved attempt — and that wording names the class the repeat was RELEASED with (it rides the round record, @@ -138,8 +138,11 @@ def test_wait_episode_exhausted_on_a_round_holding_a_repeat_record_takes_the_unk else: _presence_turn(kwargs) result, usage, trace = run_llm_loop(**kwargs) - # One walk before the wait; a round record (an unresolved paid repeat) blocks even that one. - assert walks == ([] if with_record else ["transport_unavailable"]) + # Each failed redial may revisit alternatives; an unresolved Presence repeat never may. + if with_record: + assert walks == [] + else: + assert len(walks) > 1 and set(walks) == {"transport_unavailable"} assert (3 if with_record else 2) <= llm.calls <= len(script) # the episode ended; the turn never recovered assert no_sleep == ([4.0] if with_record else []) # one repeat backoff; released redials never re-arm it @@ -251,7 +254,7 @@ def test_generic_terminal_names_the_class_the_repeat_was_released_with(tmp_path, def test_fallback_chain_fence_holds_inside_a_wait_episode_too(monkeypatch): """`fallback_chain_allowed` on the combined tree: the configured-route walk runs - BEFORE a wait episode opens (remote routes included) and never inside one; a round + before a wait episode opens and after paced redials (remote routes included); a round record — an unresolved paid repeat of this round — blocks it whatever the kind says; an unknown outcome walks only when it is eligible for a new generation.""" monkeypatch.delenv("USE_LOCAL_FALLBACK", raising=False) @@ -262,7 +265,7 @@ def test_fallback_chain_fence_holds_inside_a_wait_episode_too(monkeypatch): assert loop_transport.fallback_chain_allowed(routable, "transport_unavailable", None, {}) is True assert loop_transport.fallback_chain_allowed(routable, "provider_outcome_unknown", None, {}) is True for kind in ("transport_unavailable", "provider_outcome_unknown", "bad_request"): - assert loop_transport.fallback_chain_allowed(routable, kind, episode, {}) is False + assert loop_transport.fallback_chain_allowed(routable, kind, episode, {}) is True assert loop_transport.fallback_chain_allowed(routable, kind, None, {TRANSPORT_DEATHS_KEY: record}) is False rejoinable = {"_pending_transport_outcome": {"same_operation_recoverable": True}} assert loop_transport.fallback_chain_allowed(routable, "provider_outcome_unknown", None, rejoinable) is False From 73f314d488d0dad06400c5c1137f1613290ff5ab Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:45:53 +0300 Subject: [PATCH 03/12] test: align resource refusal expectations with visible recovery --- tests/test_config_extraction.py | 1 + tests/test_delegated_subagent_transport.py | 28 ++++++++++------------ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/test_config_extraction.py b/tests/test_config_extraction.py index 3bbe591a6..559b27e5f 100644 --- a/tests/test_config_extraction.py +++ b/tests/test_config_extraction.py @@ -55,6 +55,7 @@ _ADDED_OWNERS = { "normalize_model_role_options": model_slots, "model_role_option": model_slots, "task_model_binding": model_slots, + "route_binding": model_slots, "apply_model_role_override": model_slots, "CLAUDEXOR_MODEL_POLL_INTERVAL_SEC": runtime_limits, "CLAUDEXOR_OPERATOR_STOP_TIMEOUT_SEC": runtime_limits, diff --git a/tests/test_delegated_subagent_transport.py b/tests/test_delegated_subagent_transport.py index 2d8f49525..b58acb789 100644 --- a/tests/test_delegated_subagent_transport.py +++ b/tests/test_delegated_subagent_transport.py @@ -295,15 +295,15 @@ def test_the_transport_error_code_is_the_failure_class_name(): assert cx.ClaudexorSubscriptionWindowExhausted("x").code == SUBSCRIPTION_WINDOW_EXHAUSTED -def test_the_window_class_is_transient_and_scheduled_by_its_reset(): +def test_the_window_class_keeps_reset_as_fact_without_sleeping_the_call(): exc = cx.ClaudexorSubscriptionWindowExhausted("spent", reset_at="2030-01-01T00:00:00Z") classification = classify_llm_exception(exc) assert classification.kind == SUBSCRIPTION_WINDOW_EXHAUSTED assert classification.kind != "quota_exhausted" - assert classification.retry_same_request is True - # Scheduled by the reset instant, never by the 60s-capped exponential backoff. - assert classification.retry_after_sec is not None - assert classification.retry_after_sec > 60.0 + # Configured alternatives and the caller-owned visible wait handle access. + # A dated reset does not grant this call a blind sleep or another dispatch. + assert classification.retry_same_request is False + assert classification.retry_after_sec is None assert classification.reset_at == "2030-01-01T00:00:00Z" @@ -318,11 +318,10 @@ def _control_problem(code: str, context: dict, status: int = 409) -> cx.Claudexo return excinfo.value -def test_a_pool_heals_on_a_timer_only_when_the_engine_dated_it(): - """Owner decision: a credential pool exhausted for a STRUCTURAL reason (every enabled - account refused the model, every account disabled: the engine names no reset) must not - masquerade as a quota timer. Both producer seams (run failure, ControlProblem) key on - the structured `resetsAt` through one helper; nothing reads the prose.""" +def test_a_dated_pool_refusal_keeps_its_cause_without_inventing_quota_or_retry(): + """The existing transport wrapper may carry a dated pool refusal, but its + own code still distinguishes unavailable routes from a spent quota window. + A reset is observation, never proof of recovery or permission to resend a run.""" reset = "2030-01-01T00:00:00Z" dated_failure = {"code": "credential_pool_exhausted", "resetsAt": reset, "safeMessage": "every account is cooling down"} @@ -332,13 +331,12 @@ def test_a_pool_heals_on_a_timer_only_when_the_engine_dated_it(): assert (dated.code, dated.reset_at) == ("credential_pool_exhausted", reset) dated_run = cx.run_failure_error("run-1", "failed", dated_failure) assert dated_run.reported_cause == "every account is cooling down" - # A dated pool heals on the same timer as a spent window, so it is SCHEDULED - # against its reset (not the short backoff) and keeps its own code as evidence. + # Dated or not, a pool refusal keeps its own cause. Nothing schedules a new + # delegated run or sleeps this classifier's caller until the reported reset. dated_class = classify_llm_exception(dated_run) - assert (dated_class.kind, dated_class.retry_same_request) == (SUBSCRIPTION_WINDOW_EXHAUSTED, True) + assert (dated_class.kind, dated_class.retry_same_request) == ("provider_error", False) assert dated_class.provider_code == "credential_pool_exhausted" - assert dated_class.reset_at == reset and dated_class.retry_after_sec is not None - assert dated_class.retry_after_sec > 60.0 + assert dated_class.reset_at == reset and dated_class.retry_after_sec is None # Other direction: an absent, empty or null reset is structural: a plain refusal # under the SAME code, the engine's words still carried beside it. From e1e09a173a73ca6408851c6a68652215259e9dc2 Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:13:29 +0300 Subject: [PATCH 04/12] fix: honor primary wait on temporary provider refusals --- docs/architecture/06-agent-core.md | 2 +- ouroboros/loop_model_call.py | 3 ++- ouroboros/model_wait.py | 2 +- ...test_native_fallback_binding_acceptance.py | 22 +++++++++++++++++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/architecture/06-agent-core.md b/docs/architecture/06-agent-core.md index 34841c769..fbff914b2 100644 --- a/docs/architecture/06-agent-core.md +++ b/docs/architecture/06-agent-core.md @@ -289,7 +289,7 @@ Host-authored sampling defaults stay separate from explicit parameters: review r #### Configured routes and primary return -Route identity (`model_slots.route_binding`) includes model, locality and configured/owner-selected account or Auto; equal model ids can name different alternatives. The turn's primary binding (task override or role slot, role, locality; `loop._initial_round_route`) stays an alternative when an adopted fallback fails (`loop_model_call._route_candidates`), without duplicating Main in Settings; routes provide cognition until a model can decide. After a host-driven switch away from the primary the acting model gets one `[ROUTE FACTS]` notice (both bindings, the dated failure, any engine reset) and chooses stay, `switch_model(primary="return")`, or `primary="wait"`. The latter retains resource/transport waits instead of paid alternatives. Direct API refusals use Retry/Switch/Stop without subscription observation, paid probes or automatic-access claims. Return restores the primary role/fit and effort intent, retaining Auto; the next real request tests it. No timer, reset or metadata proves generation success (BIBLE P5/P13). +Route identity (`model_slots.route_binding`) includes model, locality and configured/owner-selected account or Auto; equal model ids can name different alternatives. The turn's primary binding (task override or role slot, role, locality; `loop._initial_round_route`) stays an alternative when an adopted fallback fails (`loop_model_call._route_candidates`), without duplicating Main in Settings; routes provide cognition until a model can decide. After a host-driven switch away from the primary the acting model gets one `[ROUTE FACTS]` notice (both bindings, the dated failure, any engine reset) and chooses stay, `switch_model(primary="return")`, or `primary="wait"`. The latter keeps resource/transport refusals (including 429/503) on the primary. Direct API refusals use Retry/Switch/Stop without subscription observation, paid probes or automatic-access claims. Return restores the primary role/fit and effort intent, retaining Auto; the next real request tests it. No timer, reset or metadata proves generation success (BIBLE P5/P13). #### Quota and auth waits diff --git a/ouroboros/loop_model_call.py b/ouroboros/loop_model_call.py index a2407e6ac..0fcf1ece6 100644 --- a/ouroboros/loop_model_call.py +++ b/ouroboros/loop_model_call.py @@ -861,7 +861,8 @@ def _dispatch_round_model( deferral = None if waiter is not None: from ouroboros.loop_llm_call import classify_llm_exception - resource_reason = (lambda error: {"auth_error": "auth", "quota_exhausted": "quota"}.get( + resource_reason = (lambda error: {"auth_error": "auth", "quota_exhausted": "quota", + "provider_transient": "unavailable", "rate_limit": "unavailable"}.get( classify_llm_exception(error).kind, "")) if getattr(ctx.tools._ctx, "route_wait_on_primary", False) else None binding.enter_context(waiter.register_reprepare( role, lambda kwargs: _reprepare_waiting_main(ctx, kwargs), resource_reason=resource_reason)) diff --git a/ouroboros/model_wait.py b/ouroboros/model_wait.py index bad5c07b5..53fa69564 100644 --- a/ouroboros/model_wait.py +++ b/ouroboros/model_wait.py @@ -69,7 +69,7 @@ def model_wait_reason(error: Exception) -> str: proves none of those facts and keeps its ordinary error path. """ authored = getattr(error, "task_resource_wait_reason", "") - if authored in {"auth", "quota"}: + if authored in {"auth", "quota", "unavailable"}: return authored # confirmed by the registered caller's refusal classifier code = getattr(error, "code", "") if code in {"auth_required", "subscription_window_exhausted"}: diff --git a/tests/test_native_fallback_binding_acceptance.py b/tests/test_native_fallback_binding_acceptance.py index cca530d6a..cb2d450e5 100644 --- a/tests/test_native_fallback_binding_acceptance.py +++ b/tests/test_native_fallback_binding_acceptance.py @@ -268,8 +268,9 @@ def test_duplicate_model_slots_keep_their_distinct_native_accounts(main_call, mo assert [row["state"] for row in ledger(ctx.drive_root)].count("unresolved") == 2 -@pytest.mark.parametrize("status,reason", [(401, "auth"), (402, "quota")]) -def test_primary_wait_honors_a_direct_api_access_refusal(main_call, monkeypatch, status, reason): +@pytest.mark.parametrize("status,reason", [(401, "auth"), (402, "quota"), (429, "unavailable"), (503, "unavailable"), (400, "")]) +@pytest.mark.parametrize("wire", ["http", "sse"]) +def test_primary_wait_honors_a_direct_api_access_refusal(main_call, monkeypatch, status, reason, wire): """A declared wait must not be undone by a paid fallback merely because access uses an API key.""" from ouroboros import usage_accounting as ua from ouroboros.llm_attempt import _attempt_request, _candidate_before_dispatch @@ -284,14 +285,20 @@ def test_primary_wait_honors_a_direct_api_access_refusal(main_call, monkeypatch, ctx.tools._ctx.model_wait_context = owner _switch_model(ctx.tools._ctx, primary="wait") sent = [] + original_send = ctx.llm._chat_remote def send(target, messages, schemas, *args, **kwargs): + if target["provider"] != "openai": + return original_send(target, messages, schemas, *args, **kwargs) assert target["provider"] == "openai" sent.append(target["provider"]) body = {"messages": deepcopy(messages), "tools": schemas or [], "model": "primary-test"} request = _attempt_request(target, body) def refused(): + if wire == "sse": + from ouroboros.llm_stream import ChatAccumulator + ChatAccumulator().accept("message", json.dumps({"error": {"code": status, "message": "Access refused"}})) raise httpx.HTTPStatusError("Access refused", request=httpx.Request("POST", "https://fixture.invalid"), response=httpx.Response(status)) @@ -300,6 +307,17 @@ def test_primary_wait_honors_a_direct_api_access_refusal(main_call, monkeypatch, monkeypatch.setattr(ctx.llm, "_chat_remote", send) monkeypatch.setattr(ctx.llm, "claudexor_model_sources", lambda: pytest.fail("API wait has no subscription observer")) owner.owner_control = lambda: "cancelled" if any(row.get("state") == "waiting" for row in owner.waits.values()) else None + if not reason: # Invalid request is not unavailable access: ordinary fallback still works. + from ouroboros.loop_model_call import _recover_failed_round + assert loop._call_round_model(ctx)[0] is None + gateway.results = [_reply(ROUTE_B, {"role": "assistant", "content": "Alternative accepted the request."})] + gateway.dispatch = ["response_received"] + fallback_cooldown.reset_for_tests() + reply, *_ = _recover_failed_round(ctx, ctx.tools, None, None, context_fit_plan=ctx.context_fit_plan, + active_context_mode="max", emit_progress=lambda *_a, **_k: None) + assert reply["content"] == "Alternative accepted the request." + assert not owner.waits and sent == ["openai"] and len(gateway.creates) == 1 + return with pytest.raises(ModelWaitInterrupted, match="cancelled"): loop._call_round_model(ctx) waits = [row for row in list(events.queue) if row.get("type") == "task_model_wait"] From 10e7a5c810e109508d03cebcbc0fec79c4f800a0 Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:25:40 +0300 Subject: [PATCH 05/12] fix: pace primary refusal waits without catalog-driven resend --- docs/architecture/06-agent-core.md | 2 +- ouroboros/loop.py | 17 ++-- ouroboros/loop_model_call.py | 3 +- ouroboros/loop_transport.py | 63 ++++++++---- ouroboros/model_wait.py | 2 +- ...test_native_fallback_binding_acceptance.py | 95 ++++++++++++++++++- tests/test_unknown_fallback_first.py | 21 ++++ 7 files changed, 170 insertions(+), 33 deletions(-) diff --git a/docs/architecture/06-agent-core.md b/docs/architecture/06-agent-core.md index fbff914b2..2084d003a 100644 --- a/docs/architecture/06-agent-core.md +++ b/docs/architecture/06-agent-core.md @@ -289,7 +289,7 @@ Host-authored sampling defaults stay separate from explicit parameters: review r #### Configured routes and primary return -Route identity (`model_slots.route_binding`) includes model, locality and configured/owner-selected account or Auto; equal model ids can name different alternatives. The turn's primary binding (task override or role slot, role, locality; `loop._initial_round_route`) stays an alternative when an adopted fallback fails (`loop_model_call._route_candidates`), without duplicating Main in Settings; routes provide cognition until a model can decide. After a host-driven switch away from the primary the acting model gets one `[ROUTE FACTS]` notice (both bindings, the dated failure, any engine reset) and chooses stay, `switch_model(primary="return")`, or `primary="wait"`. The latter keeps resource/transport refusals (including 429/503) on the primary. Direct API refusals use Retry/Switch/Stop without subscription observation, paid probes or automatic-access claims. Return restores the primary role/fit and effort intent, retaining Auto; the next real request tests it. No timer, reset or metadata proves generation success (BIBLE P5/P13). +Route identity (`model_slots.route_binding`) includes model, locality and the configured/owner-selected account or Auto. The task primary (`loop._initial_round_route`) stays an alternative when an adopted fallback fails (`loop_model_call._route_candidates`), without duplicating Main in Settings. After fallback, `[ROUTE FACTS]` names both bindings, dated failure and any reset; cognition chooses stay, `switch_model(primary="return")`, or `primary="wait"`. Return restores primary role/fit and effort intent; Auto stays Auto. Waiting preserves resource controls; temporary 429/503 refusals exhaust the ordinary retry burst, then use `loop_transport`'s paced episode on that primary, with its existing Stop/deadline/budget bounds and original cost records. Account catalog readiness cannot end this refusal wait: the next real answer does. Direct API auth/billing waits offer Retry/Switch/Stop without a subscription observer. No timer, reset or metadata proves generation success (BIBLE P5/P13). #### Quota and auth waits diff --git a/ouroboros/loop.py b/ouroboros/loop.py index 7ac8f2c2b..d3c715238 100644 --- a/ouroboros/loop.py +++ b/ouroboros/loop.py @@ -63,6 +63,7 @@ from ouroboros.loop_transport import ( continue_unknown_transport as _continue_unknown_transport, TransportWaitEpisode, # noqa: F401 -- the loop module keeps its historical import surface for the L-B leaves end_episode_budget as _end_episode_budget, # noqa: F401 -- the loop module keeps its historical import surface for the L-B leaves + PRIMARY_REFUSAL_KINDS, fallback_chain_allowed as _fallback_chain_allowed, # noqa: F401 -- resolved by loop_model_call._recover_failed_round at call time finalize_now_transport_terminal as _finalize_now_transport_terminal, # noqa: F401 -- the loop module keeps its historical import surface for the L-B leaves last_assistant_text as _last_assistant_text, @@ -200,16 +201,10 @@ def _provider_unavailable_result( # round record (a granted transport-death repeat, no usable response since) leaves an attempt # unresolved and outranks the wait terminal, which in turn outranks the overflow salvage. record = isinstance(ctx.accumulated_usage.get(TRANSPORT_DEATHS_KEY), dict) - # The UNKNOWN-OUTCOME predicate, spelled exactly as the two seams that already - # own it: `provider_no_call_source` (the no-resend decision) and - # `provider_terminal_fallback_text` (the owner sentence). The durable source - # asked only for the round record, so an episode whose attempt was interrupted - # in flight — no record, sticky kind `provider_outcome_unknown` — told the owner - # its outcome was unknown while stamping `transport_unavailable_no_resend` on the - # trace (#869). One question, one answer, on all three surfaces. + # Pending custody outranks later failures; its unknown cost survives every wait. unknown_outcome = record or bool(ctx.accumulated_usage.get("_pending_transport_outcome")) or str( ctx.accumulated_usage.get("_last_llm_error_kind") or "") == "provider_outcome_unknown" - is_transport_wait = wait_cause == "transport_unavailable" + is_transport_wait = wait_cause == "transport_unavailable" or wait_cause in PRIMARY_REFUSAL_KINDS is_context_overflow = (kind == "context_overflow" and not (record or is_transport_wait) and not ctx.accumulated_usage.get("resource_refusal")) is_deadline_exhausted = kind == "deadline_exhausted" or str(ctx.accumulated_usage.get("_last_llm_error_kind") or "") == "deadline_exhausted" @@ -223,7 +218,7 @@ def _provider_unavailable_result( usage["terminal_provider_notice"] = _provider_terminal_fallback_text( usage, is_context_overflow=is_context_overflow, is_transport_wait=is_transport_wait, waited_sec=waited_sec, interactive=interactive, - is_deadline_exhausted=is_deadline_exhausted, control_reason=control_reason, + is_deadline_exhausted=is_deadline_exhausted, control_reason=control_reason, wait_cause=wait_cause, ) return text, usage, trace @@ -243,7 +238,7 @@ def _provider_unavailable_result( is_transport_wait=is_transport_wait, waited_sec=waited_sec, interactive=interactive, is_deadline_exhausted=is_deadline_exhausted, - control_reason=control_reason, + control_reason=control_reason, wait_cause=wait_cause, ) if is_context_overflow: text, usage, llm_trace = _forced_fallback_result( @@ -261,7 +256,7 @@ def _provider_unavailable_result( ctx.accumulated_usage.update(execution_status=RESULT_INFRA_FAILED, reason_code="provider_unavailable") text, usage, llm_trace = _forced_fallback_result( ctx, llm_trace, fallback, reason_code="provider_unavailable", - source="provider_outcome_unknown_no_resend" if unknown_outcome else "transport_unavailable_no_resend", + source="provider_outcome_unknown_no_resend" if unknown_outcome else f"{wait_cause}_no_resend", ) if usage.get("reason_code") == "provider_unavailable": usage["execution_status"] = RESULT_INFRA_FAILED diff --git a/ouroboros/loop_model_call.py b/ouroboros/loop_model_call.py index 0fcf1ece6..a2407e6ac 100644 --- a/ouroboros/loop_model_call.py +++ b/ouroboros/loop_model_call.py @@ -861,8 +861,7 @@ def _dispatch_round_model( deferral = None if waiter is not None: from ouroboros.loop_llm_call import classify_llm_exception - resource_reason = (lambda error: {"auth_error": "auth", "quota_exhausted": "quota", - "provider_transient": "unavailable", "rate_limit": "unavailable"}.get( + resource_reason = (lambda error: {"auth_error": "auth", "quota_exhausted": "quota"}.get( classify_llm_exception(error).kind, "")) if getattr(ctx.tools._ctx, "route_wait_on_primary", False) else None binding.enter_context(waiter.register_reprepare( role, lambda kwargs: _reprepare_waiting_main(ctx, kwargs), resource_reason=resource_reason)) diff --git a/ouroboros/loop_transport.py b/ouroboros/loop_transport.py index f8cc610b1..314dde743 100644 --- a/ouroboros/loop_transport.py +++ b/ouroboros/loop_transport.py @@ -73,7 +73,7 @@ _FINAL_REDIAL_MARGIN_SEC = 3.0 @dataclass class TransportWaitEpisode: - """Episode-local latch for one remote pre-dispatch transport outage. + """One remote outage or chosen wait for temporary primary-provider refusals. The latch — not the mutable ``_last_llm_error_kind`` projection — carries the terminal cause: later failures (a failed local fallback pass, the deadline @@ -240,6 +240,15 @@ def continue_unknown_transport(episode: TransportWaitEpisode, *, llm: Any, tools return True +PRIMARY_REFUSAL_KINDS = frozenset({"provider_transient", "rate_limit"}) + + +def primary_refusal_wait(ctx: Any, error_kind: str) -> bool: + """A declared primary wait keeps temporary refusals on ordinary paced recovery.""" + return bool(getattr(ctx, "route_wait_on_primary", False) and error_kind in PRIMARY_REFUSAL_KINDS + and not inline_presence(ctx) and not getattr(ctx, "exact_model_route", False)) + + def fallback_chain_allowed( ctx: Any, last_error_kind: str, episode: Optional[TransportWaitEpisode], accumulated_usage: Optional[Dict[str, Any]] = None, @@ -259,6 +268,8 @@ def fallback_chain_allowed( # repeat with no usable response since): no paid candidate may dial over # it, whatever the last kind says. return False + if primary_refusal_wait(ctx, last_error_kind): + return False if getattr(ctx, "route_wait_on_primary", False) and ( last_error_kind in ("transport_unavailable", "provider_outcome_unknown") or usage.get("_last_llm_resource_refusal")): @@ -281,6 +292,8 @@ def reconcile_transport_wait( ) -> Optional[TransportWaitEpisode]: """Reconcile the episode latch with one dispatch outcome. + A declared primary wait also admits temporary provider refusals after ordinary retries, + without treating a ready account catalog as proof of generation recovery. Enters a new episode on a fresh ``transport_unavailable`` failure, or an eligible unknown outcome (``new_generation_after_unknown``), that the configured routes did not answer (durable ``entered`` event; the first owner @@ -314,6 +327,12 @@ def reconcile_transport_wait( """ pending = dict((getattr(ctx, "_accumulated_usage", {}) or {}).get("_pending_transport_outcome") or {}) unknown_again = error_kind == "provider_outcome_unknown" and new_generation_after_unknown(ctx) + refused = not msg_present and primary_refusal_wait(ctx, error_kind) + if refused and episode is not None: + # Catalog readiness is not generation recovery. Keep the existing episode's + # pacing, elapsed bound and prior unknown custody; only a real answer ends it. + episode.wait_cause, episode.continuation_granted = error_kind, False + return episode if episode is not None and episode.continuation_granted: if (msg_present or error_kind not in ("provider_outcome_unknown", "transport_unavailable") or (error_kind == "provider_outcome_unknown" and pending and not unknown_again)): @@ -371,11 +390,11 @@ def reconcile_transport_wait( return episode if episode is None: unknown = unknown_again - if msg_present or (error_kind != "transport_unavailable" and not unknown): + if msg_present or (error_kind != "transport_unavailable" and not unknown and not refused): return None interactive = bool(getattr(ctx, "is_direct_chat", False)) episode = TransportWaitEpisode( - wait_cause="provider_outcome_unknown" if unknown else "transport_unavailable", + wait_cause=error_kind, outcome_custody=dict((getattr(ctx, "_accumulated_usage", {}) or {}).get("_pending_transport_outcome") or {}), started_monotonic=time.monotonic(), interactive=interactive, @@ -383,13 +402,15 @@ def reconcile_transport_wait( ) emit_network_wait_event( drive_logs, task_id=task_id, phase="entered", - elapsed_sec=0.0, redials=0, model=model, + elapsed_sec=0.0, redials=0, model=model, detail=error_kind, ) episode.last_note_monotonic = time.monotonic() # Interactive notes keep their existing wording; direct-turn Stop is # separately handled through its typed mailbox control. emit_progress( - ("🌐 Provider connection was lost after dispatch. The outcome and any unreported cost remain unknown. " + ("🌐 The primary provider temporarily refused the request. Waiting as chosen, then continuing " + "on the same primary with paced requests; prior attempt costs remain recorded." if refused else + "🌐 Provider connection was lost after dispatch. The outcome and any unreported cost remain unknown. " "Waiting for connectivity, then continuing from saved work with a new attempt; another charge is possible." if unknown else "🌐 Could not establish a provider connection — waiting and " "redialing automatically (failed attempts are $0).") @@ -404,7 +425,9 @@ def reconcile_transport_wait( redials=episode.redials, model=model, ) emit_progress( - f"🌐 Provider connection restored after {elapsed / 60.0:.1f} min — resuming.", + (f"🌐 Primary provider answered after {elapsed / 60.0:.1f} min — resuming." + if episode.wait_cause in PRIMARY_REFUSAL_KINDS else + f"🌐 Provider connection restored after {elapsed / 60.0:.1f} min — resuming."), incident=None, ) return None @@ -607,7 +630,8 @@ def transport_wait_step( ) if episode.interactive: emit_progress( - "🌐 Stopped waiting for a provider connection after " + ("🌐 Stopped waiting for the primary provider after " if episode.wait_cause in PRIMARY_REFUSAL_KINDS + else "🌐 Stopped waiting for a provider connection after ") + f"{elapsed / 60.0:.1f} min — this turn ends as a provider outage.", incident=None, ) @@ -644,7 +668,8 @@ def transport_wait_step( if time.monotonic() - episode.last_note_monotonic >= note_interval: episode.last_note_monotonic = time.monotonic() emit_progress( - f"🌐 Still waiting for a provider connection — {elapsed / 60.0:.0f} min " + ("🌐 Still waiting for the primary provider — " if episode.wait_cause in PRIMARY_REFUSAL_KINDS + else "🌐 Still waiting for a provider connection — ") + f"{elapsed / 60.0:.0f} min " f"elapsed, {episode.redials} redials; will resume automatically.", incident=None, # a periodic note is never a toast; the episode always passes incident= ) @@ -680,12 +705,10 @@ def finalize_now_transport_terminal( """Route a finalize_now that lands during an active episode to the honest transport no-resend terminal. - Every finalize_now flavor (supervisor deadline, cost ceiling, owner stop) - normally dispatches one forced summarize call — but over a proven-dead - egress that paid path can only fail at $0 with identical salvage, so the - deterministic no-resend terminal wins. The episode's durable evidence is - closed with an ``ended`` row first; the caller passes a partial of its - ``_handle_provider_unavailable`` so terminal composition stays in loop.py. + Supervisor deadline, cost ceiling and owner stop send no forced summary + over a waited-out outage or chosen primary refusal wait. Preserve the actual + cause and prior attempt costs. Close the durable episode first; the caller's + ``_handle_provider_unavailable`` keeps terminal composition in loop.py. """ emit_network_wait_event( drive_logs, task_id=task_id, phase="ended", @@ -693,7 +716,7 @@ def finalize_now_transport_terminal( redials=episode.redials, model=model, detail="finalize_now", ) return handle_provider_unavailable( - error_kind="transport_unavailable", + error_kind=episode.wait_cause, wait_cause=episode.wait_cause, waited_sec=episode.waited_sec, interactive=episode.interactive, @@ -779,7 +802,7 @@ def provider_terminal_fallback_text( waited_sec: float, interactive: bool = False, is_deadline_exhausted: bool, - control_reason: str = "", + control_reason: str = "", wait_cause: str = "", ) -> str: """Owner-facing terminal text when provider death left nothing to salvage. @@ -798,13 +821,19 @@ def provider_terminal_fallback_text( from supervisor.owner_stop import REASON_OWNER_STOPPED_DIRECT_TURN unknown = (isinstance(accumulated_usage.get(TRANSPORT_DEATHS_KEY), dict) + or bool(accumulated_usage.get("_pending_transport_outcome")) or accumulated_usage.get("_last_llm_error_kind") == "provider_outcome_unknown") if control_reason in {REASON_OWNER_REQUESTED_FINALIZATION, REASON_OWNER_STOPPED_DIRECT_TURN}: action = "Stop" if control_reason == REASON_OWNER_STOPPED_DIRECT_TURN else "Wrap up" waited = f" The wait ended after {waited_sec / 60.0:.1f} min;" if waited_sec else "" - return (f"⚠️ The owner requested {action} while the provider connection was unavailable." + subject = "the primary provider was refusing requests" if wait_cause in PRIMARY_REFUSAL_KINDS else "the provider connection was unavailable" + return (f"⚠️ The owner requested {action} while {subject}." f"{waited} No new summary request was sent. Any files written so far are preserved." + (provider_recovery_hint(accumulated_usage) if unknown else "")) + if wait_cause in PRIMARY_REFUSAL_KINDS and not unknown: + return ("⚠️ The primary provider kept refusing requests; its selected wait ended " + f"after {waited_sec / 60.0:.1f} min. No new summary request was sent. " + "Completed work and the original attempt-cost records are preserved.") if is_context_overflow: return ( "⚠️ The context exceeded the selected model window; no further provider call was made. " diff --git a/ouroboros/model_wait.py b/ouroboros/model_wait.py index 53fa69564..bad5c07b5 100644 --- a/ouroboros/model_wait.py +++ b/ouroboros/model_wait.py @@ -69,7 +69,7 @@ def model_wait_reason(error: Exception) -> str: proves none of those facts and keeps its ordinary error path. """ authored = getattr(error, "task_resource_wait_reason", "") - if authored in {"auth", "quota", "unavailable"}: + if authored in {"auth", "quota"}: return authored # confirmed by the registered caller's refusal classifier code = getattr(error, "code", "") if code in {"auth_required", "subscription_window_exhausted"}: diff --git a/tests/test_native_fallback_binding_acceptance.py b/tests/test_native_fallback_binding_acceptance.py index cb2d450e5..26fa56ba8 100644 --- a/tests/test_native_fallback_binding_acceptance.py +++ b/tests/test_native_fallback_binding_acceptance.py @@ -268,7 +268,7 @@ def test_duplicate_model_slots_keep_their_distinct_native_accounts(main_call, mo assert [row["state"] for row in ledger(ctx.drive_root)].count("unresolved") == 2 -@pytest.mark.parametrize("status,reason", [(401, "auth"), (402, "quota"), (429, "unavailable"), (503, "unavailable"), (400, "")]) +@pytest.mark.parametrize("status,reason", [(401, "auth"), (402, "quota"), (400, "")]) @pytest.mark.parametrize("wire", ["http", "sse"]) def test_primary_wait_honors_a_direct_api_access_refusal(main_call, monkeypatch, status, reason, wire): """A declared wait must not be undone by a paid fallback merely because access uses an API key.""" @@ -401,3 +401,96 @@ def test_primary_choice_survives_shared_cold_continuation(main_call, monkeypatch assert route_binding(model, local, plan.model_role, overrides=owner.overrides) == (MODEL, False, primary_account) assert inner.primary_route == {"model": MODEL, "use_local": False, "role": "main"} assert inner._route_facts_pending == "dated refusal fact" and effort == "high" + + +@pytest.mark.parametrize("wire", ["managed", "http", "sse"]) +@pytest.mark.parametrize("status", [429, 503, 529]) +@pytest.mark.parametrize("ending", ["answer", "stop", "deadline"]) +def test_primary_refusal_wait_paces_real_requests_without_catalog_recovery( + main_call, monkeypatch, wire, status, ending): + from datetime import datetime, timedelta, timezone + from ouroboros import loop_llm_call, owner_mailbox, usage_accounting as ua + from ouroboros.llm_attempt import _attempt_request, _candidate_before_dispatch + from supervisor.owner_stop import REASON_OWNER_STOPPED_DIRECT_TURN + + ctx, gateway, owner, events, _decide, _observations = main_call + primary = MODEL if wire == "managed" else "openai::primary-test" + for key, value in {"OUROBOROS_CONTEXT_MODE": "max", "OUROBOROS_TASK_REVIEW_MODE": "off", + "OUROBOROS_SAFETY_MODE": "off", "MCP_ENABLED": "false", + "OUROBOROS_TRANSIENT_RETRY_MAX": "6", "OUROBOROS_MODEL_FALLBACKS": MODEL, + "OPENAI_API_KEY": "fixture-key"}.items(): + monkeypatch.setenv(key, value) + monkeypatch.setenv(MODEL_ACCOUNTS_KEY, json.dumps({"main": "account-a", "fallback": ["account-b"]})) + ctx.context_fit_plan = replace(ctx.context_fit_plan, model=primary, + provider="claudexor" if wire == "managed" else "openai", model_route=ROUTE if wire == "managed" else {}) + tools = _loop_tools(ctx, owner) + tools._ctx.task_model_override, tools._ctx.is_direct_chat = primary, True + tools._ctx.model_wait_context = owner + owner.worker_slot_held = False + wait_call = {"id": "wait-primary", "type": "function", "function": { + "name": "switch_model", "arguments": json.dumps({"primary": "wait"})}} + first = {"role": "assistant", "content": "", "tool_calls": [wait_call]} + final = {"role": "assistant", "content": "The primary answered."} + failures = 12 + gateway.results = [_reply(ROUTE, first)] + [result(outcome="failed", problem={ + "code": "provider_failed", "message": "Upstream refused", "retryable": True, + "context": {"httpStatus": status}}) for _ in range(failures)] + [_reply(ROUTE, final)] + gateway.dispatch = ["response_received"] * (failures + 2) + _bounded_creates(monkeypatch, gateway, failures + 2) + sent = [] + if wire != "managed": + def send(target, messages, schemas, *args, **kwargs): + assert target["provider"] == "openai", "a chosen primary wait must not dispatch a fallback" + sent.append(target["provider"]) + body = {"messages": deepcopy(messages), "tools": schemas or [], "model": "primary-test"} + request = _attempt_request(target, body) + def physical(): + if len(sent) in (1, failures + 2): + return (first if len(sent) == 1 else final), {"prompt_tokens": 1, "completion_tokens": 1, "cost": 0.01, "cost_final": True} + if wire == "sse": + from ouroboros.llm_stream import ChatAccumulator + ChatAccumulator().accept("message", json.dumps({"error": {"code": status, "message": "Temporary refusal"}})) + raise httpx.HTTPStatusError("Temporary refusal", request=httpx.Request("POST", "https://fixture.invalid"), + response=httpx.Response(status)) + return ua.execute_physical_attempt(request, physical, + extractor=lambda value: (value[1], 0.01, True), before_dispatch=_candidate_before_dispatch(body, request)) + monkeypatch.setattr(ctx.llm, "_chat_remote", send) + def no_probe(*_a, **_kw): + pytest.fail("an available catalog does not prove generation recovery") + monkeypatch.setattr(ctx.llm, "claudexor_model_catalog", no_probe) + monkeypatch.setattr(ctx.llm, "claudexor_model_sources", no_probe) + bursts, waits, notes = [], [], [] + monkeypatch.setattr(loop_llm_call, "_sleep_within_deadline", lambda seconds, *_a, **_k: bursts.append(seconds) or True) + def pause(seconds, _wake): + waits.append(seconds) + if ending == "stop": + owner_mailbox.write_owner_message(ctx.drive_root, REASON_OWNER_STOPPED_DIRECT_TURN, "task-one", + msg_id="stop-primary-wait", kind=owner_mailbox.KIND_FINALIZE_NOW) + elif ending == "deadline": + tools._ctx.task_metadata["deadline_at"] = (datetime.now(timezone.utc) - timedelta(seconds=1)).isoformat() + return ending != "answer" + monkeypatch.setattr(loop_transport, "interruptible_wait_sleep", pause) + text, usage, _trace = loop.run_llm_loop( + ctx.messages, tools, ctx.llm, ctx.drive_logs, lambda text, **_kw: notes.append(text), queue.Queue(), + task_id="task-one", drive_root=ctx.drive_root, event_queue=events) + attempts = len(gateway.creates) if wire == "managed" else len(sent) + assert attempts == (failures + 2 if ending == "answer" else 7) + assert waits == ([4.0, 8.0] if ending == "answer" else [4.0]) + assert bursts == [4.0, 8.0, 16.0, 32.0, 60.0] * (2 if ending == "answer" else 1) + assert not owner.waits # no immediate resource_available row/new paid call loop + if wire == "managed": + assert all(payload["account"] == {"mode": "pin", "profileId": "account-a"} for payload, _ in gateway.uploads) + rows = ledger(ctx.drive_root) + assert sum(row["state"] == "dispatched" for row in rows) == attempts + assert not any(row["state"] == "released" for row in rows) + assert any("primary provider temporarily refused" in note for note in notes) + assert not any("$0" in note or "connection restored" in note.lower() for note in notes) + if ending == "answer": + assert text == final["content"] + else: + notice = usage.get("terminal_provider_notice", text) + if ending == "stop": + assert "owner stopped this chat turn" in notice + else: + assert "No new summary request was sent" in notice + assert "provider connection was unavailable" not in notice and "$0" not in notice diff --git a/tests/test_unknown_fallback_first.py b/tests/test_unknown_fallback_first.py index 366015cb5..d4b28b0dc 100644 --- a/tests/test_unknown_fallback_first.py +++ b/tests/test_unknown_fallback_first.py @@ -443,3 +443,24 @@ def test_a_readable_unknown_after_a_granted_continuation_ends_the_episode(tmp_pa readable = SimpleNamespace(task_id="t", _accumulated_usage={"_pending_transport_outcome": { "physical_attempt_id": "new", "outcome": "unknown", "same_operation_recoverable": True}}) assert loop_transport.reconcile_transport_wait(granted(), readable, **kwargs) is None + + +@pytest.mark.parametrize("kind", ["provider_transient", "rate_limit"]) +def test_chosen_primary_refusal_keeps_existing_episode_bounds_and_unknown_custody(tmp_path, kind): + ctx = SimpleNamespace(task_id="t", route_wait_on_primary=True, is_direct_chat=True, _accumulated_usage={}) + assert not loop_transport.fallback_chain_allowed(ctx, kind, None, {}) + episode = loop_transport.TransportWaitEpisode(wait_cause="provider_outcome_unknown", started_monotonic=12, + wait_iterations=3, redials=3, continuation_granted=True, outcome_custody={"physical_attempt_id": "old"}) + kwargs = dict(drive_logs=tmp_path, task_id="t", model="primary", emit_progress=lambda *_a, **_kw: None) + resumed = loop_transport.reconcile_transport_wait(episode, ctx, msg_present=False, error_kind=kind, **kwargs) + assert resumed is episode and episode.wait_cause == kind and not episode.continuation_granted + assert (episode.started_monotonic, episode.wait_iterations, episode.redials) == (12, 3, 3) + assert episode.outcome_custody == {"physical_attempt_id": "old"} + assert loop_transport.reconcile_transport_wait(episode, ctx, msg_present=True, error_kind="", **kwargs) is None + ctx.route_wait_on_primary = False + assert loop_transport.fallback_chain_allowed(ctx, kind, None, {}) + assert loop_transport.reconcile_transport_wait(None, ctx, msg_present=False, error_kind=kind, **kwargs) is None + ctx.route_wait_on_primary, ctx.current_task_type = True, "presence" + assert not loop_transport.primary_refusal_wait(ctx, kind) + ctx.current_task_type, ctx.exact_model_route = "task", True + assert not loop_transport.primary_refusal_wait(ctx, kind) From ffa72a214a87d97dc2978ad56e6602624047e7e2 Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:26:26 +0300 Subject: [PATCH 06/12] test: stabilize shared unread checks and finish full browser CI --- .github/workflows/ui-browser.yml | 10 ++++----- .../test_project_unread_read_band_browser.py | 21 +++++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ui-browser.yml b/.github/workflows/ui-browser.yml index 32cee7399..fb5a30633 100644 --- a/.github/workflows/ui-browser.yml +++ b/.github/workflows/ui-browser.yml @@ -20,10 +20,10 @@ jobs: ui-smoke: name: ${{ inputs.diagnostic == 'full' && 'ui-smoke' || format('PARTIAL DIAGNOSTIC UI - {0}', inputs.diagnostic) }} runs-on: ubuntu-latest - # A bound, because ci.yml's release chain needs this job: the complete lane - # ran in about 51 minutes on a workstation, and a hung engine must end as a - # red run rather than an open one. - timeout-minutes: 120 + # The complete 513-case lane reached 98% at the former two-hour limit on + # GitHub's runner. Keep a finite whole-job ceiling with room for the tail + # and evidence upload; per-test assertions and collection guards remain. + timeout-minutes: 180 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: @@ -66,7 +66,7 @@ jobs: OUROBOROS_EXPECT_BROWSER_ENGINES: chromium,webkit PLAYWRIGHT_BROWSERS_PATH: ${{ runner.temp }}/ui-browsers run: >- - python -I -S scripts/safe_test.py -- python -m pytest tests/ -m ui_browser --require-ui-browser -q --tb=short + python -I -S scripts/safe_test.py -- python -m pytest tests/ -m ui_browser --require-ui-browser -v --tb=short --junitxml="${{ runner.temp }}/ci-private/ui/host.xml" --ci-evidence-dir="${{ runner.temp }}/ci-evidence/ui/host" - name: Run PARTIAL DIAGNOSTIC host UI scenario diff --git a/tests/test_project_unread_read_band_browser.py b/tests/test_project_unread_read_band_browser.py index 96b6359cf..9ba53874d 100644 --- a/tests/test_project_unread_read_band_browser.py +++ b/tests/test_project_unread_read_band_browser.py @@ -355,6 +355,22 @@ def test_a_room_read_on_one_client_clears_the_dot_on_another_at_its_next_state_r browser = getattr(pw, engine).launch() try: other, other_row = _client(browser, direct_server_with_data["url"], "shared-room", other_acks) + # Hold this client's shared-cursor responses until the reader has + # acknowledged. Existing background refreshes may otherwise win + # the race with the assertion below on a slower browser. + other.evaluate("""() => { + const fetch = window.fetch.bind(window); + window.__holdPreferences = true; + window.__pendingPreferences = []; + window.fetch = async (input, init) => { + const url = new URL(typeof input === 'string' ? input : input.url, location.href); + if (url.pathname === '/api/ui/preferences' && (!init?.method || init.method === 'GET') + && window.__holdPreferences) { + await new Promise(resolve => window.__pendingPreferences.push(resolve)); + } + return fetch(input, init); + }; + }""") reader, row = _client(browser, direct_server_with_data["url"], "shared-room", reader_acks) row.evaluate("el => el.click()") reader.locator(PANEL).locator(".chat-bubble").filter(has_text="Reply 3").wait_for(state="attached") @@ -364,6 +380,11 @@ def test_a_room_read_on_one_client_clears_the_dot_on_another_at_its_next_state_r row.locator(".nav-unread-dot").wait_for(state="detached") assert other_row.locator(".nav-unread-dot").count() == 1, "the other client has not refreshed yet" _emit_ws_frame(other, {"type": "projects_changed"}) + other.wait_for_function("() => window.__pendingPreferences.length > 0") + other.evaluate("""() => { + window.__holdPreferences = false; + window.__pendingPreferences.splice(0).forEach(resolve => resolve()); + }""") other_row.locator(".nav-unread-dot").wait_for(state="detached") other.screenshot(path=str(evidence / f"read-band-{engine}-other-client.png")) assert [ack for ack in other_acks if ack.get("project_seen_revision")] == [], \ From 9f4c9eb9ddd9de17caf42e8463b8a2293dfd55cd Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:44:48 +0300 Subject: [PATCH 07/12] docs: refresh generated fallback recovery facade inventory --- docs/inventories/FACADE_INVENTORY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/inventories/FACADE_INVENTORY.md b/docs/inventories/FACADE_INVENTORY.md index e5ab1c8a5..314419d4c 100644 --- a/docs/inventories/FACADE_INVENTORY.md +++ b/docs/inventories/FACADE_INVENTORY.md @@ -2,7 +2,7 @@ AST-derived inventory of compatibility facades, regenerated by `python scripts/regenerate_inventories.py`. Do not edit. A facade row is any runtime module whose top-level `from import ...` statements carry the `noqa: F401` re-export marker — the codebase's declared "this binding exists for its binding, not for this module's own use" convention (reference FACADE_CONSUMERS method). Leaf domains come from `ouroboros/domains.toml`; a leaf outside the facade's domain is marked ✗ (that edge also appears in the manifest's pinned direction matrix). `tests/test_generated_inventories.py` pins byte-identity, so any re-export surface change must regenerate this file. -- facade modules: **62**; marked re-export bindings: **2349**; cross-domain facade→leaf pairs: **131** +- facade modules: **62**; marked re-export bindings: **2350**; cross-domain facade→leaf pairs: **131** | facade | domain | bindings | leaves | |---|---|---:|---| @@ -19,7 +19,7 @@ AST-derived inventory of compatibility facades, regenerated by `python scripts/r | `ouroboros/gateway/tasks.py` | D11 | 7 | `ouroboros/gateway/cost_breakdown.py` (1)
`ouroboros/gateway/task_decision.py` (1)
`ouroboros/gateway/task_events.py` (4)
`ouroboros/gateway/task_hurry.py` (1) | | `ouroboros/headless.py` | D17 | 45 | `ouroboros/contracts/task_constraint.py` (1 ✗D19)
`ouroboros/headless_status.py` (11)
`ouroboros/workspace_patch_capture.py` (20)
`ouroboros/workspace_patch_rules.py` (13) | | `ouroboros/llm.py` | D02 | 92 | `ouroboros/anthropic_native_custody.py` (7)
`ouroboros/context_budget.py` (2 ✗D03)
`ouroboros/llm_anthropic.py` (2)
`ouroboros/llm_attempt.py` (22)
`ouroboros/llm_capability_policy.py` (5)
`ouroboros/llm_fallback.py` (1)
`ouroboros/llm_gigachat.py` (1)
`ouroboros/llm_local.py` (7)
`ouroboros/llm_messages.py` (1)
`ouroboros/llm_openai_compatible.py` (5)
`ouroboros/llm_pricing.py` (4)
`ouroboros/llm_routing.py` (3)
`ouroboros/openrouter_attribution.py` (1)
`ouroboros/provider_models.py` (7)
`ouroboros/request_wire_recovery.py` (7)
`ouroboros/transport_custody.py` (1)
`ouroboros/usage_accounting.py` (14 ✗D16)
`ouroboros/utils.py` (2 ✗D18) | -| `ouroboros/loop.py` | D01 | 263 | `ouroboros/config.py` (6 ✗D12)
`ouroboros/context.py` (1 ✗D03)
`ouroboros/context_budget.py` (1 ✗D03)
`ouroboros/context_compaction.py` (2 ✗D03)
`ouroboros/context_fit.py` (2 ✗D03)
`ouroboros/deadline_utils.py` (2)
`ouroboros/llm.py` (3 ✗D02)
`ouroboros/loop_acceptance.py` (21)
`ouroboros/loop_acceptance_review.py` (29)
`ouroboros/loop_budget.py` (14)
`ouroboros/loop_delivery.py` (29)
`ouroboros/loop_forced_finalization.py` (22)
`ouroboros/loop_llm_call.py` (6)
`ouroboros/loop_messages.py` (11)
`ouroboros/loop_model_call.py` (22)
`ouroboros/loop_nudges.py` (16)
`ouroboros/loop_round_limits.py` (17)
`ouroboros/loop_tool_execution.py` (5)
`ouroboros/loop_transport.py` (10)
`ouroboros/nanny_pacing.py` (4 ✗D07)
`ouroboros/observability.py` (1 ✗D16)
`ouroboros/outcomes.py` (17)
`ouroboros/pricing.py` (1 ✗D02)
`ouroboros/review_cycles.py` (1 ✗D06)
`ouroboros/task_finalization.py` (3)
`ouroboros/task_pacing.py` (1)
`ouroboros/tool_policy.py` (4 ✗D04)
`ouroboros/usage_accounting.py` (5 ✗D16)
`ouroboros/utils.py` (3 ✗D18)
`supervisor/owner_stop.py` (4 ✗D09) | +| `ouroboros/loop.py` | D01 | 264 | `ouroboros/config.py` (6 ✗D12)
`ouroboros/context.py` (1 ✗D03)
`ouroboros/context_budget.py` (1 ✗D03)
`ouroboros/context_compaction.py` (2 ✗D03)
`ouroboros/context_fit.py` (2 ✗D03)
`ouroboros/deadline_utils.py` (2)
`ouroboros/llm.py` (3 ✗D02)
`ouroboros/loop_acceptance.py` (21)
`ouroboros/loop_acceptance_review.py` (29)
`ouroboros/loop_budget.py` (14)
`ouroboros/loop_delivery.py` (29)
`ouroboros/loop_forced_finalization.py` (22)
`ouroboros/loop_llm_call.py` (6)
`ouroboros/loop_messages.py` (11)
`ouroboros/loop_model_call.py` (22)
`ouroboros/loop_nudges.py` (16)
`ouroboros/loop_round_limits.py` (17)
`ouroboros/loop_tool_execution.py` (5)
`ouroboros/loop_transport.py` (11)
`ouroboros/nanny_pacing.py` (4 ✗D07)
`ouroboros/observability.py` (1 ✗D16)
`ouroboros/outcomes.py` (17)
`ouroboros/pricing.py` (1 ✗D02)
`ouroboros/review_cycles.py` (1 ✗D06)
`ouroboros/task_finalization.py` (3)
`ouroboros/task_pacing.py` (1)
`ouroboros/tool_policy.py` (4 ✗D04)
`ouroboros/usage_accounting.py` (5 ✗D16)
`ouroboros/utils.py` (3 ✗D18)
`supervisor/owner_stop.py` (4 ✗D09) | | `ouroboros/loop_acceptance.py` | D01 | 5 | `ouroboros/loop_messages.py` (5) | | `ouroboros/loop_acceptance_review.py` | D01 | 3 | `ouroboros/acceptance_retrieving.py` (3) | | `ouroboros/loop_llm_call.py` | D01 | 2 | `ouroboros/config.py` (2 ✗D12) | From f133c7bf65ba0c668e0dce4d2085d934fd4aef0c Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:51:20 +0300 Subject: [PATCH 08/12] fix: preserve existing transport episode event details --- ouroboros/loop_transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ouroboros/loop_transport.py b/ouroboros/loop_transport.py index 314dde743..db56d8045 100644 --- a/ouroboros/loop_transport.py +++ b/ouroboros/loop_transport.py @@ -402,7 +402,7 @@ def reconcile_transport_wait( ) emit_network_wait_event( drive_logs, task_id=task_id, phase="entered", - elapsed_sec=0.0, redials=0, model=model, detail=error_kind, + elapsed_sec=0.0, redials=0, model=model, detail=error_kind if refused else "", ) episode.last_note_monotonic = time.monotonic() # Interactive notes keep their existing wording; direct-turn Stop is From 4e44be6dce767674c62a03a67f82b7293816bb9b Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:58:35 +0300 Subject: [PATCH 09/12] test: cover queued primary wait finalization and document its cause --- docs/architecture/06-agent-core.md | 4 +-- ...test_native_fallback_binding_acceptance.py | 29 ++++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/architecture/06-agent-core.md b/docs/architecture/06-agent-core.md index 2084d003a..7ac34bb91 100644 --- a/docs/architecture/06-agent-core.md +++ b/docs/architecture/06-agent-core.md @@ -20,7 +20,7 @@ The delivery-control protocol is resolved here and only here (DEVELOPMENT keeps The child-absorption gate is an action gate: while undispositioned direct children remain, the loop HOLDS the candidate (`child_absorption_or_revision_required`) instead of arming the JSON-only control instruction, so the model never receives two contradictory instructions in one round. A typed keep cannot close the gate; after the one bounded reminder it forces the best-effort `children_unabsorbed` rail with a current `id [status] sha256` listing. The absorption digest's `## child` header carries the child's typed custody debt (`delegated_runs_unreconciled`, bounded, with a `get_task_result` pointer) as visibility only — the parent's authority over that patch is exactly the orphan rule, and a child's debt never relabels the root card (DESIGN §4). Finalizing over an UNDISPOSED OWN delegated patch is deliberately NOT gated: the consequence is disclosed where the decision is made (the `integrate_delegated_patch` schema, the apply receipts) and lands as the additive Done-with-warnings custody overlay rather than a hold; a pre-finalization reminder and propagation of child custody debt into the acceptance-subtree snapshot remain disclosed deferred gaps. -Provider death is the one forced rail that is NOT best-effort completion: it salvages the best text but stamps `infra_failed`, so the task ends `failed` with the typed `provider_unavailable` reason and an immediate "provider outage — NOT completed" owner notification; a waited-out transport outage reaches it through the deterministic no-resend branch (`transport_unavailable_no_resend`, or `provider_outcome_unknown_no_resend` while the round holds a transport-death repeat record). Its one forced model call runs only while a call can land: a spent same-model retry wall stamps `_llm_retry_wall_exhausted`, a no-call gate beside `context_overflow` and `provider_outcome_unknown`, so no second retry window is paid over a proven-dead provider (disclosed residual: a last-invocation bool on the shared usage dict). Terminal delivery preserves producer authorship: `terminal_provider_notice` is host presentation beside the raw result, quoting only `_last_llm_provider_message`, which `loop_transport.stamp_owner_provider_message` resets per failed call to authored provider text or nothing (a Claudexor engine error authors none: its typed provider fields reset alongside in `_last_llm_provider_fields`; else words for `_last_llm_error_kind`; repr stays in `_last_llm_error`/`llm_api_error`); receipts and secondary System incidents carry wait duration, finalization cause and unknown-outcome warning and never recommend a blind rerun. Every forced rail stamps one closed-vocabulary producer word at the single forced-finalization sink: `model_final` for complete model text (host-authored notices stay separate), `host_notice` for host-written terminal text (an owner System row, never automatic external Presence speech), and `host_salvage` only on the provider-death rail, where managed/direct delivery replaces the text with the enriched outage receipt and the full bytes stay in task details. Missing origin stays unknown; `agent.py` resets it on host text replacement. Presence synthesis seals the recorded adapter body and internal result separately (chapter 12), never a delivery receipt. +Provider death is the one forced rail that is NOT best-effort completion: it salvages the best text but stamps `infra_failed`, so the task ends `failed` with the typed `provider_unavailable` reason and an immediate "provider outage — NOT completed" owner notification; wait expiry takes a no-resend source: `transport_unavailable_no_resend`, `provider_outcome_unknown_no_resend`, or the chosen primary refusal's `provider_transient_no_resend`/`rate_limit_no_resend`. Its one forced model call runs only while a call can land: a spent same-model retry wall stamps `_llm_retry_wall_exhausted`, a no-call gate beside `context_overflow` and `provider_outcome_unknown`, so no second retry window is paid over a proven-dead provider (disclosed residual: a last-invocation bool on the shared usage dict). Terminal delivery preserves producer authorship: `terminal_provider_notice` is host presentation beside the raw result, quoting only `_last_llm_provider_message`, which `loop_transport.stamp_owner_provider_message` resets per failed call to authored provider text or nothing (a Claudexor engine error authors none: its typed provider fields reset alongside in `_last_llm_provider_fields`; else words for `_last_llm_error_kind`; repr stays in `_last_llm_error`/`llm_api_error`); receipts and secondary System incidents carry wait duration, finalization cause and unknown-outcome warning and never recommend a blind rerun. Every forced rail stamps one closed-vocabulary producer word at the single forced-finalization sink: `model_final` for complete model text (host-authored notices stay separate), `host_notice` for host-written terminal text (an owner System row, never automatic external Presence speech), and `host_salvage` only on the provider-death rail, where managed/direct delivery replaces the text with the enriched outage receipt and the full bytes stay in task details. Missing origin stays unknown; `agent.py` resets it on host text replacement. Presence synthesis seals the recorded adapter body and internal result separately (chapter 12), never a delivery receipt. Finalization controls are typed owner-mailbox entries, not injected owner prose. The supervisor may request one bounded tool-less answer, salvage the last persisted assistant text, and retain a full canonical copy when a preview would truncate it. A grace episode has one durable control and can be revoked atomically when the task itself resumes; descendant activity does not count as the task's own progress. An unkillable process remains visibly running; custody checks prevent another runtime instance from reaping work it does not own. @@ -173,7 +173,7 @@ Retry rails nest, each with its own owner and bound; the table exists so the mul | transport-death repeat | `call_llm_with_retry`, inline Presence primary round only | ≤2 per round, 4 s then 8 s deadline-aware backoff | a dispatched request that died with a typed transport death, as a new attempt | | configured-route walk | `loop_model_call._run_cross_model_fallback_chain` | bounded attempts per binding per walk; retried with existing wait pacing | the round on each other configured binding, the primary first when a fallback acts | | transient burst | `call_llm_with_retry` | `OUROBOROS_TRANSIENT_RETRY_MAX` attempts per call (the outer ceiling of the same loop) | typed 408/429/5xx and empty/incomplete responses | -| round redial | `loop_transport.py` wait episode | a managed task: owner deadline, budget, Stop, absolute ceiling; an interactive turn: the task idle timeout measured from episode entry (`OUROBOROS_TASK_IDLE_TIMEOUT_SEC`), or an explicit deadline when that is shorter | a released ($0) pre-dispatch failure of the same round, or an eligible unknown as a marked new attempt | +| round redial | `loop_transport.py` wait episode | a managed task: owner deadline, budget, Stop, absolute ceiling; an interactive turn: the task idle timeout measured from episode entry (`OUROBOROS_TASK_IDLE_TIMEOUT_SEC`), or an explicit deadline when that is shorter | the same round: released ($0), eligible unknown (new attempt), or chosen primary refusal (ordinary burst) | | served-model redo | `llm_substitution.py` inside one `chat_claudexor` | `OUROBOROS_SERVED_MODEL_REDOS` per call, and none under a pin, an admitted candidate, a spent send budget or a spent owner window | a round the engine says another model answered, as a new operation. It nests inside the rails above, so a repeat granted by one of them starts a call whose redo budget begins again | | review physical rail | `review_substrate.py`, `review_native_episode.py` | 2 sends per packet or session actor on the P3/acceptance surfaces; a native-retrieval slot carries no send count (its bounds are the episode's own, enumerated below); no rail elsewhere | a released review send, never an unknown one | diff --git a/tests/test_native_fallback_binding_acceptance.py b/tests/test_native_fallback_binding_acceptance.py index 26fa56ba8..13b11a4e3 100644 --- a/tests/test_native_fallback_binding_acceptance.py +++ b/tests/test_native_fallback_binding_acceptance.py @@ -403,9 +403,10 @@ def test_primary_choice_survives_shared_cold_continuation(main_call, monkeypatch assert inner._route_facts_pending == "dated refusal fact" and effort == "high" -@pytest.mark.parametrize("wire", ["managed", "http", "sse"]) -@pytest.mark.parametrize("status", [429, 503, 529]) -@pytest.mark.parametrize("ending", ["answer", "stop", "deadline"]) +@pytest.mark.parametrize("wire,status,ending", [ + (wire, status, ending) for wire in ("managed", "http", "sse") + for status in (429, 503, 529) for ending in ("answer", "stop", "deadline") +] + [("managed", 503, "wrap")]) def test_primary_refusal_wait_paces_real_requests_without_catalog_recovery( main_call, monkeypatch, wire, status, ending): from datetime import datetime, timedelta, timezone @@ -424,9 +425,9 @@ def test_primary_refusal_wait_paces_real_requests_without_catalog_recovery( ctx.context_fit_plan = replace(ctx.context_fit_plan, model=primary, provider="claudexor" if wire == "managed" else "openai", model_route=ROUTE if wire == "managed" else {}) tools = _loop_tools(ctx, owner) - tools._ctx.task_model_override, tools._ctx.is_direct_chat = primary, True + tools._ctx.task_model_override, tools._ctx.is_direct_chat = primary, ending != "wrap" tools._ctx.model_wait_context = owner - owner.worker_slot_held = False + owner.worker_slot_held = ending == "wrap" wait_call = {"id": "wait-primary", "type": "function", "function": { "name": "switch_model", "arguments": json.dumps({"primary": "wait"})}} first = {"role": "assistant", "content": "", "tool_calls": [wait_call]} @@ -463,9 +464,18 @@ def test_primary_refusal_wait_paces_real_requests_without_catalog_recovery( monkeypatch.setattr(loop_llm_call, "_sleep_within_deadline", lambda seconds, *_a, **_k: bursts.append(seconds) or True) def pause(seconds, _wake): waits.append(seconds) - if ending == "stop": - owner_mailbox.write_owner_message(ctx.drive_root, REASON_OWNER_STOPPED_DIRECT_TURN, "task-one", - msg_id="stop-primary-wait", kind=owner_mailbox.KIND_FINALIZE_NOW) + if ending in {"stop", "wrap"}: + from ouroboros.outcomes import REASON_OWNER_REQUESTED_FINALIZATION + reason = REASON_OWNER_STOPPED_DIRECT_TURN if ending == "stop" else REASON_OWNER_REQUESTED_FINALIZATION + msg_id = "stop-primary-wait" + if ending == "wrap": + from ouroboros import cancel_intents + from supervisor.owner_stop import owner_stop_control_id + intent = cancel_intents.request_cancel(ctx.drive_root, "task-one", + requested_stop_policy=cancel_intents.STOP_POLICY_FINALIZE) + msg_id = owner_stop_control_id(intent) + owner_mailbox.write_owner_message(ctx.drive_root, reason, "task-one", + msg_id=msg_id, kind=owner_mailbox.KIND_FINALIZE_NOW) elif ending == "deadline": tools._ctx.task_metadata["deadline_at"] = (datetime.now(timezone.utc) - timedelta(seconds=1)).isoformat() return ending != "answer" @@ -493,4 +503,7 @@ def test_primary_refusal_wait_paces_real_requests_without_catalog_recovery( assert "owner stopped this chat turn" in notice else: assert "No new summary request was sent" in notice + if ending == "wrap": + assert "owner requested Wrap up while the primary provider was refusing requests" in notice + assert any(note.endswith("Stop cancels.") for note in notes) assert "provider connection was unavailable" not in notice and "$0" not in notice From 5569faca53a9692e3a58564e3df4665204f2c5f4 Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:01:06 +0300 Subject: [PATCH 10/12] test: bind initial cursor reads and effective CI verbosity --- .github/workflows/ui-browser.yml | 2 +- tests/test_project_unread_read_band_browser.py | 12 ++++++------ tests/test_system_e2e_ci_lane.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ui-browser.yml b/.github/workflows/ui-browser.yml index fb5a30633..66e72f879 100644 --- a/.github/workflows/ui-browser.yml +++ b/.github/workflows/ui-browser.yml @@ -66,7 +66,7 @@ jobs: OUROBOROS_EXPECT_BROWSER_ENGINES: chromium,webkit PLAYWRIGHT_BROWSERS_PATH: ${{ runner.temp }}/ui-browsers run: >- - python -I -S scripts/safe_test.py -- python -m pytest tests/ -m ui_browser --require-ui-browser -v --tb=short + python -I -S scripts/safe_test.py -- python -m pytest tests/ -m ui_browser --require-ui-browser -vv --tb=short --junitxml="${{ runner.temp }}/ci-private/ui/host.xml" --ci-evidence-dir="${{ runner.temp }}/ci-evidence/ui/host" - name: Run PARTIAL DIAGNOSTIC host UI scenario diff --git a/tests/test_project_unread_read_band_browser.py b/tests/test_project_unread_read_band_browser.py index 9ba53874d..67cd0ff87 100644 --- a/tests/test_project_unread_read_band_browser.py +++ b/tests/test_project_unread_read_band_browser.py @@ -354,11 +354,9 @@ def test_a_room_read_on_one_client_clears_the_dot_on_another_at_its_next_state_r with sync_playwright() as pw: browser = getattr(pw, engine).launch() try: - other, other_row = _client(browser, direct_server_with_data["url"], "shared-room", other_acks) - # Hold this client's shared-cursor responses until the reader has - # acknowledged. Existing background refreshes may otherwise win - # the race with the assertion below on a slower browser. - other.evaluate("""() => { + # Install before navigation so initial and background cursor reads + # both wait for the reader's ACK, regardless of browser speed. + hold_preferences = """() => { const fetch = window.fetch.bind(window); window.__holdPreferences = true; window.__pendingPreferences = []; @@ -370,7 +368,9 @@ def test_a_room_read_on_one_client_clears_the_dot_on_another_at_its_next_state_r } return fetch(input, init); }; - }""") + }""" + other, other_row = _client(browser, direct_server_with_data["url"], "shared-room", other_acks, + init=(hold_preferences,)) reader, row = _client(browser, direct_server_with_data["url"], "shared-room", reader_acks) row.evaluate("el => el.click()") reader.locator(PANEL).locator(".chat-bubble").filter(has_text="Reply 3").wait_for(state="attached") diff --git a/tests/test_system_e2e_ci_lane.py b/tests/test_system_e2e_ci_lane.py index 4bcf15a53..0736c2d49 100644 --- a/tests/test_system_e2e_ci_lane.py +++ b/tests/test_system_e2e_ci_lane.py @@ -61,7 +61,7 @@ def test_pull_requests_and_ouroboros_pushes_share_one_full_browser_lane(): steps = {step.get("name"): step for step in shared["jobs"]["ui-smoke"]["steps"] if step.get("name")} full = steps["Run complete host UI lane with collection and availability guards"] assert full["if"] == "${{ !cancelled() && steps.install_browsers.outcome == 'success' && inputs.diagnostic == 'full' }}" - assert "python -m pytest tests/ -m ui_browser --require-ui-browser -q --tb=short" in full["run"] + assert "python -m pytest tests/ -m ui_browser --require-ui-browser -vv --tb=short" in full["run"] assert full["env"]["OUROBOROS_RUN_UI_SMOKE"] == "1" assert full["env"]["OUROBOROS_EXPECT_BROWSER_ENGINES"] == "chromium,webkit" assert steps["Run browser tools Chromium/WebKit smoke"]["if"] == ( From 7e787588e8d5b0c3e5e48ae61abe604069b454b6 Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:09:39 +0300 Subject: [PATCH 11/12] docs: keep integrated recovery description within chapter budget --- docs/architecture/06-agent-core.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/06-agent-core.md b/docs/architecture/06-agent-core.md index 7ac34bb91..8d3f81f63 100644 --- a/docs/architecture/06-agent-core.md +++ b/docs/architecture/06-agent-core.md @@ -289,7 +289,7 @@ Host-authored sampling defaults stay separate from explicit parameters: review r #### Configured routes and primary return -Route identity (`model_slots.route_binding`) includes model, locality and the configured/owner-selected account or Auto. The task primary (`loop._initial_round_route`) stays an alternative when an adopted fallback fails (`loop_model_call._route_candidates`), without duplicating Main in Settings. After fallback, `[ROUTE FACTS]` names both bindings, dated failure and any reset; cognition chooses stay, `switch_model(primary="return")`, or `primary="wait"`. Return restores primary role/fit and effort intent; Auto stays Auto. Waiting preserves resource controls; temporary 429/503 refusals exhaust the ordinary retry burst, then use `loop_transport`'s paced episode on that primary, with its existing Stop/deadline/budget bounds and original cost records. Account catalog readiness cannot end this refusal wait: the next real answer does. Direct API auth/billing waits offer Retry/Switch/Stop without a subscription observer. No timer, reset or metadata proves generation success (BIBLE P5/P13). +`model_slots.route_binding` identifies model, locality and configured/owner-selected account or Auto. The primary (`loop._initial_round_route`) remains an alternative if fallback fails; no duplicate Main setting is needed. On fallback, `[ROUTE FACTS]` names both routes, dated failure and reset; cognition chooses stay, `switch_model(primary="return")`, or `primary="wait"`. Return keeps primary role/fit, effort and Auto. Waiting keeps resource controls; temporary 429/503 refusals exhaust ordinary retries, then use `loop_transport`'s paced episode on that primary under existing Stop/deadline/budget bounds, preserving cost records. A real answer, never catalog readiness, ends this refusal wait. Direct API auth/billing waits offer Retry/Switch/Stop without a subscription observer. No timer, reset or metadata proves generation success (BIBLE P5/P13). #### Quota and auth waits From ead87c217760d804509d800afa8b531995b89384 Mon Sep 17 00:00:00 2001 From: Ouroboros <311266734+ouroboros-agent@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:22:56 +0300 Subject: [PATCH 12/12] test: settle startup reads before shared cursor refresh --- tests/test_project_unread_read_band_browser.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_project_unread_read_band_browser.py b/tests/test_project_unread_read_band_browser.py index 67cd0ff87..94a51d4ca 100644 --- a/tests/test_project_unread_read_band_browser.py +++ b/tests/test_project_unread_read_band_browser.py @@ -354,23 +354,33 @@ def test_a_room_read_on_one_client_clears_the_dot_on_another_at_its_next_state_r with sync_playwright() as pw: browser = getattr(pw, engine).launch() try: - # Install before navigation so initial and background cursor reads - # both wait for the reader's ACK, regardless of browser speed. + # Control cursor reads from navigation onward, then settle startup + # reads before the ACK so only a later refresh can clear the dot. hold_preferences = """() => { const fetch = window.fetch.bind(window); window.__holdPreferences = true; window.__pendingPreferences = []; + window.__settledPreferences = 0; window.fetch = async (input, init) => { const url = new URL(typeof input === 'string' ? input : input.url, location.href); if (url.pathname === '/api/ui/preferences' && (!init?.method || init.method === 'GET') && window.__holdPreferences) { await new Promise(resolve => window.__pendingPreferences.push(resolve)); + try { return await fetch(input, init); } finally { window.__settledPreferences += 1; } } return fetch(input, init); }; }""" other, other_row = _client(browser, direct_server_with_data["url"], "shared-room", other_acks, init=(hold_preferences,)) + # Keep the barrier armed after draining the initial page-load reads. + released = other.evaluate("""() => { + const held = window.__pendingPreferences.splice(0); + held.forEach(resolve => resolve()); + return held.length; + }""") + assert released >= 1, "the initial preferences read must settle before the room is read" + other.wait_for_function("n => window.__settledPreferences >= n", arg=released) reader, row = _client(browser, direct_server_with_data["url"], "shared-room", reader_acks) row.evaluate("el => el.click()") reader.locator(PANEL).locator(".chat-bubble").filter(has_text="Reply 3").wait_for(state="attached")