mirror of
https://github.com/razzant/ouroboros.git
synced 2026-10-21 14:12:28 +00:00
Merge pull request #1414 from razzant/fix/1372-effort-contract
Some checks failed
CI / quick-test (push) Waiting to run
CI / benchmark-methodology (push) Waiting to run
CI / full-test (push) Waiting to run
CI / betterleaks-platform-smoke (macos-latest) (push) Waiting to run
CI / betterleaks-platform-smoke (ubuntu-latest) (push) Waiting to run
CI / betterleaks-platform-smoke (windows-latest) (push) Waiting to run
CI / Android emulator smoke (API 30) (push) Blocked by required conditions
CI / Android emulator smoke (API 33) (push) Blocked by required conditions
CI / Android emulator smoke (API 36) (push) Blocked by required conditions
CI / android-build (push) Blocked by required conditions
CI / release-preflight (push) Blocked by required conditions
CI / build (dmg, macos-latest, macos-arm64, syft_1.50.0_darwin_arm64.tar.gz, syft, e32fdb9d47823fa633748a1efca2528fd77c37469ea93c9e40ab835da44e4cce) (push) Blocked by required conditions
CI / build (tar.gz, ubuntu-latest, linux-x86_64, syft_1.50.0_linux_amd64.tar.gz, syft, bf7b29ff57f06da30918266a0e1c2885a8f99784798d1bdb1628886aa015d788) (push) Blocked by required conditions
CI / build (zip, windows-latest, windows-x64, syft_1.50.0_windows_amd64.zip, syft.exe, 815ee6973ec5dff6a671d7f41b0e78835a8c45b91d5a39f4743ea1cee833d3be) (push) Blocked by required conditions
CI / vendor-package-smoke (push) Blocked by required conditions
CI / release (push) Blocked by required conditions
CI / integration-test (push) Waiting to run
CI / skill-smoke (macos-latest) (push) Waiting to run
CI / skill-smoke (ubuntu-latest) (push) Waiting to run
CI / skill-smoke (windows-latest) (push) Waiting to run
CI / marker-guards (push) Waiting to run
CI / ui-smoke (push) Waiting to run
CI / docker-ui-smoke (push) Waiting to run
CI / docker-portable-test (push) Waiting to run
CI / system-e2e-mock (push) Waiting to run
CI / e2e-live (SM1 x1 — largest subset feasible under the $30 cap) (push) Waiting to run
CI / android-test (push) Waiting to run
CI / Android emulator smoke (API 26) (push) Blocked by required conditions
CI / Android emulator smoke (API 29) (push) Blocked by required conditions
Claudexor platform gate (API keys — subscription auth NOT covered) / toolchain · macos-latest · real managed Node/npm, no ambient Node, no harness install (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / fixture · macos-latest · exact managed runtime, fake harness, no model (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / fixture · ubuntu-latest · exact managed runtime, fake harness, no model (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / fixture · windows-latest · exact managed runtime, fake harness, no model (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / toolchain · ubuntu-latest · real managed Node/npm, no ambient Node, no harness install (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / toolchain · windows-latest · real managed Node/npm, no ambient Node, no harness install (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / consumer · windows-latest · real pinned Codex local install + doctor, no login, no model task (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / live · macos-latest · claude · API key only, subscription NOT covered (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / live · ubuntu-latest · claude · API key only, subscription NOT covered (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / live · windows-latest · claude · API key only, subscription NOT covered (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / live · macos-latest · codex · API key only, subscription NOT covered (push) Waiting to run
UI browser (ouroboros push) / ui-smoke (push) Waiting to run
Submit Claudexor dependency / Submit managed runtime snapshot (push) Has been cancelled
Some checks failed
CI / quick-test (push) Waiting to run
CI / benchmark-methodology (push) Waiting to run
CI / full-test (push) Waiting to run
CI / betterleaks-platform-smoke (macos-latest) (push) Waiting to run
CI / betterleaks-platform-smoke (ubuntu-latest) (push) Waiting to run
CI / betterleaks-platform-smoke (windows-latest) (push) Waiting to run
CI / Android emulator smoke (API 30) (push) Blocked by required conditions
CI / Android emulator smoke (API 33) (push) Blocked by required conditions
CI / Android emulator smoke (API 36) (push) Blocked by required conditions
CI / android-build (push) Blocked by required conditions
CI / release-preflight (push) Blocked by required conditions
CI / build (dmg, macos-latest, macos-arm64, syft_1.50.0_darwin_arm64.tar.gz, syft, e32fdb9d47823fa633748a1efca2528fd77c37469ea93c9e40ab835da44e4cce) (push) Blocked by required conditions
CI / build (tar.gz, ubuntu-latest, linux-x86_64, syft_1.50.0_linux_amd64.tar.gz, syft, bf7b29ff57f06da30918266a0e1c2885a8f99784798d1bdb1628886aa015d788) (push) Blocked by required conditions
CI / build (zip, windows-latest, windows-x64, syft_1.50.0_windows_amd64.zip, syft.exe, 815ee6973ec5dff6a671d7f41b0e78835a8c45b91d5a39f4743ea1cee833d3be) (push) Blocked by required conditions
CI / vendor-package-smoke (push) Blocked by required conditions
CI / release (push) Blocked by required conditions
CI / integration-test (push) Waiting to run
CI / skill-smoke (macos-latest) (push) Waiting to run
CI / skill-smoke (ubuntu-latest) (push) Waiting to run
CI / skill-smoke (windows-latest) (push) Waiting to run
CI / marker-guards (push) Waiting to run
CI / ui-smoke (push) Waiting to run
CI / docker-ui-smoke (push) Waiting to run
CI / docker-portable-test (push) Waiting to run
CI / system-e2e-mock (push) Waiting to run
CI / e2e-live (SM1 x1 — largest subset feasible under the $30 cap) (push) Waiting to run
CI / android-test (push) Waiting to run
CI / Android emulator smoke (API 26) (push) Blocked by required conditions
CI / Android emulator smoke (API 29) (push) Blocked by required conditions
Claudexor platform gate (API keys — subscription auth NOT covered) / toolchain · macos-latest · real managed Node/npm, no ambient Node, no harness install (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / fixture · macos-latest · exact managed runtime, fake harness, no model (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / fixture · ubuntu-latest · exact managed runtime, fake harness, no model (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / fixture · windows-latest · exact managed runtime, fake harness, no model (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / toolchain · ubuntu-latest · real managed Node/npm, no ambient Node, no harness install (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / toolchain · windows-latest · real managed Node/npm, no ambient Node, no harness install (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / consumer · windows-latest · real pinned Codex local install + doctor, no login, no model task (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / live · macos-latest · claude · API key only, subscription NOT covered (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / live · ubuntu-latest · claude · API key only, subscription NOT covered (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / live · windows-latest · claude · API key only, subscription NOT covered (push) Waiting to run
Claudexor platform gate (API keys — subscription auth NOT covered) / live · macos-latest · codex · API key only, subscription NOT covered (push) Waiting to run
UI browser (ouroboros push) / ui-smoke (push) Waiting to run
Submit Claudexor dependency / Submit managed runtime snapshot (push) Has been cancelled
fix: bind adaptive effort recovery to published Claudexor 3.17.1
This commit is contained in:
commit
f84d5830ac
73 changed files with 2664 additions and 368 deletions
|
|
@ -14,7 +14,7 @@ The manifest is the SSOT of the module→domain assignment (1:1, complete over t
|
|||
| D04 | Tool execution: registry, access & typed results | 21 | 0 |
|
||||
| D05 | Tool surfaces: files, code, shell, media, external | 29 | 0 |
|
||||
| D06 | Review stack | 71 | 0 |
|
||||
| D07 | Delegation, subagents & Claudexor | 57 | 0 |
|
||||
| D07 | Delegation, subagents & Claudexor | 58 | 0 |
|
||||
| D08 | Supervisor: queue, workers, events & runtime control | 50 | 0 |
|
||||
| D09 | Cancellation, owner control & process custody | 14 | 0 |
|
||||
| D10 | Git, update & release machinery | 29 | 0 |
|
||||
|
|
@ -28,7 +28,7 @@ The manifest is the SSOT of the module→domain assignment (1:1, complete over t
|
|||
| D18 | Launcher, packaging, platform & shared substrate | 15 | 0 |
|
||||
| D19 | Frozen contracts (ABI) | 10 | 0 |
|
||||
| D20 | Presence | 10 | 0 |
|
||||
| **total** | | **597** | **0** |
|
||||
| **total** | | **598** | **0** |
|
||||
|
||||
## Dependency direction matrix (strict, pinned)
|
||||
|
||||
|
|
@ -447,6 +447,7 @@ No function body (≥ 10 normalized lines) is shared verbatim across domains. Ne
|
|||
- `ouroboros/delegate_target_drift.py`
|
||||
- `ouroboros/delegate_terminal.py`
|
||||
- `ouroboros/depth_evidence.py`
|
||||
- `ouroboros/effort_evidence.py`
|
||||
- `ouroboros/gateways/__init__.py`
|
||||
- `ouroboros/gateways/claudexor.py`
|
||||
- `ouroboros/gateways/claudexor_run_events.py`
|
||||
|
|
|
|||
|
|
@ -175,10 +175,11 @@ server.py (Starlette+uvicorn) ← HTTP + WebSocket on configurable host:port (de
|
|||
├── openrouter_attribution.py ← Canonical OpenRouter application attribution, centralized so forks do not compete under the same external application identity (§7)
|
||||
├── openai_chat_custom.py ← Pure direct-OpenAI Chat function→custom codec: compact schemas, exact catalog binding, tool-choice projection, prior-call replay, canonical response normalization; no Responses transcript or second stored history
|
||||
├── openai_chat_dispatch.py ← Direct-OpenAI Chat policy leaf: custom+requested reasoning first, exact-dialect fallback with the same reasoning, then task-local explicit `none` only when the physical-attempt rail still permits it; owns the bounded schema-error continuation
|
||||
├── request_wire_contract.py ← Provider-neutral exact-route request profile: closed `set_value`/`drop_field`/registered `replace_dialect` actions, 14-day success-only evidence store `data/state/request_wire_compatibility.json`; task-local explicit `none` can never become durable dispatch authority — a task-local availability fallback must not teach the route (§6 Context fitting, retry, and compaction; §10 invariant 13)
|
||||
├── request_wire_contract.py ← Exact-route request-profile actions and 14-day success-only evidence store; task-local `none` never teaches future dispatch (§6 Context fitting, retry, and compaction; §10 invariant 13)
|
||||
├── request_wire_resolution.py ← Deterministic request-profile composition with source-predicated effort bounds/transitions; contradictions fail open to the requested effort with `conflict=True`
|
||||
├── effort_evidence.py ← Live/recovered effort evidence: validate requested, prepared and reported; missing observations stay unknown, native ranking stays with the engine (§6)
|
||||
├── request_wire_receipts.py ← Factory-bound wire candidates + semantic-success receipts (exact serializer digests; tool-choice semantics cannot change)
|
||||
├── request_wire_attempt.py ← Physical-attempt validation split by authority: `validate_wire_attempt_identity` is the exact candidate/attempt identity every consumer shares, `validate_physical_wire_attempt` composes it with the settled-state requirement for the compatibility receipt, and the public `usage.request_wire` disclosure uses identity alone
|
||||
├── request_wire_attempt.py ← Shared exact candidate/attempt identity; compatibility receipts also require settlement, while `usage.request_wire` discloses identity alone
|
||||
├── request_wire_custom_validation.py ← Custom-call validation; a validation failure may prove wire acceptance but cannot authorize tool execution (`allows_execution` gate)
|
||||
├── request_wire_recovery.py ← One sync/async wire-recovery machine: durable evidence applied immediately before send, reactive evidence committed only after terminal semantic success, ordered terminal disclosures
|
||||
├── anthropic_native_custody.py ← Whole-block replay custody for Anthropic native reasoning (same provider/endpoint/API/model); opaque type/order/size/digest projections
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ Shared frontend primitives keep pages from acquiring competing contracts — fro
|
|||
- `client_surface.js` ← the send-time sending-surface snapshot (raw observables, no device taxonomy) that `chat.js` spreads into each chat frame.
|
||||
- `chat_markdown.js` ← marked+DOMPurify, chat URLs (http(s), mailto, `/api/files/download`), KaTeX (no single `$`), lazy Mermaid and bounded chart blocks. `mountChatMarkdown` binds HTML to `.ui-rich-content` on existing hosts; bubble/card owners retain enhancement/teardown and density. `chat.js` disposes on removal. Vendors: `web/vendor/VENDOR-MANIFEST.md`.
|
||||
- `log_events.js` ← event classification, shared technical outcome reducers and the one factual task-presentation projection for Chat and Logs: task truth maps only to `Working` / `Done` / `Done with warnings` / `Failed` / `Cancelled`; it owns no actions, incidents or notifications, compact headlines never expose raw reason codes, and a task-authored message is visible in the receiving task's timeline (`task_message_injected`, sender by value) and in the sender's (`task_message_routed`, written or refused with the host reason).
|
||||
- `effort_evidence.js` ← Logs: requested effort, host carrier, engine preparation, sourced provider observation; missing echoes stay unknown.
|
||||
- `toast.js`, `masonry.js`, `widget_frame.js`, `widget_job.js`, CSS tokens ← notifications, layout, framed-widget bootstrap/lifecycle, bounded widget request/job policy.
|
||||
- `result_files.js` ← a settled task card's one Files row: root files plus one folder record per nested tree with its member tooltip, links only from the host's `artifacts`/`artifact_archives`.
|
||||
- `history_retention.js` ← history-transfer details and storage problems for Chat and Logs; routine transfer does not change task outcome or collapsed-card state.
|
||||
|
|
@ -313,7 +314,7 @@ Account status refresh runs immediately and on visible page/tab activation; hidd
|
|||
|
||||
#### Review lanes and Available subagents
|
||||
|
||||
Review lanes edits one structured reviewer configuration (contract: §7 Reviewer slots). Each triad, scope, optional advisory or deep self-review row picks its reviewer from ONE flat select: Available-subagents roster rows lead as handle-named references, then inline channels — API delivery or a coding-agent session — then model, account and effort; direct API triad rows also choose Reads the work itself or Packet. Scope and deep self-review always retrieve: an API row runs bounded native inspection, a session row reads through its own harness. Recorded reads are diagnostic; scope runs in every context mode. Rows ask for no window acknowledgement. The deep-review block describes native inspection with the memory whitelist inline, or a delegated session, and labels a row synthesized from `OUROBOROS_MODEL_DEEP_SELF_REVIEW`. Unrelated Save omits untouched defaults, including repair placeholders; first saving a default panel preserves its displayed deep row. Saved panels keep legacy synthesis until that row is edited; blank models are refused at save. Saved choices that disappear from discovery stay visible as unavailable rather than silently changing; capability labels configure nothing, and server-returned limits plus the last effective execution disclose what a saved row actually ran as. The `CATEGORIES` and `SINGLETONS` tables share renderers and binders across the respective row groups. The serializer preserves a loaded empty triad/scope draft for local validation before POST; the backend independently enforces the same configuration contract, and an unloaded view authors no replacement. A row pinned to an account discovery no longer lists keeps its pin, disclosed once above the rows and only on the word of a facet actually read (account pins answer to `accounts`, models to `catalog`; an unread or failed facet yields "not checked", not "not in discovery"). The standing note states the rule, never the situation: every review surface — commit, scope, plan, advisory, skill review and task acceptance — follows its configured rows and waits for subscription capacity rather than falling back to API spend. An agent-session row's meta line carries the live availability sentence the Available-subagents cards compute (`sessionRouteVerdict`): the row, never the model option, says whether an account can run the selected model.
|
||||
Review lanes edit one configuration (§7 Reviewer slots). Captions refresh on edit; row effort overrides roster defaults, compound slugs stay intact. Behavior labels effort a preference; Logs shows requested/sent/reported. Triad/scope/advisory/deep-review rows share a flat select: roster handles, then inline API/session channels with model, account and effort; direct API triad rows also choose Reads the work itself or Packet. Scope/deep review always retrieve via bounded API inspection or the session harness. Reads are diagnostic; scope runs in every context mode. No row asks for window acknowledgement. The deep-review block describes native inspection with the memory whitelist inline, or a delegated session, and labels a row synthesized from `OUROBOROS_MODEL_DEEP_SELF_REVIEW`. Unrelated Save omits untouched defaults, including repair placeholders; first saving a default panel preserves its displayed deep row. Saved panels keep legacy synthesis until that row is edited; blank models are refused at save. Saved choices that disappear from discovery stay visible as unavailable rather than silently changing; capability labels configure nothing, and server-returned limits plus the last effective execution disclose what a saved row actually ran as. `CATEGORIES`/`SINGLETONS` share renderers/binders. The serializer preserves a loaded empty triad/scope draft for local validation before POST; the backend independently enforces the same configuration contract, and an unloaded view authors no replacement. A row pinned to an account discovery no longer lists keeps its pin, disclosed once above the rows and only on the word of a facet actually read (account pins answer to `accounts`, models to `catalog`; an unread or failed facet yields "not checked", not "not in discovery"). The standing note states the rule, never the situation: every review surface — commit, scope, plan, advisory, skill review and task acceptance — follows its configured rows and waits for subscription capacity rather than falling back to API spend. An agent-session row's meta line carries the live availability sentence the Available-subagents cards compute (`sessionRouteVerdict`): the row, never the model option, says whether an account can run the selected model.
|
||||
|
||||
**Available subagents** is the single task-actor editor. Its list-level Enabled flag and at most ten stable rows are the saved `OUROBOROS_SUBAGENTS` intent. The owner sees numbered compact cards (`docs/DESIGN.md` §6 row anatomy) and authors one prose field, Description (`recommended_use`), beside the structured API-model or Agent-session route, optional effort and optional managed-model/session account pin (empty pin = Claudexor's compatible-account rotation). Each card leads with its own `.ui-checkbox` switch before the `Subagent N` title, writing the row's `enabled` (§6). It is a draft the section's one Save writes, never an instant save; a switched-off card keeps every control editable and is never dimmed, and a save that would strand a reviewer reference on it is refused whole with that reference named. Each card's status dot composes two axes — intent (saved / draft / generated) and availability (an API-model route reads Checked at start) — with the tone the worse of the two (`subagent_status_primitives.sessionRouteVerdict` / `rowStatus`). A row is NAMED by its handle — route target plus EFFECTIVE non-default facets, as `codex=gpt-6-astra/xhigh` (`subagent_handle`, `subagentHandle`) — a stored label rots once the owner re-points the row (`name`, then preset ids); `subagent_id` is a hidden join key. Roster edits never save twin engines, off rows too: a Duplicate names its twin; old twins show `<handle>~<subagent_id>`. Agent-session rows also select native Access: Full system access (default) or Working files. A saved lower choice is explicit; rows saved without the field default to full for new tasks, while running tasks keep their captured profile and read-only assignments stay read-only. The editor shares only neutral route/model/account/status primitives with Review lanes; Add and Duplicate reveal the new entry through the shared `ui_helpers.revealNewRow`. A fresh entry is not an error: its meta line carries a neutral hint until a save attempt (`noteSaveAttempt`; `validate()` stays pure), after which the offending card is tinted and names its error.
|
||||
|
||||
|
|
|
|||
|
|
@ -154,7 +154,7 @@ Main also uses this materializer for an actor-authored working view. At the exis
|
|||
|
||||
Ouroboros stores one provider-neutral, function-shaped conversation. OpenRouter omits the neutral `tool_choice="auto"` while building the physical request, before request-wire identity is bound, so an implicit API default cannot become an unsupported parameter under strict routing. Direct OpenAI agent/tool traffic stays on Chat Completions: `openai_chat_custom.py` translates only the physical copy into Chat custom tools and normalizes returned custom calls back into canonical function calls; the dialect ladder is fixed at custom/function/none (ordinals 1/2/3), only an exact dialect rejection advances a rung, the custom→function control-flow fallback is never learned as a durable dialect action, and the compatibility layer never migrates the conversation to Responses or changes the model/provider/API surface. Direct Anthropic tool turns keep a private route-bound receipt with the complete native assistant `content`, replayed byte-for-byte on the immediately matching same-route continuation and scrubbed on any provider, endpoint, API-surface or model change; opaque values never reach summarizer or public projections. Owner-requested `none` is sent as `thinking.type=disabled`; no guessed legacy `budget_tokens` mapping is invented.
|
||||
|
||||
`request_wire_recovery.py` is the single provider-neutral adaptation driver for both exception and HTTP-200 body-error shapes, keyed on one credential-free exact route/profile: only bounded `set_value`, `drop_field` and registered `replace_dialect` actions, at most eight per call. A learnable reactive action becomes durable — the 14-day cross-process store `state/request_wire_compatibility.json` — only when a semantically valid normalized response is bound to the exact settled physical-attempt capture; provider prose cannot switch routes. Disclosure needs only the identity half: the candidate a RETURNED response carried is disclosed as `usage.request_wire` once its capture passes the exact-identity check (`request_wire_attempt.validate_wire_attempt_identity`), because a ledger write that fails after the provider answered cannot unmake which request shape was sent; the finalizer's place on the successful-return path is what proves a response arrived, and money keeps its own authority in the attempt ledger, so an unsettled attempt discloses but never teaches. Disclosures aggregate in order as bounded `request_wire_history` — not the physical-attempt ledger; `state/usage_attempts.jsonl` remains the complete monetary authority. Private custom-argument receipts never enter stored history or public observability.
|
||||
`request_wire_recovery.py` adapts exception/body errors for one credential-free exact route/profile with `set_value`, `drop_field` or registered `replace_dialect`: at most eight actions, existing send/pricing/Stop/deadline limits. Requested effort follows native mapping, never catalog ceilings. Positive field-bound enums need no scalar echo: highest comparable lower tier, or known minimum when all are higher. Negative quotes prescribe nothing; value rejection cannot drop the carrier. Mandatory floors/unsupported-carrier repairs remain. Only valid normalized success bound to the exact settled physical capture teaches the 14-day `state/request_wire_compatibility.json`; prose cannot reroute. Returned responses disclose `usage.request_wire` via `request_wire_attempt.validate_wire_attempt_identity` despite failed settlement; unsettled attempts never teach. Ordered, bounded `request_wire_history` is disclosure; `state/usage_attempts.jsonl` owns money. Private custom-argument receipts stay out of history/public views. Physical `effort` separates original preference, sent fields/omission and sourced reports (absent = unknown). Settlement keeps candidate facts; late receipts replace observations. Provider usage supplies neither effort nor engine reports. Legacy `requested_effort`/`applied_effort` mean provider projection/sent candidate with source labels, never provider-execution proof. Facts reach usage/events/Logs without automatic effort chat, toast or incident.
|
||||
|
||||
Retry budgets are failure-class specific: empty/incomplete responses and transient 429/5xx failures may retry the same model with deadline-bounded backoff; auth, quota, permanent bad requests and confirmed oversize fail fast; exhausted compatibility returns to the configured model fallback chain rather than becoming a second router. `LLMClient` keeps leading system messages authoritative and demotes later notices to visibly marked user notices; on the OpenAI family and the Claudexor route (`llm_claudexor._request`) the LEADING system message's builder-declared mutable blocks are also projected to one such notice placed before the task (prompt caching, below).
|
||||
|
||||
|
|
@ -266,7 +266,7 @@ The startup seal audit selects its bounded manifest set before reading live atte
|
|||
|
||||
`ClaudexorGateway` uses purpose-bound byte uploads, one idempotent operation ID, status/result reads, cancellation and explicit result acknowledgement. Claudexor performs at most one generation per operation: a lost local HTTP reply rejoins the same operation, and an unknown provider outcome never authorizes another generation. It shares its managed profile with Agent sessions — the adapter reads current access credentials while the official CLI keeps refresh ownership — and no shared external engine, relay process, copied operator auth store or public OpenAI-compatible endpoint. Metadata helpers use `read_owned_gateway` (owned-only discovery and handshake; they never install or start the engine); actual model calls and explicit lifecycle actions use `ensure_owned_gateway`.
|
||||
|
||||
Failure evidence: before creating an operation the host discovers the catalog's `captureFailureEvidence=true` query and freezes that choice for the invocation, same-key rejoin after control loss included (older catalogs keep the legacy request; the serving runtime pin remains the independent adoption boundary). A supporting engine puts every received failed-response byte and processing exception — incomplete responses and HTTP refusal bodies too — in the private result, which the observability CAS retains before ACK; usage and events carry only compact problem context, and a received prefix never claims the missing provider tail. A coherent provider terminal whose message cannot be normalized keeps its outcome, route and usage: the operation fails `response_rejected`, the host settles the reported usage and raises the non-repeatable `stream_rejected` class — no unknown-outcome recovery, no healthy account marked failed. `ClaudexorModelError.display_message` adds sanitized typed `vendorCode`/`parameter` details to the error event and terminal preview only after classification; unknown outcomes expose no provider details.
|
||||
One pre-create catalog read negotiates `captureFailureEvidence=true`/`captureEffortEvidence=true` on `POST /v2/model-operations`; invocation manifests freeze both for same-key rejoin. Unadvertised flags stay absent; runtime pinning remains independent. Supporting engines retain all failed-response bytes (incomplete/HTTP refusals too) and processing exceptions privately. CAS retains exact result bytes before ACK; GET never projects another view. Usage/events show compact problem context; a received prefix cannot prove the missing tail. Coherent provider terminals with unusable messages keep outcome/route/usage: `response_rejected` settles usage then raises non-repeatable `stream_rejected`, without unknown recovery or marking healthy accounts failed. After classification only, `ClaudexorModelError.display_message` adds sanitized typed `vendorCode`/`parameter` to error events/terminal previews; unknown outcomes expose no provider details.
|
||||
|
||||
Money and bytes: physical attempts stay in the usage ledger, where exact zero cash, known charges, estimates and unknown cash remain distinct — a subscription is not evidence of zero cost. Claudexor removes terminal request bytes and acknowledged result bytes, an unacknowledged ready result expires after 30 days, and compact operation receipts survive body deletion, so an expired read cannot start a new generation. Model-purpose resources are not valid Agent attachments. Private continuation envelopes stay in canonical history but out of public/summarizer projections; their reuse is bound to the actual source/model/profile/account identity.
|
||||
|
||||
|
|
@ -274,13 +274,15 @@ Image capability comes from the selected role/account's model catalog, not the g
|
|||
|
||||
The send copy omits foreign provider envelope metadata such as `stop_reason` (canonical history retains it), preserves a nonempty `refusal` as assistant text even when the provider supplied no ordinary content, and normalizes all-text tool results so moving the message-side cache boundary never rewrites bytes already sent (non-text blocks stay intact). For the same reason the transcript is append-only between the sends of one execution; compaction is the one sanctioned rewrite and every other break is recorded, never blocked (§6 Task lifecycle). Only the exact `model_request_invalid` create refusal proves validation failed before command admission; a generic HTTP error or failed status read cannot prove non-dispatch.
|
||||
|
||||
`effort_evidence.py` validates additive model/unique final session evidence without re-ranking native tokens or engine order on the host scale. Null submitted means the option was omitted; its known parameter may remain. Paired observations supply engine-reported effort. Missing/malformed stays unknown. Usage, settlement and Logs keep evidence separate from host-requested options.
|
||||
|
||||
#### The live turn slot
|
||||
|
||||
Beside assistant-level history the engine keeps ONE live turn per model client session, and its opaque token is transport, not content: it belongs to the caller that is running, not to any stored message. So the caller owns a single mutable slot (`llm_claudexor.ModelTurnState`) that rides the ordinary `LLMClient.chat` parameters to the engine boundary, the only writer — a dispatched durable result replaces the value, a released `invalid_continuation` repair clears it together with message envelopes; other non-dispatched, unknown or legacy-shaped outcomes preserve it. Holding state never licenses another generation, and silence about a turn never ends one. WHY a slot rather than the transcript: the live turn survives compaction that drops the messages and outlives a body that fails after its headers, so the last stored assistant envelope would rebuild the wrong lifetime. One `run_llm_loop` invocation is one turn (rounds, owner steering, acceptance follow-up, forced finalization, quota waits and reprepares continue it; a next loop and a cold restart start empty), and a consciousness wake-up is one such ordinary turn keyed by its own task id. A dispatch that leaves this transport for a direct API or local route ends the turn at the caller and never revives it on return; the engine alone compares route identity. Opting in requires a serving engine at `config.CLAUDEXOR_MODEL_TURN_STATE_MIN_VERSION`, read from the last SUCCESSFUL handshake — a failed probe never un-proves it, so concurrent status polling of that singleton cannot change the shape a running caller sends — and an older or not-yet-observed engine keeps the legacy request shape instead of risking a schema refusal. The token is never usage, an event, a progress note or a task card; mechanism in the `llm_claudexor.py` docstring.
|
||||
|
||||
#### Roles, accounts and context
|
||||
|
||||
Model roles own account pins and context assertions through `OUROBOROS_MODEL_ACCOUNTS` and `OUROBOROS_MODEL_CONTEXT_WINDOWS`; ordered fallback entries keep their original ordinal, reviewers and configured actors reuse their route credential fields, and identical Main/Light or reviewer model strings never imply identical roles or accounts. Auto uses the largest advertised window of the exact account route, not CLI compaction thresholds; a manual value is a sizing assertion, not a provider unlock or reviewer authority; unknown capacity stays unknown, and input size, response reservation and capacity are separate. Each operation records its submitted options beside the engine's applied options (an absent applied report stays explicitly unknown), and the first changed reasoning effort of each model in a task produces one typed owner-visible notice. An account change rebinds preparation before another physical send; ordinary sends, prospective wrap-up payloads and forced final replies share one acting-role/account binding, and prospective subscription accounting uses the dispatch serializer, not an OpenAI-compatible approximation. The final header identifies the responding model/account; a paid review that ends incomplete retains its full report and custody with that classification, never switches to another delivery or inherits the previous route's window.
|
||||
Model roles bind accounts/windows through `OUROBOROS_MODEL_ACCOUNTS` and `OUROBOROS_MODEL_CONTEXT_WINDOWS`. Fallback entries retain ordinals; reviewers and actors reuse route credentials. Matching model names imply neither matching roles nor accounts. Auto uses the exact account route's largest advertised window, not CLI compaction thresholds. Manual windows assert sizing, not provider/reviewer authority; capacity, input size and output reservation remain separate, with unknown capacity unknown. Usage pairs submitted options and provider response echoes; missing keys remain unknown individually. Effort disclosure stays in Logs; model-substitution and terminal notices remain. Account changes rebind preparation before dispatch. Ordinary, wrap-up and forced-final sends share the acting role/account; prospective subscription accounting uses the dispatch serializer. Final headers identify responding model/account. Incomplete paid reviews retain their report and custody without switching delivery or borrowing the prior window.
|
||||
|
||||
Host-authored sampling defaults stay separate from explicit parameters: review requests carry `default_temperature` as a hint each effective API route resolves at dispatch, raw model operations use their provider default, and an explicit `temperature` (zero included) is preserved and may be refused. Routes without structured output use the existing safety text-JSON parser, repair and refusal path — no new safety bypass. Ordinary confirmed provider errors keep each helper's previous handling, while quota, owner interruption and unknown physical custody propagate without becoming a helper result or verdict.
|
||||
|
||||
|
|
|
|||
|
|
@ -172,7 +172,7 @@ A registry of `config.SETTINGS_DEFAULTS` (exact defaults canonical in `settings_
|
|||
| OUROBOROS_HUB_CATALOG_URL | `https://raw.githubusercontent.com/razzant/OuroborosHub/main/catalog.json` | OuroborosHub catalog URL (automatic fetch limited to catalog JSON; installs verify SHA-256) |
|
||||
| OUROBOROS_CLAWHUB_REGISTRY_URL | `https://clawhub.ai/api/v1` | ClawHub registry URL |
|
||||
| OUROBOROS_PROMPT_CACHE_TTL | 1h | Prompt-cache tier default/5m/1h for cache markers on compatible Anthropic-family wire payloads; the final send boundary legalizes ordering, so prompt builders own no provider TTL policy; `review_helpers.cached_prompt_blocks` and `usage_accounting._reservation_cost` also consult it; usage records the applied tier |
|
||||
| OUROBOROS_EFFORT_TASK | medium | Task reasoning effort (none/minimal/low/medium/high/xhigh/max/ultra; Settings hides `minimal`); adaptation is exact-route, success-confirmed, disclosed in `request_wire`. Also the depth of post-task synthesis on the Light route (reflection, its Pattern Register update, episodic summary), which has no level of its own |
|
||||
| OUROBOROS_EFFORT_TASK | medium | Task reasoning effort (none/minimal/low/medium/high/xhigh/max/ultra; Settings hides `minimal`); preferred tier; exact-route, success-confirmed adaptation, original/sent/reported facts stay in usage/Logs. Controls Light post-task synthesis (reflection, Pattern Register update, episodic summary), which has no separate level |
|
||||
| OUROBOROS_EFFORT_EVOLUTION | high | Evolution effort |
|
||||
| OUROBOROS_EFFORT_REVIEW | high | Review effort for rows that pin none; a plan envelope's `reviewer_effort` outranks it and a row's pinned effort for that plan (a compound route slug keeps its encoded effort); the effective per-seat effort is recorded and a panel ordered weaker than the owner's setting is named |
|
||||
| OUROBOROS_EFFORT_SCOPE_REVIEW | high | Scope-review effort |
|
||||
|
|
|
|||
|
|
@ -613,25 +613,26 @@ and what enforces each.
|
|||
|
||||
#### Subscription transport
|
||||
|
||||
- Claudexor model calls are a transport, not delegated reasoning: model content and
|
||||
native continuation stay byte-faithful through the purpose-bound engine operation;
|
||||
never inject its credentials, run its tools, compact inside the adapter or silently
|
||||
repeat a generation. A lost connection rejoins the same operation ID, unknown stays
|
||||
unknown, ACK only after the private CAS owns the exact result. Failed-response capture
|
||||
uses the catalog's optional query, frozen before create and reused under the same
|
||||
idempotency key (absence keeps the legacy shape); received bytes and exception chains
|
||||
stay private, diagnostics compact. A known terminal with unusable output is a
|
||||
settled provider result plus local rejection (`stream_rejected`), never unknown or
|
||||
not-dispatched — keep both markers across sync, async and process boundaries; a
|
||||
local rejection never rotates accounts. A round the ENGINE says another model answered
|
||||
is not that class; the host matches no model strings.
|
||||
- Host hints are chosen by their caller from transport capability; an explicitly
|
||||
unsupported option refuses rather than being silently dropped and retried. Submitted
|
||||
options are recorded beside applied options on the usage row (absent report =
|
||||
unknown), and that record covers every submitted option while the owner line speaks
|
||||
only for the thinking horizon: the first changed reasoning effort of each model in a
|
||||
task emits one typed owner line (keyed by task and model, never per round, naming
|
||||
only the reporting route). A mismatch is disclosure, never a dispatch gate.
|
||||
- Claudexor preserves model content/native continuation bytes: no credential
|
||||
injection, tools, adapter compaction or silent regeneration. Lost control rejoins
|
||||
the same operation ID; unknown stays unknown. ACK after private CAS owns exact
|
||||
result bytes. Freeze catalog-advertised failure/effort queries before create for
|
||||
same-key rejoin; absent flags keep legacy shape. Bytes/exception chains private; diagnostics compact. Unusable terminals settle with
|
||||
`stream_rejected`, never unknown/not-dispatched; keep both across sync/async/process boundaries. Local rejection never rotates accounts.
|
||||
Only the engine classifies substitution; no host model-string matching.
|
||||
- Caller hints use capabilities; explicitly unsupported options refuse. Record
|
||||
submitted options/`appliedOptions` provider echoes; absent keys stay unknown.
|
||||
Effort: usage/events/Logs, no automatic chat/toast/incident or mismatch dispatch
|
||||
gate. Keep substitution/terminal notices and deliberate weaker-reviewer plan facts.
|
||||
- `effort_evidence.py` checks shape/coherence of model `effortResolution` (stored as
|
||||
`effort_resolution`) and sessions' unique final `final/telemetry.yaml` attempt.
|
||||
Live/late reads share `model_effort_usage`; missing/malformed clears stale reports.
|
||||
`submitted` is preparation, never dispatch;
|
||||
null means omission, not `none`. Known native `parameter` may remain when null
|
||||
and is required when submitted is non-null. Paired `observed`/`observedSource`
|
||||
supply provider observation; `source` names resolution authority. Preserve
|
||||
source-qualified `downward`/`floor` and future tokens: engine native order,
|
||||
never host preference-scale re-ranking.
|
||||
- The engine's active-turn token is a transport fact: the CALLER owns the slot
|
||||
(`llm_claudexor.ModelTurnState` on the loop context, also used by consciousness);
|
||||
only the engine boundary writes it. Fresh slot per logical
|
||||
|
|
@ -679,8 +680,12 @@ and what enforces each.
|
|||
dialect, no Responses migration, owner `none` on direct Anthropic =
|
||||
`thinking.type=disabled` (`tests/test_request_wire_contract.py`,
|
||||
`tests/test_openai_chat_custom_contract.py`, `tests/test_anthropic_native_custody.py`).
|
||||
`usage.request_wire` is one call's terminal candidate; nested aggregation keeps the
|
||||
ordered `request_wire_history` with explicit omission accounting.
|
||||
`usage.request_wire` is a call's terminal candidate; nested aggregation preserves
|
||||
`request_wire_history` order/omissions. Legacy `applied_effort` is sent candidate,
|
||||
never provider-applied. Capture preference before native projection; separate
|
||||
request/sent/report. Positive field-bound enums need
|
||||
no scalar echo; negative quotes prescribe nothing, value rejection cannot drop
|
||||
the carrier. Existing attempt/pricing/Stop/deadline limits apply.
|
||||
- Every core-mediated physical provider send goes through
|
||||
`usage_accounting.execute_physical_attempt[_async]` (`tests/test_usage_accounting.py`);
|
||||
custody classifiers read the explicit `__cause__` chain, never `__context__`, and an
|
||||
|
|
@ -854,8 +859,7 @@ and what enforces each.
|
|||
the same incident pair naming both models, the account the send's own binding
|
||||
selects when the route has accounts (a task-local wait override included, never the
|
||||
configured value alone) and the typed failure reason when the round record has one;
|
||||
the applied-option mismatch line rides the same callable, and the frozen
|
||||
`ToolContext.emit_progress_fn` takes one argument, never the pair.
|
||||
the frozen `ToolContext.emit_progress_fn` takes one argument, never the pair.
|
||||
- Timeout classes are separate axes. A transport timeout
|
||||
(`OUROBOROS_LLM_TRANSPORT_READ_TIMEOUT_SEC`) bounds only a dead socket — never a
|
||||
reasoning cutoff or evidence of a stall. API review uses it as a settlement fallback
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Machine extraction of the `docs/ARCHITECTURE.md` "Data layout (`~/Ouroboros/`)" tree — the durable-file orientation carrier (this tree's counterpart of the reference PERSISTENCE_OWNERS derivation checklist) — regenerated by `python scripts/regenerate_inventories.py`. Do not edit. Every entry is probed against reality: repo entries must exist as tracked paths; data-plane entries must appear as a literal in the runtime sources that construct them. A durable file renamed or removed in code while its tree row survives = red (`tests/test_generated_inventories.py`).
|
||||
|
||||
Source: `docs/architecture/01-high-level-architecture.md`, physical LF lines 617-709; UTF-8 SHA-256 `02c0531d3142158974a70220ecc38f256871b6db318697d02c5098ab5c3af722`.
|
||||
Source: `docs/architecture/01-high-level-architecture.md`, physical LF lines 618-710; UTF-8 SHA-256 `1826c67b1828cb088e99cbc460b751cbddf3517d5afaac37e05cd032e02eeb4b`.
|
||||
|
||||
- entries: **82** (code-ref: 75, repo-dir: 6, repo-path: 1)
|
||||
|
||||
|
|
|
|||
|
|
@ -154,6 +154,10 @@ def usage_from_response(response: Any) -> Tuple[Dict[str, Any], Optional[float],
|
|||
"cached_tokens": cache_read,
|
||||
"cache_write_tokens": cache_write,
|
||||
}
|
||||
# Candidate facts and validated engine reports come from their host owners,
|
||||
# never arbitrary extensions inside a provider's token-usage block.
|
||||
normalized.pop("effort", None)
|
||||
normalized.pop("effort_resolution", None)
|
||||
if isinstance(payload, dict):
|
||||
if "service_tier" in payload:
|
||||
normalized["service_tier"] = payload["service_tier"]
|
||||
|
|
|
|||
|
|
@ -352,7 +352,7 @@ _SKILL_ATTEMPT_FIELDS = (
|
|||
"cost_usd", "cost_final", "reservation_upper_bound_usd", "pricing_known",
|
||||
"prompt_tokens", "completion_tokens", "cached_tokens", "subscription_route",
|
||||
"subscription_reset_at", "credential_profile_id", "access_profile",
|
||||
"processing", "processing_basis", "cost_evidence", "attempt_execution",
|
||||
"effort", "effort_resolution", "processing", "processing_basis", "cost_evidence", "attempt_execution",
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"schema_version": 1,
|
||||
"release": {
|
||||
"version": "3.16.0",
|
||||
"build_sha": "80ef38394f0a9f316d36a0835f7106dc7b40d164",
|
||||
"version": "3.17.1",
|
||||
"build_sha": "92c537456e6cc3e9a6d5ba3ee3a83fcc229b2752",
|
||||
"protocol_major": 3,
|
||||
"archive_url": "https://github.com/razzant/claudexor/releases/download/v3.16.0/claudexor-runtime-3.16.0.tar.gz",
|
||||
"sha256": "bfd757458deb9beea0f07041f3024f3e56aa83544d2ca6ff0e0bcad135758a52",
|
||||
"size_bytes": 7911192,
|
||||
"archive_url": "https://github.com/razzant/claudexor/releases/download/v3.17.1/claudexor-runtime-3.17.1.tar.gz",
|
||||
"sha256": "7e4d36a3d00fc43ae262e4190fc6942db3ad5e453e75bb2c9d10077735c12010",
|
||||
"size_bytes": 7929769,
|
||||
"node_version": "24.16.0",
|
||||
"node_artifacts": {
|
||||
"darwin-arm64": {
|
||||
|
|
|
|||
|
|
@ -1099,6 +1099,7 @@ def settle_run(drive_root: Any, gateway: Any, custody: RunCustody, detail: Dict[
|
|||
# and the ledger writer decides what is usable.
|
||||
input_token_usage=summary.get("inputTokenUsage"),
|
||||
attempt_execution=detail.get("attemptExecution"),
|
||||
effort_resolution=observed.get("effort_resolution"),
|
||||
spend_usd=spend,
|
||||
spend_estimated=estimated,
|
||||
credential_profile_id=applied_profile,
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@ D20 = "Presence"
|
|||
"ouroboros/depth_evidence.py" = "D07"
|
||||
"ouroboros/dialogue_evidence.py" = "D06"
|
||||
"ouroboros/dialogue_provenance.py" = "D15"
|
||||
"ouroboros/effort_evidence.py" = "D07"
|
||||
"ouroboros/event_bus.py" = "D14"
|
||||
"ouroboros/evolution_checkpoints.py" = "D15"
|
||||
"ouroboros/evolution_fingerprint.py" = "D15"
|
||||
|
|
|
|||
65
ouroboros/effort_evidence.py
Normal file
65
ouroboros/effort_evidence.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
"""Validate Claudexor's additive effort evidence without resolving any route."""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from typing import Any
|
||||
|
||||
|
||||
def model_effort_usage(result: dict, request_facts: Any) -> dict:
|
||||
"""Decode only this result's reports, retaining the exact attempt's intent.
|
||||
|
||||
Live calls and late receipts share this decoder. An older observation on a
|
||||
retained candidate is never evidence about the result being settled now.
|
||||
"""
|
||||
effort = copy.deepcopy(request_facts) if isinstance(request_facts, dict) else {}
|
||||
effort.update(reported=None, report_source=None)
|
||||
usage = {"effort": effort}
|
||||
resolution = validated_effort_resolution(result.get("effortResolution"))
|
||||
if "effortResolution" in result:
|
||||
usage["effort_resolution"] = resolution
|
||||
applied = result.get("appliedOptions")
|
||||
reported = applied.get("reasoningEffort") if isinstance(applied, dict) else None
|
||||
if isinstance(reported, str) and reported.strip():
|
||||
effort.update(reported=reported, report_source="provider_applied_options")
|
||||
if resolution and resolution["observed"] is not None:
|
||||
effort.update(reported=resolution["observed"], report_source=resolution["observedSource"])
|
||||
return usage
|
||||
|
||||
|
||||
def validated_effort_resolution(value: Any) -> dict | None:
|
||||
"""Prepared native options are not dispatch proof or provider observations.
|
||||
|
||||
Unknown/malformed reports stay unknown. Claudexor owns native vocabulary
|
||||
and ordering; validate the evidence shape without re-resolving its claim.
|
||||
"""
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
nullable = ("requested", "submitted", "parameter", "observed", "observedSource")
|
||||
if any(key not in value or (value[key] is not None and
|
||||
(not isinstance(value[key], str) or not value[key].strip())) for key in nullable):
|
||||
return None
|
||||
resolution = value.get("resolution")
|
||||
if not isinstance(resolution, str) or resolution not in {"exact", "downward", "floor", "omitted", "unverifiable", "rejected"}:
|
||||
return None
|
||||
source = value.get("source")
|
||||
if not isinstance(source, str) or source not in {"account_catalog", "live_probe", "versioned_snapshot", "adapter"}:
|
||||
return None
|
||||
if "reason" in value and not isinstance(value["reason"], str):
|
||||
return None
|
||||
requested, submitted = value["requested"], value["submitted"]
|
||||
# An omitted option may still name its known native parameter.
|
||||
if submitted is not None and value["parameter"] is None:
|
||||
return None
|
||||
if (value["observed"] is None) != (value["observedSource"] is None):
|
||||
return None
|
||||
if resolution == "exact":
|
||||
if requested is None or submitted != requested:
|
||||
return None
|
||||
elif resolution in {"downward", "floor"}:
|
||||
if requested is None or submitted is None:
|
||||
return None
|
||||
elif submitted is not None:
|
||||
return None
|
||||
if resolution == "rejected" and value["observed"] is not None:
|
||||
return None
|
||||
return copy.deepcopy(value)
|
||||
|
|
@ -30,6 +30,7 @@ from dataclasses import dataclass
|
|||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
from ouroboros.effort_evidence import validated_effort_resolution
|
||||
|
||||
from ouroboros.config import (
|
||||
CLAUDEXOR_MIN_VERSION,
|
||||
|
|
@ -653,10 +654,14 @@ class ClaudexorGateway:
|
|||
return ref
|
||||
|
||||
def create_model_operation(self, request_ref: Dict[str, Any], *,
|
||||
idempotency_key: str, capture_failure_evidence: bool = False) -> Dict[str, Any]:
|
||||
idempotency_key: str, capture_failure_evidence: bool = False,
|
||||
capture_effort_evidence: bool = False) -> Dict[str, Any]:
|
||||
"""Create or rejoin exactly one caller-identified generation; never mint a retry key."""
|
||||
key = _model_idempotency_key(idempotency_key)
|
||||
path = "/v2/model-operations" + ("?captureFailureEvidence=true" if capture_failure_evidence else "")
|
||||
query = "&".join(f"{name}=true" for name, enabled in (
|
||||
("captureFailureEvidence", capture_failure_evidence),
|
||||
("captureEffortEvidence", capture_effort_evidence)) if enabled)
|
||||
path = "/v2/model-operations" + (f"?{query}" if query else "")
|
||||
return _model_operation(self._request(
|
||||
"POST", path, json_body={"request": _model_payload_ref(request_ref)},
|
||||
headers={"Idempotency-Key": key},
|
||||
|
|
@ -674,14 +679,10 @@ class ClaudexorGateway:
|
|||
def get_model_result(self, operation_id: str, *, expected_ref: Dict[str, Any],
|
||||
timeout_sec: Optional[float] = None,
|
||||
raw_bytes: bool = False) -> Dict[str, Any] | bytes:
|
||||
"""Read and verify the complete result, without ACK, redaction or artifact caps.
|
||||
|
||||
The expected reference comes from this operation's ready custody record.
|
||||
Failure preserves that handle: only another read of the same operation is
|
||||
appropriate here, never a new generation. The caller acknowledges after
|
||||
it has retained the returned result under its own custody contract.
|
||||
``raw_bytes`` retains the exact verified JSON encoding for that custody;
|
||||
it never skips the size, digest, UTF-8 or object validation below.
|
||||
"""Verify the complete result against ready custody; no ACK, redaction or caps.
|
||||
Failure preserves the handle: re-read this operation, never regenerate.
|
||||
The caller ACKs after retaining the result under its own custody contract.
|
||||
``raw_bytes`` keeps exact JSON encoding; size, digest, UTF-8 and object checks apply.
|
||||
"""
|
||||
from urllib.parse import quote
|
||||
|
||||
|
|
@ -1442,8 +1443,8 @@ def attempt_containment(run_dir: str) -> List[AttemptContainment]:
|
|||
return applied
|
||||
|
||||
|
||||
def final_attempt_facts(detail: Dict[str, Any], run_id: str) -> Dict[str, str]:
|
||||
"""Read the final attempt's route facts from engine-owned telemetry.
|
||||
def final_attempt_facts(detail: Dict[str, Any], run_id: str) -> Dict[str, Any]:
|
||||
"""Read the final attempt's route and effort facts from engine-owned telemetry.
|
||||
|
||||
The summary's model and harnesses echo the request; its route/authRoute
|
||||
projections may borrow facts from earlier attempts. Only the unique row
|
||||
|
|
@ -1472,13 +1473,16 @@ def final_attempt_facts(detail: Dict[str, Any], run_id: str) -> Dict[str, str]:
|
|||
if len(matching) != 1:
|
||||
return {}
|
||||
row = matching[0]
|
||||
return {
|
||||
facts = {
|
||||
target: row.get(source) if isinstance(row.get(source), str) else ""
|
||||
for target, source in (
|
||||
("attempt_id", "attempt_id"), ("harness_id", "harness_id"),
|
||||
("model", "observed_model"), ("profile_id", "profile_id"),
|
||||
)
|
||||
}
|
||||
if "effort_resolution" in row:
|
||||
facts["effort_resolution"] = validated_effort_resolution(row["effort_resolution"])
|
||||
return facts
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
|
|
|||
|
|
@ -232,6 +232,8 @@ class LLMClient(
|
|||
if use_local:
|
||||
turn_state_for_route(model_turn_state, "local")
|
||||
local_kwargs = {"timeout": timeout}
|
||||
if reasoning_effort is not None:
|
||||
local_kwargs["reasoning_effort"] = reasoning_effort
|
||||
if processing_preference:
|
||||
local_kwargs["processing_preference"] = processing_preference
|
||||
if context_mode:
|
||||
|
|
@ -311,6 +313,8 @@ class LLMClient(
|
|||
def local_call():
|
||||
adopt_physical_attempt_capture(None)
|
||||
local_kwargs = {"timeout": timeout}
|
||||
if reasoning_effort is not None:
|
||||
local_kwargs["reasoning_effort"] = reasoning_effort
|
||||
if processing_preference:
|
||||
local_kwargs["processing_preference"] = processing_preference
|
||||
if context_mode:
|
||||
|
|
|
|||
|
|
@ -448,6 +448,7 @@ class _AnthropicLaneMixin:
|
|||
target, messages, reasoning_effort, max_tokens, tool_choice,
|
||||
temperature, tools, **remote_kwargs,
|
||||
)
|
||||
target["requested_reasoning_effort"] = normalize_reasoning_effort(reasoning_effort)
|
||||
system, messages = self._build_anthropic_messages(messages, target)
|
||||
payload: Dict[str, Any] = {
|
||||
"model": str(target.get("resolved_model") or ""),
|
||||
|
|
|
|||
|
|
@ -296,6 +296,10 @@ def attach_processing_receipt(target: Dict[str, Any], usage: Dict[str, Any]) ->
|
|||
model = str(target.get("usage_model") or target.get("resolved_model") or "")
|
||||
capture = last_physical_attempt_capture()
|
||||
matched = capture is not None and capture.provider == provider and capture.model == model
|
||||
usage.pop("effort", None) # Host evidence cannot be supplied by a provider usage echo.
|
||||
usage.pop("effort_resolution", None)
|
||||
if matched and capture.effort is not None:
|
||||
usage["effort"] = copy.deepcopy(capture.effort)
|
||||
requested = (capture.processing_preference if matched
|
||||
else str(target.get("processing_preference") or ""))
|
||||
submitted = capture.submitted_processing_mode if matched else ""
|
||||
|
|
@ -380,6 +384,31 @@ def processing_refusal(target: Dict[str, Any], payload: Dict[str, Any],
|
|||
return error
|
||||
|
||||
|
||||
def effort_request_facts(target: Dict[str, Any], payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Original preference and actual host-send fields, never provider execution.
|
||||
|
||||
Kept with every physical attempt, including refused/failed sends. Native
|
||||
mappings are facts of the payload; an absent knob means vendor default.
|
||||
"""
|
||||
fields = {key: copy.deepcopy(payload[key]) for key in
|
||||
("reasoning_effort", "reasoning", "thinking") if key in payload}
|
||||
output = payload.get("output_config")
|
||||
if isinstance(output, dict) and "effort" in output:
|
||||
fields["output_config.effort"] = copy.deepcopy(output["effort"])
|
||||
extra = payload.get("extra_body")
|
||||
extra = extra if isinstance(extra, dict) else {}
|
||||
for key in ("reasoning", "thinking"):
|
||||
if key in extra:
|
||||
fields[f"extra_body.{key}"] = copy.deepcopy(extra[key])
|
||||
options = payload.get("options")
|
||||
options = options if isinstance(options, dict) else {}
|
||||
if "reasoningEffort" in options:
|
||||
fields["options.reasoningEffort"] = options["reasoningEffort"]
|
||||
return {"requested": target.get("requested_reasoning_effort"),
|
||||
"sent": fields, "sent_state": "explicit" if fields else "omitted",
|
||||
"sent_source": "host_candidate", "reported": None, "report_source": None}
|
||||
|
||||
|
||||
def _attempt_request(
|
||||
target: Dict[str, Any],
|
||||
payload: Dict[str, Any],
|
||||
|
|
@ -439,6 +468,7 @@ def _attempt_request(
|
|||
processing_preference=str(target.get("processing_preference") or ""),
|
||||
submitted_processing_mode=submitted_processing_mode(target, payload),
|
||||
processing_basis=copy.deepcopy(target.get("processing_basis")),
|
||||
effort=effort_request_facts(target, payload),
|
||||
candidate_clock_free_sha256=(
|
||||
hashlib.sha256(_canonical_candidate_bytes(clock_free)).hexdigest()
|
||||
if clock_note is not None else None
|
||||
|
|
@ -469,14 +499,16 @@ def _finalized_physical_candidate(
|
|||
) -> Dict[str, Any]:
|
||||
from ouroboros.request_wire_recovery import refresh_wire_clock
|
||||
|
||||
physical = _physical_candidate(payload)
|
||||
# Bind the caller's exact logical input before removing host/socket metadata
|
||||
# or sampling a fresh clock; recovery must not guess those transformations.
|
||||
physical = _physical_candidate({key: value for key, value in payload.items() if key != "timeout"})
|
||||
if fresh_clock:
|
||||
physical = refresh_wire_clock(physical, api_surface=api_surface)
|
||||
if target.get("context_mode") == "nano":
|
||||
physical = _fit_output_payload(target, physical, api_surface)
|
||||
return prepare_wire_payload_for_send(
|
||||
{**target, "contract_headers": processing_contract_headers(target, physical)},
|
||||
physical, api_surface=api_surface,
|
||||
physical, api_surface=api_surface, logical_payload=payload,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ from ouroboros.anthropic_native_custody import scrub_native_custody
|
|||
from ouroboros.claudexor_daemon import ensure_owned_gateway, owned_engine_version, read_owned_gateway
|
||||
from ouroboros.deadline_utils import llm_transport_timeout_sec
|
||||
from ouroboros.gateways.claudexor import (
|
||||
ClaudexorUnavailable, engine_at_least, model_failure_evidence_supported, _READ_TIMEOUT_SEC)
|
||||
ClaudexorUnavailable, engine_at_least, model_failure_evidence_supported,
|
||||
operation_query_supported, _READ_TIMEOUT_SEC)
|
||||
from ouroboros.llm_attempt import _attempt_request, _candidate_before_dispatch
|
||||
from ouroboros.send_clock import stamp_clock_note
|
||||
from ouroboros.llm_substitution import (
|
||||
|
|
@ -80,8 +81,7 @@ def prepare_processing_target(target: dict) -> dict:
|
|||
"""
|
||||
if not target.get("processing_preference") or "processing_preferences" in target:
|
||||
return target
|
||||
prepared = dict(target)
|
||||
prepared["processing_preferences"] = []
|
||||
prepared = {**target, "processing_preferences": []}
|
||||
try:
|
||||
sources = model_sources(processing_view=True)
|
||||
except ClaudexorUnavailable:
|
||||
|
|
@ -173,8 +173,10 @@ def propagate_model_error(error: Exception) -> None:
|
|||
raise error
|
||||
|
||||
|
||||
def _usage(result: dict) -> tuple[dict, float | None, bool]:
|
||||
def _usage(result: dict, effort: dict | None = None) -> tuple[dict, float | None, bool]:
|
||||
"""Normalize explicit model usage; never run the generic body-error/free branch."""
|
||||
from ouroboros.effort_evidence import model_effort_usage
|
||||
|
||||
counters = result.get("usage") or {}
|
||||
cost_evidence = result.get("cost") or {}
|
||||
cash = provider_cost_value(cost_evidence.get("cashUsd"))
|
||||
|
|
@ -182,13 +184,11 @@ def _usage(result: dict) -> tuple[dict, float | None, bool]:
|
|||
cost = cash if knowledge in {"exact", "estimated"} else None
|
||||
if knowledge == "estimated" and cost is None:
|
||||
cost = provider_cost_value(cost_evidence.get("estimatedUsd"))
|
||||
usage = {
|
||||
"prompt_tokens": counters.get("input_tokens"),
|
||||
"completion_tokens": counters.get("output_tokens"),
|
||||
"cached_tokens": counters.get("cached_input_tokens"),
|
||||
"cache_write_tokens": counters.get("cache_write_tokens"),
|
||||
"reasoning_tokens": counters.get("reasoning_tokens"),
|
||||
}
|
||||
usage = {key: counters.get(native) for key, native in (
|
||||
("prompt_tokens", "input_tokens"), ("completion_tokens", "output_tokens"),
|
||||
("cached_tokens", "cached_input_tokens"), ("cache_write_tokens", "cache_write_tokens"),
|
||||
("reasoning_tokens", "reasoning_tokens"))}
|
||||
usage.update(model_effort_usage(result, effort))
|
||||
if isinstance(result.get("processing"), dict):
|
||||
usage["processing"] = copy.deepcopy(result["processing"])
|
||||
if cost_evidence:
|
||||
|
|
@ -230,9 +230,7 @@ def turn_state_for_route(slot: ModelTurnState | None, provider: str) -> ModelTur
|
|||
the caller, and returning later starts a fresh one rather than reviving a
|
||||
token the engine no longer owns.
|
||||
"""
|
||||
if slot is None:
|
||||
return None
|
||||
if str(provider or "") != "claudexor":
|
||||
if slot is not None and str(provider or "") != "claudexor":
|
||||
slot.envelope = None
|
||||
return None
|
||||
return slot
|
||||
|
|
@ -290,6 +288,7 @@ def cache_key_for_model(model: str) -> str:
|
|||
def _request(target: dict, messages: list, tools: list | None, parameters: dict) -> dict:
|
||||
from ouroboros.llm_messages import _MessageShapingMixin, project_declared_system_prefix
|
||||
|
||||
target["requested_reasoning_effort"] = parameters.get("reasoning_effort")
|
||||
for name in ("response_format", "allow_server_web_search", "bypass_response_cache"):
|
||||
if parameters.get(name) or (name == "response_format" and parameters.get(name) is not None):
|
||||
raise ClaudexorModelError({"code": "unsupported_parameter", "message": f"Claudexor model transport does not support {name}.",
|
||||
|
|
@ -389,6 +388,7 @@ class _ModelInvocation:
|
|||
self.interrupt_reason = ""
|
||||
self.create_attempted = False
|
||||
self.capture_failure_evidence = False
|
||||
self.capture_effort_evidence = False
|
||||
self.defer_close = False
|
||||
self.io_active = False
|
||||
self.io_lock = threading.Lock()
|
||||
|
|
@ -429,13 +429,17 @@ class _ModelInvocation:
|
|||
self.gateway = ensure_owned_gateway()
|
||||
# Freeze once before create. A lost create reply or replaced gateway
|
||||
# must reuse this same operation's diagnostic/idempotency contract.
|
||||
self.capture_failure_evidence = model_failure_evidence_supported(self.gateway.operations())
|
||||
operations = self.gateway.operations()
|
||||
self.capture_failure_evidence = model_failure_evidence_supported(operations)
|
||||
self.capture_effort_evidence = operation_query_supported(operations, method="POST",
|
||||
path="/v2/model-operations", name="captureEffortEvidence", value="true")
|
||||
self.request_ref = self.gateway.upload_model_request(self.payload, idempotency_key=self.invocation_id)
|
||||
self.request_manifest_ref = persist_call(self.root, task_id=self.task_id, call_id=f"{self.invocation_id}_model_request",
|
||||
call_type="llm_claudexor_request", payload=self.payload, keep_raw=True,
|
||||
manifest={"invocation_id": self.invocation_id, "request_ref": self.request_ref,
|
||||
"model_role": self.role,
|
||||
"capture_failure_evidence": self.capture_failure_evidence})["manifest_ref"]
|
||||
"capture_failure_evidence": self.capture_failure_evidence,
|
||||
"capture_effort_evidence": self.capture_effort_evidence})["manifest_ref"]
|
||||
except ClaudexorUnavailable as error:
|
||||
raise ClaudexorModelError({"code": error.code, "message": str(error)}, model_role=self.role) from None
|
||||
|
||||
|
|
@ -450,7 +454,8 @@ class _ModelInvocation:
|
|||
call_type="llm_claudexor_request", payload=self.payload, keep_raw=True,
|
||||
manifest={"invocation_id": self.invocation_id, "request_ref": self.request_ref,
|
||||
"model_role": self.role, "operation_id": self.operation_id,
|
||||
"capture_failure_evidence": self.capture_failure_evidence})["manifest_ref"]
|
||||
"capture_failure_evidence": self.capture_failure_evidence,
|
||||
"capture_effort_evidence": self.capture_effort_evidence})["manifest_ref"]
|
||||
except Exception as error:
|
||||
self.request_manifest_ref = {}
|
||||
log.warning("Model custody checkpoint unavailable: %s", type(error).__name__)
|
||||
|
|
@ -479,7 +484,8 @@ class _ModelInvocation:
|
|||
self.create_attempted = True
|
||||
self.observe_operation()
|
||||
detail = self.gateway.create_model_operation(self.request_ref, idempotency_key=self.invocation_id,
|
||||
**({"capture_failure_evidence": True} if self.capture_failure_evidence else {}))
|
||||
**({"capture_failure_evidence": True} if self.capture_failure_evidence else {}),
|
||||
**({"capture_effort_evidence": True} if self.capture_effort_evidence else {}))
|
||||
self.operation_id = detail["id"]
|
||||
self.observe_operation(accepted=True)
|
||||
else:
|
||||
|
|
@ -488,7 +494,6 @@ class _ModelInvocation:
|
|||
if self.outage_episode is not None:
|
||||
self._control_outage(recovered=True)
|
||||
if detail.get("state") not in {"queued", "running"}:
|
||||
self.detail = detail
|
||||
response = detail.get("response") or {}
|
||||
if response.get("state") != "ready":
|
||||
raise self.error(detail.get("problem"), detail,
|
||||
|
|
@ -575,13 +580,10 @@ class _ModelInvocation:
|
|||
previous, self.gateway = self.gateway, replacement
|
||||
if previous is not None:
|
||||
previous.close()
|
||||
def controlled():
|
||||
self.check_control()
|
||||
return False
|
||||
# Unlike an owner-mail peek, check_control's exception must propagate.
|
||||
deadline = time.monotonic() + backoff
|
||||
while time.monotonic() < deadline:
|
||||
controlled()
|
||||
self.check_control()
|
||||
time.sleep(min(config.CLAUDEXOR_MODEL_POLL_INTERVAL_SEC, max(0, deadline - time.monotonic())))
|
||||
return True
|
||||
|
||||
|
|
@ -616,7 +618,9 @@ class _ModelInvocation:
|
|||
return custody
|
||||
|
||||
def extract_usage(self, result: dict) -> tuple[dict, float | None, bool]:
|
||||
usage, cost, final = _usage(result)
|
||||
from ouroboros.llm_attempt import effort_request_facts
|
||||
|
||||
usage, cost, final = _usage(result, effort_request_facts(self.target, self.payload))
|
||||
# Settlement reads this row before the caller decides anything, and the
|
||||
# density witness must know whose tokenizer it measured: a generation
|
||||
# another model produced teaches nothing about the requested one. The
|
||||
|
|
@ -638,10 +642,6 @@ class _ModelInvocation:
|
|||
def finish(self, result: dict) -> tuple[dict, dict]:
|
||||
usage, cost, final = self.extract_usage(result)
|
||||
route = result.get("route") or {}
|
||||
requested_options = copy.deepcopy(self.payload.get("options") or {})
|
||||
applied_options = copy.deepcopy(result.get("appliedOptions"))
|
||||
options_honored = "unknown" if applied_options is None else (
|
||||
"mismatch" if any(applied_options[key] != value for key, value in requested_options.items() if key in applied_options) else "confirmed")
|
||||
usage.pop("wire_layout", None) # host-owned: the projection fact of THIS call's target
|
||||
usage.update(provider="claudexor", resolved_model=self.target["usage_model"], cost=cost, cost_final=final,
|
||||
cost_estimated=cost is not None and not final,
|
||||
|
|
@ -650,8 +650,9 @@ class _ModelInvocation:
|
|||
"requested_profile": str((self.payload.get("account") or {}).get("profileId") or ""),
|
||||
"route": copy.deepcopy(route), "cost_evidence": copy.deepcopy(result.get("cost")),
|
||||
"outcome": result.get("outcome"), "problem": copy.deepcopy(result.get("problem")),
|
||||
"requested_options": requested_options, "applied_options": applied_options,
|
||||
"options_honored": options_honored,
|
||||
"requested_options": copy.deepcopy(self.payload.get("options") or {}),
|
||||
"applied_options": copy.deepcopy(result.get("appliedOptions")),
|
||||
"applied_options_source": "provider_response",
|
||||
"output_reserve_tokens": self.output_reserve, "output_cap_applied": False,
|
||||
"result_custody": {"state": "pending", "operation_id": self.operation_id,
|
||||
"response_ref": self.response_ref,
|
||||
|
|
@ -855,8 +856,7 @@ def chat_claudexor(target: dict, messages: list, tools: list | None, **parameter
|
|||
cause.physical_attempt_capture = error.physical_attempt_capture
|
||||
all_operations_not_started &= (isinstance(cause, ClaudexorModelNotDispatched)
|
||||
and error.physical_attempt_capture.state == "released")
|
||||
if isinstance(cause, ClaudexorModelError):
|
||||
cause.presence_all_operations_not_started = all_operations_not_started
|
||||
cause.presence_all_operations_not_started = all_operations_not_started
|
||||
raise cause from None
|
||||
raise
|
||||
finally:
|
||||
|
|
@ -918,7 +918,7 @@ def recover_model_attempt(drive_root, row: dict, *, gateway_factory=None):
|
|||
result = json.loads(raw)
|
||||
if not isinstance(result, dict) or result.get("outcome") not in {"completed", "incomplete", "failed"}:
|
||||
return "abandoned", {}, None, False
|
||||
usage, cost, final = _usage(result)
|
||||
usage, cost, final = _usage(result, row.get("effort"))
|
||||
return "settled", usage, cost, final
|
||||
finally:
|
||||
if gateway is not None and gateway_factory is None:
|
||||
|
|
|
|||
|
|
@ -372,8 +372,6 @@ class _RecoveryLadderMixin:
|
|||
|
||||
def _send(candidate: Dict[str, Any]) -> Any:
|
||||
nonlocal prior_capture
|
||||
# Socket policy is not model input; seal only the provider payload.
|
||||
candidate = {key: value for key, value in candidate.items() if key != "timeout"}
|
||||
candidate = _finalized_physical_candidate(
|
||||
target, candidate, "messages" if target.get("provider") == "anthropic" else "chat.completions",
|
||||
fresh_clock=True,
|
||||
|
|
@ -534,7 +532,6 @@ class _RecoveryLadderMixin:
|
|||
|
||||
async def _send(candidate: Dict[str, Any]) -> Any:
|
||||
nonlocal prior_capture
|
||||
candidate = {key: value for key, value in candidate.items() if key != "timeout"}
|
||||
candidate = _finalized_physical_candidate(target, candidate, "chat.completions", fresh_clock=True)
|
||||
request = _attempt_request(target, candidate)
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from ouroboros.llm_attempt import (
|
|||
_candidate_before_dispatch,
|
||||
_execute_candidate,
|
||||
_physical_candidate,
|
||||
attach_processing_receipt,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -233,6 +234,7 @@ class _GigaChatLaneMixin:
|
|||
# no_proxy (a macOS fork-safety flag for the OpenAI/requests paths) does
|
||||
# not apply here.
|
||||
del no_proxy
|
||||
target["requested_reasoning_effort"] = reasoning_effort
|
||||
|
||||
client = self._get_gigachat_client(target, timeout=timeout)
|
||||
|
||||
|
|
@ -333,4 +335,5 @@ class _GigaChatLaneMixin:
|
|||
|
||||
usage["processing"] = processing_receipt(
|
||||
"gigachat", usage, requested=target["processing_preference"])
|
||||
attach_processing_receipt(target, usage)
|
||||
return message, usage
|
||||
|
|
|
|||
|
|
@ -276,18 +276,23 @@ class _LocalLaneMixin:
|
|||
max_tokens: int, tool_choice: str, timeout: Optional[float] = None,
|
||||
processing_preference: Optional[str] = None,
|
||||
context_mode: Optional[str] = None,
|
||||
reasoning_effort: Optional[str] = None,
|
||||
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
||||
"""Send exactly the previously prepared complete local candidate."""
|
||||
client = self._get_local_client()
|
||||
local_target, candidate = self._build_local_candidate(
|
||||
messages, tools, max_tokens, tool_choice, timeout, processing_preference, context_mode)
|
||||
# The shared finalizer removes SDK transport options from the measured
|
||||
# and sealed payload. Preserve the builder's positive caller override
|
||||
# separately for the actual send, including chat_async's local thread.
|
||||
transport_kwargs = {"timeout": candidate["timeout"]} if "timeout" in candidate else {}
|
||||
local_target["requested_reasoning_effort"] = reasoning_effort
|
||||
from ouroboros.send_clock import stamp_clock_note
|
||||
|
||||
# ONE clock line per call, sampled before both finalizations: the serving
|
||||
# instance measures exactly the bytes that are then sealed and sent.
|
||||
candidate = self._finalize_local_candidate(local_target, stamp_clock_note(candidate))
|
||||
clean_tools = candidate.get("tools")
|
||||
preference = local_target["processing_preference"]
|
||||
# ONE physical attempt per call. Re-sending here spent the caller's
|
||||
# physical-attempt budget without the caller authorising it, so a
|
||||
# transient local failure now surfaces to the single retry policy that
|
||||
|
|
@ -311,7 +316,7 @@ class _LocalLaneMixin:
|
|||
|
||||
resp = _execute_candidate(
|
||||
request,
|
||||
lambda: client.chat.completions.create(**candidate),
|
||||
lambda: client.chat.completions.create(**candidate, **transport_kwargs),
|
||||
check_instance,
|
||||
)
|
||||
except UsageAccountingError:
|
||||
|
|
@ -354,8 +359,7 @@ class _LocalLaneMixin:
|
|||
finish_reason = first_choice.get("finish_reason")
|
||||
if isinstance(finish_reason, str) and finish_reason.strip():
|
||||
usage["response_finish_reason"] = finish_reason.strip()[:64]
|
||||
if preference:
|
||||
from ouroboros._usage_response import processing_receipt
|
||||
from ouroboros.llm_attempt import attach_processing_receipt
|
||||
|
||||
usage["processing"] = processing_receipt("local", usage, requested=preference)
|
||||
attach_processing_receipt(local_target, usage)
|
||||
return msg, usage
|
||||
|
|
|
|||
|
|
@ -120,6 +120,7 @@ class _OpenAICompatibleLaneMixin:
|
|||
bypass_response_cache: bool = False,
|
||||
stream: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
target["requested_reasoning_effort"] = normalize_reasoning_effort(reasoning_effort)
|
||||
messages = self._normalize_system_message_placement(messages)
|
||||
resolved_model = str(target.get("resolved_model") or "")
|
||||
provider = str(target.get("provider") or "")
|
||||
|
|
|
|||
|
|
@ -1431,7 +1431,9 @@ def call_llm_with_retry(
|
|||
)
|
||||
host_route = usage.get("model_role_route") or {}
|
||||
model, use_local = host_route.get("model", model), host_route.get("use_local", use_local)
|
||||
accumulated_usage["_model_route"], accumulated_usage["_options"], accumulated_usage["_model_substitutions"] = dict((usage.get("claudexor") or {}).get("route") or {}), ({key: (usage.get("claudexor") or {}).get(key) for key in ("requested_options", "applied_options", "options_honored", "route")} if usage.get("claudexor") else {}), ((usage.get("claudexor") or {}).get("substituted") or [])
|
||||
model_facts = usage.get("claudexor") or {}
|
||||
accumulated_usage["_model_route"] = dict(model_facts.get("route") or {})
|
||||
accumulated_usage["_model_substitutions"] = model_facts.get("substituted") or []
|
||||
context_fit_event_fields = _context_fit_event_fields(accumulated_usage) if physical_context is not None else {}
|
||||
_take_custom_receipts(usage, msg, accumulated_usage)
|
||||
for stale in ("_last_llm_error", "_last_llm_error_kind", "_last_llm_retry_same_request",
|
||||
|
|
@ -1517,10 +1519,7 @@ def call_llm_with_retry(
|
|||
for stale in ("execution_status", "result_status", "reason_code", RETRY_WALL_EXHAUSTED_KEY, TRANSPORT_DEATHS_KEY):
|
||||
accumulated_usage.pop(stale, None) # a USABLE response closes the round's repeat record
|
||||
accumulated_usage["rounds"] = accumulated_usage.get("rounds", 0) + 1
|
||||
prompt_tokens = int(usage.get("prompt_tokens") or 0)
|
||||
completion_tokens = int(usage.get("completion_tokens") or 0)
|
||||
cached_tokens = int(usage.get("cached_tokens") or 0)
|
||||
cache_write_tokens = int(usage.get("cache_write_tokens") or 0)
|
||||
prompt_cache_ttl, cache_hit_rate, cache_cold_restart, gap_since_prev_round_sec = (
|
||||
_record_round_cache_facts(accumulated_usage, usage, round_idx=round_idx))
|
||||
_round_event = {
|
||||
|
|
@ -1531,13 +1530,14 @@ def call_llm_with_retry(
|
|||
"llm_call_id": llm_call_id,
|
||||
"round": round_idx, "model": display_model,
|
||||
"reasoning_effort": effort,
|
||||
**{key: usage[key] for key in ("effort", "effort_resolution", "request_wire", "claudexor") if key in usage},
|
||||
"provider": provider,
|
||||
"source": "loop",
|
||||
"model_category": infer_model_category(display_model),
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": completion_tokens,
|
||||
"prompt_tokens": int(usage.get("prompt_tokens") or 0),
|
||||
"completion_tokens": int(usage.get("completion_tokens") or 0),
|
||||
"cached_tokens": (cached_tokens if usage.get("cached_tokens") is not None else None),
|
||||
"cache_write_tokens": cache_write_tokens,
|
||||
"cache_write_tokens": int(usage.get("cache_write_tokens") or 0),
|
||||
"prompt_cache_ttl": prompt_cache_ttl,
|
||||
"cache_hit_rate": cache_hit_rate,
|
||||
"cache_cold_restart": cache_cold_restart,
|
||||
|
|
@ -1558,10 +1558,9 @@ def call_llm_with_retry(
|
|||
"task_attempt": task_attempt,
|
||||
"attempt": attempt + 1,
|
||||
"model": display_model,
|
||||
"reasoning_effort": effort,
|
||||
**{key: _round_event[key] for key in (
|
||||
"prompt_tokens", "completion_tokens", "cached_tokens", "cache_write_tokens", "prompt_cache_ttl")},
|
||||
"cost_usd": cost,
|
||||
"reasoning_effort", "cost_usd", "prompt_tokens", "completion_tokens", "cached_tokens",
|
||||
"cache_write_tokens", "prompt_cache_ttl", "effort", "effort_resolution", "request_wire", "claudexor") if key in _round_event},
|
||||
"response_kind": "tool_calls" if tool_calls else "message",
|
||||
"tool_call_count": len(tool_calls),
|
||||
"has_text": bool(content and str(content).strip()),
|
||||
|
|
|
|||
|
|
@ -661,7 +661,7 @@ def _dispatch_round_model(
|
|||
) -> Tuple[Any, float]:
|
||||
from ouroboros.model_wait import current_model_wait
|
||||
from ouroboros.loop_transport import (
|
||||
emit_model_effort_mismatch, emit_model_substitution,
|
||||
emit_model_substitution,
|
||||
managed_transport_continuation, transport_repeat_stop_requested,
|
||||
)
|
||||
from ouroboros.owner_mailbox import OwnerMailboxPeek
|
||||
|
|
@ -745,8 +745,6 @@ def _dispatch_round_model(
|
|||
use_local=ctx.active_use_local, preferred_mode=ctx.active_context_mode,
|
||||
tool_schemas=ctx.tool_schemas, model_role=role, model_route=observed,
|
||||
credential_profile_id=(waiter.overrides.get(role, {}).get("model_account_override") if waiter else None))
|
||||
emit_model_effort_mismatch(ctx.accumulated_usage, task_id=ctx.task_id,
|
||||
emit_progress=getattr(ctx, "emit_progress", None))
|
||||
emit_model_substitution(ctx.accumulated_usage, task_id=ctx.task_id,
|
||||
emit_progress=getattr(ctx, "emit_progress", None))
|
||||
call = ctx.accumulated_usage.get("_last_llm_call_meta")
|
||||
|
|
|
|||
|
|
@ -844,42 +844,6 @@ def provider_failure_hint(accumulated_usage: Dict[str, Any]) -> str:
|
|||
return f" Last provider error: {detail}"
|
||||
|
||||
|
||||
def emit_model_effort_mismatch(
|
||||
accumulated_usage: Dict[str, Any], *, task_id: str, emit_progress: Optional[Callable[..., None]],
|
||||
) -> None:
|
||||
"""Disclose an engine-applied reasoning-effort change once per task and model.
|
||||
|
||||
One line per (task, model), never per round: a second mismatch on the same
|
||||
model in the same task stays in the durable usage rows only. The options
|
||||
must belong to the route the record now names: an error round rewrites
|
||||
`_model_route` from its own failure, and that model must never inherit an
|
||||
earlier route's applied options. The durable state stays generic over every
|
||||
submitted option; this line speaks only for the thinking horizon, so an
|
||||
engine that echoes another option differently never reaches the owner as an
|
||||
effort claim.
|
||||
"""
|
||||
options = accumulated_usage.get("_options")
|
||||
route = accumulated_usage.get("_model_route") or {}
|
||||
model = str(route.get("model") or "")
|
||||
notified = accumulated_usage.setdefault("_options_mismatch_notified", [])
|
||||
if (emit_progress is None or not isinstance(options, dict)
|
||||
or options.get("options_honored") != "mismatch" or model in notified
|
||||
or (options.get("route") or {}) != route):
|
||||
return
|
||||
requested_effort = (options.get("requested_options") or {}).get("reasoningEffort")
|
||||
applied_effort = (options.get("applied_options") or {}).get("reasoningEffort")
|
||||
if requested_effort is None or applied_effort is None or requested_effort == applied_effort:
|
||||
return
|
||||
account = str(route.get("credentialProfileId") or "")
|
||||
notified.append(model)
|
||||
emit_progress(
|
||||
f"⚠️ Claudexor served at {applied_effort} effort while {requested_effort} was requested"
|
||||
f"{f' (Claudexor account {account})' if account else ''}.",
|
||||
incident={"task_incident": "model_effort_mismatch",
|
||||
"toast_once": ":".join(part for part in (task_id, "model_effort_mismatch", model) if part)},
|
||||
)
|
||||
|
||||
|
||||
# What the host KNOWS it did. It asks again and names no account; which account
|
||||
# answers the redo is the engine's choice, so no wording here may claim the
|
||||
# round moved (architecture: rotation is possible, not guaranteed).
|
||||
|
|
|
|||
|
|
@ -90,6 +90,7 @@ class WireUsageDisclosure:
|
|||
ladder_ordinal: int
|
||||
applied_actions: Tuple[Mapping[str, Any], ...]
|
||||
task_local: bool = False
|
||||
original_requested_effort: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_candidate(
|
||||
|
|
@ -115,6 +116,7 @@ class WireUsageDisclosure:
|
|||
ladder_ordinal=candidate.ladder_ordinal,
|
||||
applied_actions=candidate.disclosed_actions(),
|
||||
task_local=candidate.task_local,
|
||||
original_requested_effort=(physical_attempt.effort or {}).get("requested"),
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
|
|
@ -156,6 +158,11 @@ class WireUsageDisclosure:
|
|||
return {
|
||||
"requested_effort": self.requested_effort,
|
||||
"applied_effort": self.applied_effort,
|
||||
"original_requested_effort": self.original_requested_effort,
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": None,
|
||||
"reported_effort_source": None,
|
||||
"requested_tool_dialect": self.requested_tool_dialect,
|
||||
"applied_tool_dialect": self.applied_tool_dialect,
|
||||
"reason_code": self.reason_code,
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import contextvars
|
|||
import copy
|
||||
import functools
|
||||
import inspect
|
||||
import re
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any, Dict, Iterator, Mapping, Optional, Sequence, Tuple
|
||||
|
||||
|
|
@ -84,6 +85,8 @@ class _WireCallState:
|
|||
settled: Optional[Tuple[_RegisteredCandidate, PhysicalAttemptCapture]] = None
|
||||
metadata_drop_fields: Tuple[str, ...] = ()
|
||||
disclosures: Tuple[Mapping[str, Any], ...] = ()
|
||||
physical_payload: Optional[Mapping[str, Any]] = None
|
||||
logical_payload_sha256: str = ""
|
||||
|
||||
|
||||
_WIRE_CALL_STATE: contextvars.ContextVar[_WireCallState] = contextvars.ContextVar(
|
||||
|
|
@ -128,6 +131,7 @@ def _safe_target(target: Mapping[str, Any]) -> Dict[str, Any]:
|
|||
"provider", "resolved_model", "usage_model", "base_url", "contract_headers",
|
||||
"processing_preference",
|
||||
"processing_native_origin",
|
||||
"requested_reasoning_effort",
|
||||
)
|
||||
if target.get(key) is not None
|
||||
}
|
||||
|
|
@ -138,6 +142,7 @@ def register_wire_candidate(
|
|||
*,
|
||||
source_payload: Mapping[str, Any],
|
||||
target: Mapping[str, Any],
|
||||
logical_payload: Optional[Mapping[str, Any]] = None,
|
||||
) -> None:
|
||||
"""Register a factory-bound candidate (the Phase-2A custom seam)."""
|
||||
if not isinstance(candidate, WireCandidateManifest):
|
||||
|
|
@ -159,6 +164,9 @@ def register_wire_candidate(
|
|||
registered=(*kept, registered),
|
||||
current=registered,
|
||||
settled=None,
|
||||
physical_payload=candidate.physical_payload(),
|
||||
logical_payload_sha256=physical_candidate_sha256(
|
||||
logical_payload if logical_payload is not None else source_payload),
|
||||
))
|
||||
|
||||
|
||||
|
|
@ -517,6 +525,7 @@ def prepare_wire_payload_for_send(
|
|||
payload: Mapping[str, Any],
|
||||
*,
|
||||
api_surface: str,
|
||||
logical_payload: Optional[Mapping[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Apply frozen evidence and bind the exact payload immediately before send."""
|
||||
detached = copy.deepcopy(dict(payload))
|
||||
|
|
@ -528,7 +537,8 @@ def prepare_wire_payload_for_send(
|
|||
None,
|
||||
)
|
||||
if existing is not None:
|
||||
_WIRE_CALL_STATE.set(replace(state, current=existing, settled=None))
|
||||
register_wire_candidate(existing.candidate, source_payload=existing.source_payload,
|
||||
target=existing.target, logical_payload=logical_payload)
|
||||
return existing.candidate.physical_payload()
|
||||
try:
|
||||
candidate = (
|
||||
|
|
@ -560,9 +570,15 @@ def prepare_wire_payload_for_send(
|
|||
for field in state.metadata_drop_fields:
|
||||
if field in _NON_REASONING_OPTIONAL_FIELDS:
|
||||
detached.pop(field, None)
|
||||
_WIRE_CALL_STATE.set(replace(state, current=None, settled=None))
|
||||
# Carrierless sends still own exact physical/logical identity. They can
|
||||
# repair optional fields without learning a durable wire contract.
|
||||
_WIRE_CALL_STATE.set(replace(state, current=None, settled=None,
|
||||
physical_payload=copy.deepcopy(detached),
|
||||
logical_payload_sha256=physical_candidate_sha256(
|
||||
logical_payload if logical_payload is not None else payload)))
|
||||
return detached
|
||||
register_wire_candidate(candidate, source_payload=detached, target=target)
|
||||
register_wire_candidate(candidate, source_payload=detached, target=target,
|
||||
logical_payload=logical_payload)
|
||||
return candidate.physical_payload()
|
||||
|
||||
|
||||
|
|
@ -575,7 +591,7 @@ def note_wire_send_succeeded(capture: Any) -> None:
|
|||
state = _WIRE_CALL_STATE.get()
|
||||
if state.current is None or not isinstance(capture, PhysicalAttemptCapture):
|
||||
return
|
||||
if capture.candidate_raw_sha256 != state.current.candidate.candidate_sha256:
|
||||
if getattr(capture, "candidate_raw_sha256", None) != state.current.candidate.candidate_sha256:
|
||||
return
|
||||
_WIRE_CALL_STATE.set(replace(state, settled=(state.current, capture)))
|
||||
|
||||
|
|
@ -585,27 +601,101 @@ def note_wire_send_failed() -> None:
|
|||
_WIRE_CALL_STATE.set(replace(state, settled=None))
|
||||
|
||||
|
||||
def _status_and_message_from_exception(exc: BaseException) -> Tuple[Optional[int], str]:
|
||||
response = getattr(exc, "response", None)
|
||||
status = getattr(exc, "status_code", None) or getattr(response, "status_code", None)
|
||||
try:
|
||||
status = int(status) if status is not None else None
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
status = None
|
||||
body = getattr(exc, "body", None)
|
||||
@dataclass(frozen=True)
|
||||
class _WireRejection:
|
||||
status: Optional[int]
|
||||
message: str
|
||||
param: str = ""
|
||||
code: str = ""
|
||||
allowed: Tuple[str, ...] = ()
|
||||
value_constraint: bool = False
|
||||
rejected_value: Optional[str] = None
|
||||
|
||||
|
||||
def _wire_rejection(error: Any) -> _WireRejection:
|
||||
"""Keep structured constraints through exception/body projection alike.
|
||||
|
||||
Only the positive enum clause supplies alternatives: quotes elsewhere can
|
||||
name rejected values. A structured parameter, when present, owns binding.
|
||||
"""
|
||||
response = getattr(error, "response", None)
|
||||
body = error if isinstance(error, Mapping) else getattr(error, "body", None)
|
||||
if body is None and response is not None and callable(getattr(response, "json", None)):
|
||||
try:
|
||||
body = response.json()
|
||||
except Exception:
|
||||
body = None
|
||||
error = body.get("error") if isinstance(body, Mapping) else None
|
||||
message = (
|
||||
str(error.get("message") or "")
|
||||
if isinstance(error, Mapping)
|
||||
else str((body or {}).get("message") or "") if isinstance(body, Mapping)
|
||||
else ""
|
||||
)
|
||||
return status, message or str(exc or "")
|
||||
body = body if isinstance(body, Mapping) else {}
|
||||
node = body.get("error") if isinstance(body.get("error"), Mapping) else body
|
||||
status = (getattr(error, "status_code", None) or getattr(response, "status_code", None)
|
||||
or body.get("status_code", body.get("status", body.get("code"))))
|
||||
try:
|
||||
status = int(status) if status is not None else None
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
status = None
|
||||
message = str(node.get("message") or str(error or "")).lower()
|
||||
param = str(node.get("param") or getattr(error, "param", "") or "").lower()
|
||||
code = str(node.get("code") or getattr(error, "code", "") or "").lower()
|
||||
values = node.get("allowed_values", node.get("enum"))
|
||||
allowed = tuple(v.lower() for v in values if isinstance(v, str)) if isinstance(values, list) else ()
|
||||
value_constraint = isinstance(values, list)
|
||||
if not allowed:
|
||||
clause = re.search(r"\b(?:expected|must be) one of\b|\b(?:allowed|supported) values\b|\binput should be\b", message)
|
||||
if clause:
|
||||
value_constraint = True
|
||||
prefix = message[:clause.start()]
|
||||
if re.search(r"\b(?:not|no|unsupported|disallowed)\s*$", prefix):
|
||||
return _WireRejection(status, message, param, code, value_constraint=True)
|
||||
if not param:
|
||||
fields = {*OPTIONAL_REQUEST_FIELDS, "reasoning", "reasoning.effort",
|
||||
"extra_body.reasoning", "extra_body.reasoning.effort", "output_config.effort", "thinking.type"}
|
||||
named = re.findall(r"[a-z_][a-z0-9_]*(?:\.[a-z_][a-z0-9_]*)*", prefix)
|
||||
param = next((name for name in reversed(named) if name in fields), "")
|
||||
# Consume the list itself, stopping at prose (including a negative
|
||||
# clause). This accepts quoted JSON/Zod enums and plain comma lists.
|
||||
# Each item is quoted like its predecessor: a quoted 'or' stays a
|
||||
# value and bare prose ends the list. A conjunction ("a, b, or c")
|
||||
# joins only the last item of a comma list; what follows is prose.
|
||||
tail = message[clause.end():].lstrip(" :[=({")
|
||||
tail = re.sub(r"^(?:are|is)\b", "", tail).lstrip(" :[=({")
|
||||
tokens, lead, final, piped, comma_list = [], "[\"']?", False, False, False
|
||||
while match := re.match(rf"({lead})([a-z][a-z0-9_-]*)[\"']?", tail):
|
||||
rest = tail[match.end():]
|
||||
# A token named in an explicit refusal after the positive list
|
||||
# is not itself an advertised option, even after a comma or
|
||||
# the first conjunction ("low, high and xhigh is unsupported").
|
||||
if re.match(
|
||||
r"\s*\(?\s*(?:(?:is|are)\s+not\s+(?:supported|allowed|available|permitted)\b"
|
||||
r"|(?:isn't|aren't)\s+(?:supported|allowed|available|permitted)\b"
|
||||
r"|not\s+(?:supported|allowed|available|permitted)\b"
|
||||
r"|(?:(?:is|are)\s+)?(?:unsupported|disallowed|unavailable|requires?)\b)", rest,
|
||||
):
|
||||
break
|
||||
tokens.append(match.group(2))
|
||||
tail = rest
|
||||
lead = match.group(1) or lead
|
||||
conjunction = not piped and re.match(rf"\s*,?\s*\b(?:or|and)\b\s*(?={lead}[a-z])", tail)
|
||||
separator = conjunction or re.match(r"\s*[,|]\s*", tail)
|
||||
if final or not separator:
|
||||
break
|
||||
comma_list = comma_list or "," in separator.group()
|
||||
final, piped = bool(conjunction and comma_list), piped or "|" in separator.group()
|
||||
tail = tail[separator.end():]
|
||||
if param and tokens:
|
||||
allowed = tuple(tokens)
|
||||
rejected = node.get("value")
|
||||
if rejected is None:
|
||||
from ouroboros.config import EFFORT_SCALE
|
||||
|
||||
scalar = re.search(r"\b(?:value|tier)\s+['\"]?([a-z][a-z0-9_-]*)", message)
|
||||
rejected = scalar.group(1) if scalar and scalar.group(1) in EFFORT_SCALE else None
|
||||
return _WireRejection(status, message, param, code, allowed, value_constraint,
|
||||
str(rejected).lower() if rejected is not None else None)
|
||||
|
||||
|
||||
def _status_and_message_from_exception(exc: BaseException) -> Tuple[Optional[int], str]:
|
||||
evidence = _wire_rejection(exc)
|
||||
return evidence.status, evidence.message
|
||||
|
||||
|
||||
def _context_overflow_evidence(error: Any) -> bool:
|
||||
|
|
@ -682,9 +772,9 @@ def _value_evidence(message: str) -> bool:
|
|||
def _classify_action(
|
||||
registered: _RegisteredCandidate,
|
||||
*,
|
||||
status_code: Optional[int],
|
||||
message: str,
|
||||
evidence: _WireRejection,
|
||||
) -> Optional[PendingWireAction]:
|
||||
status_code, message = evidence.status, evidence.message
|
||||
if (
|
||||
status_code is None
|
||||
or not 400 <= status_code < 500
|
||||
|
|
@ -692,27 +782,53 @@ def _classify_action(
|
|||
):
|
||||
return None
|
||||
low = str(message or "").lower()
|
||||
if not low or not any(marker in low for marker in (
|
||||
if any(token in evidence.code for token in ("quota", "rate_limit", "billing", "auth", "context", "policy")):
|
||||
return None
|
||||
if not low or not (evidence.value_constraint or any(marker in low for marker in (
|
||||
*_VALUE_REJECTION_MARKERS,
|
||||
*_CAPABILITY_REJECTION_MARKERS,
|
||||
*_MANDATORY_MARKERS,
|
||||
)):
|
||||
))):
|
||||
return None
|
||||
candidate = registered.candidate
|
||||
payload = candidate.physical_payload()
|
||||
profile = candidate.accepted_profile
|
||||
current_effort = payload_effort(payload)
|
||||
# Rejecting the disabled carrier's native TYPE is not rejecting an effort
|
||||
# tier. Preserve the bounded omission repair without dropping a valid
|
||||
# carrier when only output_config.effort (or a scalar tier) was refused.
|
||||
if (profile.provider == "anthropic" and profile.reasoning_carrier == "anthropic.disabled"
|
||||
and evidence.param == "thinking.type" and evidence.allowed
|
||||
and "disabled" not in evidence.allowed
|
||||
and evidence.rejected_value in {None, "disabled"}):
|
||||
return PendingWireAction(profile, {
|
||||
"kind": "drop_field", "fields": ["thinking"],
|
||||
"reason_code": "provider_unsupported_field",
|
||||
})
|
||||
error_tokens = _error_tokens(low)
|
||||
effort_implicated = any(
|
||||
marker in low
|
||||
for marker in ("reasoning_effort", "reasoning.effort", "reasoning", "effort", "thinking", "output_config")
|
||||
)
|
||||
value_path = {
|
||||
"reasoning_effort": "reasoning_effort",
|
||||
NESTED_REASONING_FIELD: "extra_body.reasoning.effort",
|
||||
"anthropic.adaptive": "output_config.effort",
|
||||
"anthropic.disabled": "thinking",
|
||||
}.get(profile.reasoning_carrier, "")
|
||||
aliases = {value_path, value_path.removeprefix("extra_body."),
|
||||
value_path.removeprefix("extra_body.").split(".")[0]}
|
||||
if value_path:
|
||||
aliases.update({"reasoning", "effort"})
|
||||
aliases.discard("")
|
||||
named_paths = set(re.findall(r"[a-z_][a-z0-9_]*(?:\.[a-z_][a-z0-9_]*)*", low))
|
||||
effort_implicated = evidence.param in aliases if evidence.param else bool(aliases & named_paths)
|
||||
if effort_implicated and evidence.rejected_value is not None and evidence.rejected_value != current_effort:
|
||||
return None
|
||||
named_effort_value = _names_exact_effort_value(low, current_effort)
|
||||
value_implicated = _value_evidence(low)
|
||||
value_implicated = evidence.value_constraint or _value_evidence(low) or evidence.code in {"invalid_value", "invalid_enum_value", "invalid_option"}
|
||||
if (
|
||||
profile.provider == "anthropic"
|
||||
and profile.reasoning_carrier == "anthropic.disabled"
|
||||
and effort_implicated
|
||||
and not value_implicated
|
||||
and not any(marker in low for marker in _MANDATORY_MARKERS)
|
||||
):
|
||||
return PendingWireAction(profile, {
|
||||
"kind": "drop_field",
|
||||
|
|
@ -723,6 +839,7 @@ def _classify_action(
|
|||
effort_implicated
|
||||
and current_effort in {"none", "minimal"}
|
||||
and any(marker in low for marker in _MANDATORY_MARKERS)
|
||||
and not evidence.value_constraint
|
||||
):
|
||||
return PendingWireAction(profile, {
|
||||
"kind": "set_value",
|
||||
|
|
@ -732,18 +849,22 @@ def _classify_action(
|
|||
"to": "low",
|
||||
"reason_code": "provider_required_reasoning",
|
||||
})
|
||||
if effort_implicated and named_effort_value:
|
||||
from ouroboros.config import EFFORT_SCALE, effort_one_step_down, effort_rank
|
||||
# QUOTED tiers inside [low, current) prescribe — even a negatively-quoted one (accepted FP); prose walks one rung.
|
||||
prescribed = [t for t in EFFORT_SCALE[effort_rank("low"):max(effort_rank(current_effort), 0)]
|
||||
if f"'{t}'" in low or f'"{t}"' in low]
|
||||
next_effort = prescribed[-1] if prescribed else effort_one_step_down(current_effort)
|
||||
if effort_rank(next_effort) >= effort_rank("low") and next_effort != current_effort:
|
||||
value_path = {
|
||||
"reasoning_effort": "reasoning_effort",
|
||||
NESTED_REASONING_FIELD: "extra_body.reasoning.effort",
|
||||
"anthropic.adaptive": "output_config.effort",
|
||||
}.get(profile.reasoning_carrier, "")
|
||||
if effort_implicated and (named_effort_value or evidence.allowed):
|
||||
from ouroboros.config import effort_one_step_down, effort_rank
|
||||
if current_effort in evidence.allowed:
|
||||
return None # This constraint does not reject the candidate's value.
|
||||
# Any positively advertised reasoning tier is comparable, minimal
|
||||
# included; "none" disables reasoning and unknown tokens have no rank.
|
||||
supported = [t for t in evidence.allowed if effort_rank(t) >= effort_rank("minimal")]
|
||||
prescribed = [t for t in supported if effort_rank(t) < effort_rank(current_effort)]
|
||||
# A positive enum owns the minimum when every supported tier is higher;
|
||||
# mandatory wording alone retains the legacy low-floor behavior above.
|
||||
next_effort = (max(prescribed, key=effort_rank) if prescribed else
|
||||
min(supported, key=effort_rank) if supported else
|
||||
"" if evidence.allowed else effort_one_step_down(current_effort))
|
||||
# An unadvertised prose walk still stops at low.
|
||||
floor = "minimal" if supported else "low"
|
||||
if effort_rank(next_effort) >= effort_rank(floor) and next_effort != current_effort:
|
||||
exact_profile = _profile(
|
||||
registered.target,
|
||||
payload,
|
||||
|
|
@ -767,7 +888,8 @@ def _classify_action(
|
|||
compact = low.replace(".", "_")
|
||||
for field in (*OPTIONAL_REQUEST_FIELDS, NESTED_REASONING_FIELD):
|
||||
aliases = {field, field.replace(".", "_"), field.split(".")[-1]}
|
||||
implicated = (field in error_tokens if field in {"stream", "stream_options"}
|
||||
implicated = (evidence.param in aliases if evidence.param else
|
||||
field in error_tokens if field in {"stream", "stream_options"}
|
||||
else any(alias in low or alias in compact for alias in aliases))
|
||||
if _field_is_present(payload, field) and implicated:
|
||||
if value_implicated or (
|
||||
|
|
@ -787,7 +909,7 @@ def _classify_action(
|
|||
})
|
||||
|
||||
|
||||
def _plan_retry(status_code: Optional[int], message: str) -> Optional[Dict[str, Any]]:
|
||||
def _plan_retry(evidence: _WireRejection) -> Optional[Dict[str, Any]]:
|
||||
state = _WIRE_CALL_STATE.get()
|
||||
registered = state.current
|
||||
if registered is None:
|
||||
|
|
@ -795,8 +917,7 @@ def _plan_retry(status_code: Optional[int], message: str) -> Optional[Dict[str,
|
|||
try:
|
||||
pending = _classify_action(
|
||||
registered,
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
evidence=evidence,
|
||||
)
|
||||
if pending is None:
|
||||
return None
|
||||
|
|
@ -883,8 +1004,11 @@ def _plan_direct_dialect_retry(
|
|||
def plan_wire_retry_from_exception(exc: BaseException) -> Optional[Dict[str, Any]]:
|
||||
if _context_overflow_evidence(exc):
|
||||
return None
|
||||
status, message = _status_and_message_from_exception(exc)
|
||||
return _plan_retry(status, message)
|
||||
capture = getattr(exc, "physical_attempt_capture", None)
|
||||
candidate = current_wire_candidate()
|
||||
if capture is not None and (candidate is None or getattr(capture, "candidate_raw_sha256", None) != candidate.candidate_sha256):
|
||||
return None
|
||||
return _plan_retry(_wire_rejection(exc))
|
||||
|
||||
|
||||
def plan_nonlearning_optional_retry(
|
||||
|
|
@ -947,12 +1071,7 @@ def plan_wire_retry_from_body_error(error: Any) -> Optional[Dict[str, Any]]:
|
|||
return None
|
||||
if _context_overflow_evidence(error):
|
||||
return None
|
||||
code = error.get("status_code", error.get("status", error.get("code")))
|
||||
try:
|
||||
status = int(code) if code is not None else None
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
status = None
|
||||
return _plan_retry(status, str(error.get("message") or ""))
|
||||
return _plan_retry(_wire_rejection(error))
|
||||
|
||||
|
||||
def _processing_retry(payload: Mapping[str, Any], error: Any,
|
||||
|
|
@ -970,7 +1089,7 @@ def _processing_retry(payload: Mapping[str, Any], error: Any,
|
|||
if route.get("processing_native_origin") != "preference":
|
||||
return None
|
||||
current = registered.candidate.physical_payload() if registered is not None else copy.deepcopy(dict(payload))
|
||||
if capture.candidate_raw_sha256 != physical_candidate_sha256(current):
|
||||
if getattr(capture, "candidate_raw_sha256", None) != physical_candidate_sha256(current):
|
||||
return None
|
||||
if provider in {"openai", "openrouter"}:
|
||||
field, standard = "service_tier", "default"
|
||||
|
|
@ -1010,7 +1129,19 @@ def plan_next_wire_retry(
|
|||
"""One exception/body-parity entrypoint for the bounded transport drivers."""
|
||||
if getattr(error, "stream_incomplete", False):
|
||||
return None
|
||||
processing = _processing_retry(payload, error, target)
|
||||
state = _WIRE_CALL_STATE.get()
|
||||
registered, physical = state.current, state.physical_payload
|
||||
digest = physical_candidate_sha256(physical) if physical is not None else ""
|
||||
if physical is not None and physical_candidate_sha256(payload) not in {
|
||||
digest, state.logical_payload_sha256,
|
||||
physical_candidate_sha256(registered.source_payload) if registered is not None else "",
|
||||
}:
|
||||
return None # Only inputs bound by this attempt's preparation seam may recover it.
|
||||
capture = getattr(error, "physical_attempt_capture", None)
|
||||
if capture is not None and (not digest or getattr(capture, "candidate_raw_sha256", None) != digest):
|
||||
return None
|
||||
current = physical if physical is not None else payload
|
||||
processing = _processing_retry(current, error, target)
|
||||
if processing is not None:
|
||||
return processing
|
||||
planned = (
|
||||
|
|
@ -1028,7 +1159,7 @@ def plan_next_wire_retry(
|
|||
if dialect_retry is not None:
|
||||
return dialect_retry
|
||||
return plan_nonlearning_optional_retry(
|
||||
payload,
|
||||
current,
|
||||
error=error,
|
||||
body_error=body_error,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1390,6 +1390,8 @@ class AgentSessionReviewExecutor(ReviewSlotExecutor):
|
|||
"delegated_route": effective_routes[0] if len(effective_routes) == 1 else "",
|
||||
"requested_route": facts["route_id"],
|
||||
"observed_attempt": facts.get("observed_attempt") or {},
|
||||
**({"effort_resolution": facts["observed_attempt"]["effort_resolution"]}
|
||||
if isinstance((facts.get("observed_attempt") or {}).get("effort_resolution"), dict) else {}),
|
||||
"review_thread_id": str(facts.get("thread_id") or ""),
|
||||
"review_turn_id": str(facts.get("turn_id") or ""),
|
||||
"review_thread_receipt": facts.get("thread_receipt") or {},
|
||||
|
|
|
|||
|
|
@ -982,6 +982,8 @@ def _attach_only_run(root: Any, request: Any, slot: Any, operation_id: str, run_
|
|||
usage = {"provider": "claudexor", "delegated_run_started": True, "delegated_run_id": run_id,
|
||||
"collection": "attach_only_observation", "cost": None, "run_state": state}
|
||||
observed = final_attempt_facts(detail, run_id)
|
||||
if isinstance(observed.get("effort_resolution"), dict):
|
||||
usage["effort_resolution"] = observed["effort_resolution"]
|
||||
# Requested identity comes only from this paid invocation, never live settings
|
||||
# or the summary's request echoes. Absent final-attempt facts stay unknown.
|
||||
routes = (invocation.get("request") or {}).get("harnesses") or []
|
||||
|
|
|
|||
|
|
@ -1218,15 +1218,14 @@ def record_reviewer_slot_executions(surface: str, actors: Any, slots_by_id: Dict
|
|||
# up as the applied one. An api row's sent model IS its applied one.
|
||||
"model": (str(usage.get("resolved_model") or "") if session
|
||||
else str(getattr(slot, "model", "") or "")),
|
||||
# No "effort": no APPLIED effort exists anywhere upstream (no
|
||||
# applied/resolved effort in any receipt or telemetry), so the only
|
||||
# value available is the REQUESTED one — already recorded below. Echoing
|
||||
# it here dressed the request up as the applied value, the exact thing
|
||||
# the model rule above forbids.
|
||||
# No scalar "effort": keep host-send evidence and the engine's
|
||||
# sourced report separate from the requested row below.
|
||||
"verdict_method": str(usage.get("verdict_method") or ""),
|
||||
}
|
||||
if isinstance(usage.get("processing"), dict):
|
||||
effective["processing"] = dict(usage["processing"])
|
||||
if isinstance(usage.get("effort_resolution"), dict):
|
||||
effective["effort_resolution"] = dict(usage["effort_resolution"])
|
||||
# D29 applied account/access, verbatim from the engine receipt; absent
|
||||
# keys mean the telemetry predates the receipt — shown as absence.
|
||||
if usage.get("applied_profile"):
|
||||
|
|
@ -1252,6 +1251,7 @@ def record_reviewer_slot_executions(surface: str, actors: Any, slots_by_id: Dict
|
|||
"processing_preference": str(getattr(slot, "processing_preference", "") or ""),
|
||||
},
|
||||
"effective": effective,
|
||||
**({"effort": dict(usage["effort"])} if isinstance(usage.get("effort"), dict) else {}),
|
||||
"capability_delta": usage.get("capability_delta") or [],
|
||||
"status": str(getattr(actor, "status", "") or ""),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -123,13 +123,11 @@ BAND_PATHS = {
|
|||
"ouroboros/gateway/extensions.py": "Extensions HTTP surface re-entered the band when the module endpoint moved to the in-memory reviewed bundle (widgets lifecycle 1a); shrink next touch.",
|
||||
"ouroboros/gateway/host_service.py": "The one loopback callback boundary for reviewed skills: token auth, the chat/decision/presence/WS-relay routes and, with #667, the operation read/cancel that joins existing chat, routing, turn and task records; one trust boundary, one module.",
|
||||
"ouroboros/gateway/settings.py": "Shrank INTO the band from 1532 lines: the scope-slot window-floor owner capability-ack path was removed with the scope packet (scope-review-retrieval sprint); shrink-only residue, no new content.",
|
||||
"ouroboros/gateways/claudexor.py": "The existing owned-engine gateway also owns typed model operations and exact-byte resource transfer; no second control client.",
|
||||
"ouroboros/launcher_bootstrap.py": "Native seed version resync keeps manifest parsing and equal-version payload diagnostics with the existing bootstrap owner; no separate loader or overwrite policy.",
|
||||
"ouroboros/loop_acceptance.py": "the local preparation incident rides the acceptance decision: incident ride-along, the exposed-stance finish intent and the forced-rail cause split live beside the decision authority they qualify (#1223-#1224)",
|
||||
"ouroboros/loop_delivery.py": "F6 upstream sync: the delivery-protocol upstream deltas (hold-control literals, trailing-object/fence-aware protocol parsers) folded into the campaign delivery leaf (upstream leaf delivery_protocol.py retired)",
|
||||
"ouroboros/loop_forced_finalization.py": "Forced-finalization rail of the v7 L-B loop split: one cohesive owner for the forced/orphan/absorption path, moved byte-preserving from loop.py (D01 lane).",
|
||||
"ouroboros/loop_model_call.py": "Fallback waits retain the exact quota-refusing route across failures without inventing a second router.",
|
||||
"ouroboros/loop_transport.py": "Entered the band from 981 lines with the #869 unknown-outcome terminal wording: an attempt with no provider outcome now states the real wait it spent and that its cost may be unknown or bounded, beside the existing no-resend fence. The sentence belongs with the episode facts it reads; no new terminal owner, no predicate change. Shrink-only from here.",
|
||||
"ouroboros/marketplace/ouroboroshub.py": "Entered the band from 373 lines: the hubflow sprint added the adopt transaction (eligibility prelude, CAS re-verification, move-aside + state-quintet snapshot, verified rollback with per-step error collection, retention finalize) beside the existing install/update flows (hubflow sprint, adopt-in-ouroboroshub owner decision D4).",
|
||||
"ouroboros/mcp_client.py": "F3.1 typed-organ producer cutover (D05 entry 7b) plus E5+s2r2 (#447): the MCP transport keeps the SDK-owned error bit as a typed ToolResult, follows nextCursor pagination with injective 12-hex slugs, and discloses collision/pagination omissions; grew into the band from 984 lines, shrink-only otherwise.",
|
||||
"ouroboros/memory.py": "ibl-2b09abdadd25: scratchpad content-size cap added alongside the existing block-count cap in append_scratchpad_block's eviction loop",
|
||||
|
|
|
|||
|
|
@ -510,7 +510,7 @@ def _responses_search_candidate(ctx, query: str, model: str, options: dict):
|
|||
)
|
||||
target = {"provider": "openai", "resolved_model": model, "base_url": options["base_url"],
|
||||
"usage_model": model if "/" in model else f"openai/{model}",
|
||||
"processing_preference": options["preference"]}
|
||||
"processing_preference": options["preference"], "requested_reasoning_effort": options["effort"]}
|
||||
payload = {"model": model,
|
||||
"tools": [{"type": "web_search", "search_context_size": options["search_context_size"]}],
|
||||
"reasoning": {"effort": options["effort"]}, "tool_choice": "auto", "input": query,
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ _CAPTURE = _object({
|
|||
"processing_preference": _STRING,
|
||||
"submitted_processing_mode": _STRING,
|
||||
"processing_basis": {"type": ["object", "null"]},
|
||||
"effort": {"type": ["object", "null"]},
|
||||
})
|
||||
_CUSTODY = _object({
|
||||
"operation_id": _STRING, "invocation_id": {"type": "string", "minLength": 1},
|
||||
|
|
|
|||
|
|
@ -276,14 +276,8 @@ class AttemptRequest:
|
|||
physical_context: Optional[PhysicalAttemptContext] = None
|
||||
# Route-locality fact (additive): base_url host is localhost/127.0.0.1/::1 (loopback OpenAI-compatible installs — Ollama / LM Studio / vLLM).
|
||||
route_is_loopback: bool = False
|
||||
# The fit estimator's own token count for this request
|
||||
# (context_fit.estimate_context_prompt_tokens: full projected context, tool
|
||||
# objects and schemas included, images at the billing proxy) — additive,
|
||||
# LAST (frozen dataclass), 0 = producer predates the field. The density
|
||||
# observer MUST calibrate on THIS, the exact quantity measure_main_fit
|
||||
# multiplies, so density lands ≈1.0; `prompt_tokens_estimate` above keeps
|
||||
# the raw base64 basis because budget reservation wants the conservative
|
||||
# over-count (owner decision 3=A: the two consumers intentionally split).
|
||||
# Density uses the fit estimator's count (0 = old producer), while budget
|
||||
# reservation retains the conservative raw-base64 estimate above.
|
||||
prompt_tokens_bounded_estimate: int = 0
|
||||
global_limit_source: str = ""
|
||||
global_limit_revision: Optional[str] = None
|
||||
|
|
@ -293,6 +287,7 @@ class AttemptRequest:
|
|||
# The same canonical candidate without its Main clock line (``send_clock``);
|
||||
# None when the candidate carries none. An identity, never a row field.
|
||||
candidate_clock_free_sha256: Optional[str] = None
|
||||
effort: Optional[Dict[str, Any]] = None
|
||||
@dataclass(frozen=True)
|
||||
class AttemptReservation:
|
||||
attempt_id: str
|
||||
|
|
@ -329,6 +324,7 @@ class PhysicalAttemptCapture:
|
|||
processing_preference: str = ""
|
||||
submitted_processing_mode: str = ""
|
||||
processing_basis: Optional[Dict[str, Any]] = None
|
||||
effort: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
|
|
@ -800,6 +796,7 @@ _CANDIDATE_ROW_FIELDS = (
|
|||
"candidate_context_size_bytes", "candidate_measurement_kind", "physical_context",
|
||||
"candidate_manifest_ref",
|
||||
"processing_preference", "submitted_processing_mode", "processing_basis",
|
||||
"effort",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -931,6 +928,7 @@ def reserve_attempt(request: AttemptRequest) -> AttemptReservation:
|
|||
"candidate_context_size_bytes": request.candidate_context_size_bytes,
|
||||
"candidate_measurement_kind": request.candidate_measurement_kind,
|
||||
"physical_context": asdict(request.physical_context) if request.physical_context else None,
|
||||
**({"effort": copy.deepcopy(request.effort)} if request.effort is not None else {}),
|
||||
**({"processing_preference": request.processing_preference,
|
||||
"submitted_processing_mode": request.submitted_processing_mode,
|
||||
"processing_basis": copy.deepcopy(request.processing_basis)}
|
||||
|
|
@ -1048,6 +1046,7 @@ def record_subscription_session(
|
|||
input_token_usage: Dict[str, Any] | None = None,
|
||||
review_skill: str = "", review_wave_id: str = "", review_slot_id: str = "",
|
||||
attempt_execution: Optional[list[Dict[str, Any]]] = None,
|
||||
effort_resolution: Optional[Dict[str, Any]] = None,
|
||||
) -> str:
|
||||
"""Record one idempotent session; model observation is not session identity.
|
||||
|
||||
|
|
@ -1098,13 +1097,13 @@ def record_subscription_session(
|
|||
"credential_profile_id": str(credential_profile_id or ""),
|
||||
"access_profile": str(access_profile or ""),
|
||||
"session_id_sha256": identity,
|
||||
**({"effort_resolution": copy.deepcopy(effort_resolution)} if isinstance(effort_resolution, dict) else {}),
|
||||
# Present only when the harness reported a complete, valid object.
|
||||
**({"input_token_usage": input_counters} if input_counters is not None else {}),
|
||||
**({"attempt_execution": copy.deepcopy(attempt_execution)}
|
||||
if isinstance(attempt_execution, list) else {}),
|
||||
# CPL-5 lane-level disclosure: a delegated/harness session never hands
|
||||
# the host the final wire bytes, so it carries this typed limit instead
|
||||
# of a fake model_send seal (design note §4, provider_side_transform).
|
||||
# CPL-5: sessions do not hand the host final wire bytes; disclose the
|
||||
# limit (design note §4, provider_side_transform) without a fake seal.
|
||||
"model_send_seal": "unobserved",
|
||||
}
|
||||
return _append_single_settled_row(root, row, comparable=(
|
||||
|
|
@ -1165,11 +1164,11 @@ def _transition(reservation: AttemptReservation, state: str, **fields: Any) -> D
|
|||
if limit is not None and view.exceeds_limit(limit, root_task_id=identity, dispatch=True):
|
||||
raise BudgetExceeded(f"{axis} model budget changed before dispatch", limit_scope=axis,
|
||||
root_task_id=scope.root_task_id)
|
||||
# Carry provenance, including future metadata; event-local reasons and
|
||||
# receipt observations belong only to the transition that supplied them.
|
||||
replaced = {"seq", "ts", "pre_compaction_seq", "settle_reason", "reason"}
|
||||
if state == "settled":
|
||||
replaced.update(("processing", "speed", "service_tier", "cost_basis", "cost_evidence"))
|
||||
replaced.update(("effort", "effort_resolution", "processing", "speed", "service_tier", "cost_basis", "cost_evidence"))
|
||||
if "effort" not in fields and isinstance(current.get("effort"), dict): # Keep candidate facts, not older observations.
|
||||
fields["effort"] = {**current["effort"], "reported": None, "report_source": None}
|
||||
row = {key: value for key, value in current.items() if key not in replaced}
|
||||
row.update(state=state, **fields)
|
||||
appended = view.append(reservation.drive_root, [row])
|
||||
|
|
@ -1299,7 +1298,7 @@ def _settlement_fields(reservation, usage, cost_usd, cost_final) -> Dict[str, An
|
|||
cached_tokens=cached_tokens,
|
||||
cache_write_tokens=cache_write_tokens,
|
||||
prompt_cache_ttl=str(normalized.get("prompt_cache_ttl") or ""),
|
||||
**{key: copy.deepcopy(normalized[key]) for key in ("processing", "speed", "service_tier", "cost_basis", "cost_evidence")
|
||||
**{key: copy.deepcopy(normalized[key]) for key in ("effort", "effort_resolution", "processing", "speed", "service_tier", "cost_basis", "cost_evidence")
|
||||
if key in normalized},
|
||||
)
|
||||
|
||||
|
|
@ -1438,6 +1437,7 @@ def _record_attempt_capture(
|
|||
processing_preference=request.processing_preference,
|
||||
submitted_processing_mode=request.submitted_processing_mode,
|
||||
processing_basis=copy.deepcopy(request.processing_basis),
|
||||
effort=copy.deepcopy(request.effort),
|
||||
)
|
||||
_LAST_PHYSICAL_ATTEMPT.set(capture)
|
||||
if exc is not None:
|
||||
|
|
|
|||
|
|
@ -151,6 +151,8 @@ def _handle_llm_usage(evt: Dict[str, Any], ctx: Any) -> None:
|
|||
**({"web_search_sources": web_search_sources} if isinstance(web_search_sources, list) and web_search_sources else {}),
|
||||
**({"reasoning_pin": reasoning_pin} if isinstance(reasoning_pin, dict) and reasoning_pin else {}),
|
||||
**({"reasoning_effort_clamped": effort_clamped} if isinstance(effort_clamped, dict) and effort_clamped else {}),
|
||||
**{key: usage[key] for key in ("effort", "request_wire", "effort_resolution", "claudexor")
|
||||
if isinstance(usage.get(key), dict)},
|
||||
}
|
||||
_address_ctx(ctx, usage_event)
|
||||
try:
|
||||
|
|
|
|||
9
tests/fixtures/claudexor_effort_resolution.json
vendored
Normal file
9
tests/fixtures/claudexor_effort_resolution.json
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"requested": "ultra",
|
||||
"submitted": "xhigh",
|
||||
"resolution": "downward",
|
||||
"source": "account_catalog",
|
||||
"parameter": "reasoning.effort",
|
||||
"observed": null,
|
||||
"observedSource": null
|
||||
}
|
||||
8
tests/fixtures/claudexor_effort_resolution_variants.json
vendored
Normal file
8
tests/fixtures/claudexor_effort_resolution_variants.json
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"known_parameter_unverifiable": {"requested":"ultra","submitted":null,"resolution":"unverifiable","source":"adapter","parameter":"model_reasoning_effort","observed":null,"observedSource":null},
|
||||
"known_parameter_omitted": {"requested":"ultra","submitted":null,"resolution":"omitted","source":"adapter","parameter":"model_reasoning_effort","observed":null,"observedSource":null},
|
||||
"future_downward": {"requested":"ultra","submitted":"vendor-balanced","resolution":"downward","source":"live_probe","parameter":"reasoning.effort","observed":null,"observedSource":null},
|
||||
"future_floor": {"requested":"vendor-minimal","submitted":"vendor-baseline","resolution":"floor","source":"versioned_snapshot","parameter":"reasoning.effort","observed":null,"observedSource":null},
|
||||
"engine_order_downward": {"requested":"low","submitted":"high","resolution":"downward","source":"account_catalog","parameter":"reasoning.effort","observed":null,"observedSource":null},
|
||||
"engine_order_floor": {"requested":"ultra","submitted":"low","resolution":"floor","source":"account_catalog","parameter":"reasoning.effort","observed":null,"observedSource":null}
|
||||
}
|
||||
126
tests/fixtures/llm_golden/anthropic_native.json
vendored
126
tests/fixtures/llm_golden/anthropic_native.json
vendored
|
|
@ -142,6 +142,19 @@
|
|||
"cache_write_tokens": 25,
|
||||
"provider": "anthropic",
|
||||
"resolved_model": "anthropic/claude-sonnet-4-6",
|
||||
"effort": {
|
||||
"requested": "high",
|
||||
"sent": {
|
||||
"thinking": {
|
||||
"type": "adaptive"
|
||||
},
|
||||
"output_config.effort": "high"
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"prompt_cache_ttl": "1h",
|
||||
"cache_write_tokens_by_ttl": {
|
||||
"5m": 5,
|
||||
|
|
@ -153,6 +166,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "high",
|
||||
"applied_effort": "high",
|
||||
"original_requested_effort": "high",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "function",
|
||||
"applied_tool_dialect": "function",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -409,6 +427,19 @@
|
|||
"cache_write_tokens": 0,
|
||||
"provider": "anthropic",
|
||||
"resolved_model": "anthropic/claude-sonnet-4-6",
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"thinking": {
|
||||
"type": "adaptive"
|
||||
},
|
||||
"output_config.effort": "medium"
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"prompt_cache_ttl": "1h",
|
||||
"cost": 0.001,
|
||||
"cost_estimated": true,
|
||||
|
|
@ -416,6 +447,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "function",
|
||||
"applied_tool_dialect": "function",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -665,6 +701,19 @@
|
|||
"cache_write_tokens": 0,
|
||||
"provider": "anthropic",
|
||||
"resolved_model": "anthropic/claude-sonnet-4-6",
|
||||
"effort": {
|
||||
"requested": "minimal",
|
||||
"sent": {
|
||||
"thinking": {
|
||||
"type": "adaptive"
|
||||
},
|
||||
"output_config.effort": "low"
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"prompt_cache_ttl": "1h",
|
||||
"cost": 0.0002,
|
||||
"cost_estimated": true,
|
||||
|
|
@ -672,6 +721,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "low",
|
||||
"applied_effort": "low",
|
||||
"original_requested_effort": "minimal",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "function",
|
||||
"applied_tool_dialect": "function",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -852,12 +906,30 @@
|
|||
"cache_write_tokens": 0,
|
||||
"provider": "anthropic",
|
||||
"resolved_model": "anthropic/claude-sonnet-4-6",
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"thinking": {
|
||||
"type": "adaptive"
|
||||
},
|
||||
"output_config.effort": "medium"
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"cost": 0.0002,
|
||||
"cost_estimated": true,
|
||||
"cost_final": false,
|
||||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -1015,12 +1087,30 @@
|
|||
"cache_write_tokens": 0,
|
||||
"provider": "anthropic",
|
||||
"resolved_model": "anthropic/claude-sonnet-4-6",
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"thinking": {
|
||||
"type": "adaptive"
|
||||
},
|
||||
"output_config.effort": "medium"
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"cost": 0.0002,
|
||||
"cost_estimated": true,
|
||||
"cost_final": false,
|
||||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "provider_unsupported_field",
|
||||
|
|
@ -1333,12 +1423,30 @@
|
|||
"cache_write_tokens": 0,
|
||||
"provider": "anthropic",
|
||||
"resolved_model": "anthropic/claude-sonnet-4-6",
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"thinking": {
|
||||
"type": "adaptive"
|
||||
},
|
||||
"output_config.effort": "medium"
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"cost": 0.0003,
|
||||
"cost_estimated": true,
|
||||
"cost_final": false,
|
||||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -1516,6 +1624,19 @@
|
|||
"cache_write_tokens": 0,
|
||||
"provider": "anthropic",
|
||||
"resolved_model": "anthropic/claude-sonnet-4-6",
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"thinking": {
|
||||
"type": "adaptive"
|
||||
},
|
||||
"output_config.effort": "medium"
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"prompt_cache_ttl": "1h",
|
||||
"cost": 0.0004,
|
||||
"cost_estimated": true,
|
||||
|
|
@ -1523,6 +1644,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "function",
|
||||
"applied_tool_dialect": "function",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
|
|||
18
tests/fixtures/llm_golden/aux_routes.json
vendored
18
tests/fixtures/llm_golden/aux_routes.json
vendored
|
|
@ -781,6 +781,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -789,6 +802,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
|
|||
24
tests/fixtures/llm_golden/compatible_lanes.json
vendored
24
tests/fixtures/llm_golden/compatible_lanes.json
vendored
|
|
@ -172,6 +172,14 @@
|
|||
"usage": {
|
||||
"prompt_tokens": 200,
|
||||
"completion_tokens": 10,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {},
|
||||
"sent_state": "omitted",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"provider": "openai-compatible",
|
||||
"resolved_model": "openai-compatible/glm-5",
|
||||
|
|
@ -348,6 +356,14 @@
|
|||
"usage": {
|
||||
"prompt_tokens": 200,
|
||||
"completion_tokens": 10,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {},
|
||||
"sent_state": "omitted",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"provider": "cloudru",
|
||||
"resolved_model": "cloudru/GLM-5",
|
||||
|
|
@ -477,6 +493,14 @@
|
|||
"usage": {
|
||||
"prompt_tokens": 200,
|
||||
"completion_tokens": 10,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {},
|
||||
"sent_state": "omitted",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"provider": "minimax",
|
||||
"resolved_model": "minimax/MiniMax-M2",
|
||||
|
|
|
|||
244
tests/fixtures/llm_golden/fallback_ladder.json
vendored
244
tests/fixtures/llm_golden/fallback_ladder.json
vendored
|
|
@ -71,6 +71,16 @@
|
|||
"usage": {
|
||||
"prompt_tokens": 200,
|
||||
"completion_tokens": 10,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"reasoning_effort": "medium"
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"provider": "openai",
|
||||
"resolved_model": "openai/gpt-5.5",
|
||||
|
|
@ -80,6 +90,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -282,6 +297,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -290,6 +318,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "provider_unsupported_field",
|
||||
|
|
@ -497,6 +530,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "none",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "low",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -505,6 +551,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "none",
|
||||
"applied_effort": "low",
|
||||
"original_requested_effort": "none",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "provider_required_reasoning",
|
||||
|
|
@ -720,6 +771,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -728,6 +792,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -1044,6 +1113,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -1052,6 +1134,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -1287,6 +1374,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -1295,6 +1395,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -1490,6 +1595,19 @@
|
|||
"response_id": "gen-fixture-429"
|
||||
},
|
||||
"usage": {
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"provider_error": {
|
||||
"code": 429,
|
||||
"type": null,
|
||||
|
|
@ -1717,6 +1835,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -1725,6 +1856,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -1942,6 +2078,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -1950,6 +2099,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "provider_unsupported_field",
|
||||
|
|
@ -2159,6 +2313,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -2167,6 +2334,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "provider_unsupported_field",
|
||||
|
|
@ -2389,6 +2561,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -2397,6 +2582,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -2593,6 +2783,19 @@
|
|||
"usage": {
|
||||
"prompt_tokens": 200,
|
||||
"completion_tokens": 10,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"resolved_model": "x-ai/grok-4.5",
|
||||
|
|
@ -2602,6 +2805,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -2806,6 +3014,19 @@
|
|||
"prompt_tokens": 120,
|
||||
"completion_tokens": 8,
|
||||
"cost": 0.0031,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "tool_calls",
|
||||
"provider": "openrouter",
|
||||
"resolved_model": "x-ai/grok-4.5",
|
||||
|
|
@ -2813,6 +3034,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "function",
|
||||
"applied_tool_dialect": "function",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -3005,6 +3231,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -3013,6 +3252,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
|
|||
30
tests/fixtures/llm_golden/gigachat_native.json
vendored
30
tests/fixtures/llm_golden/gigachat_native.json
vendored
|
|
@ -145,7 +145,15 @@
|
|||
"provider": "gigachat",
|
||||
"resolved_model": "gigachat/GigaChat-3-Pro",
|
||||
"cost": null,
|
||||
"cost_final": false
|
||||
"cost_final": false,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {},
|
||||
"sent_state": "omitted",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
}
|
||||
},
|
||||
"ledger_attempt_count": 1
|
||||
},
|
||||
|
|
@ -294,7 +302,15 @@
|
|||
"provider": "gigachat",
|
||||
"resolved_model": "gigachat/GigaChat-3-Pro",
|
||||
"cost": null,
|
||||
"cost_final": false
|
||||
"cost_final": false,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {},
|
||||
"sent_state": "omitted",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
}
|
||||
},
|
||||
"ledger_attempt_count": 1
|
||||
},
|
||||
|
|
@ -399,7 +415,15 @@
|
|||
"provider": "gigachat",
|
||||
"resolved_model": "gigachat/GigaChat-3-Pro",
|
||||
"cost": null,
|
||||
"cost_final": false
|
||||
"cost_final": false,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {},
|
||||
"sent_state": "omitted",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
}
|
||||
},
|
||||
"ledger_attempt_count": 1
|
||||
},
|
||||
|
|
|
|||
10
tests/fixtures/llm_golden/local_lane.json
vendored
10
tests/fixtures/llm_golden/local_lane.json
vendored
|
|
@ -114,7 +114,15 @@
|
|||
"cost_final": true,
|
||||
"provider": "local",
|
||||
"resolved_model": "local-model",
|
||||
"response_finish_reason": "stop"
|
||||
"response_finish_reason": "stop",
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {},
|
||||
"sent_state": "omitted",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
}
|
||||
},
|
||||
"ledger_attempt_count": 1
|
||||
},
|
||||
|
|
|
|||
15
tests/fixtures/llm_golden/openai_direct.json
vendored
15
tests/fixtures/llm_golden/openai_direct.json
vendored
|
|
@ -376,6 +376,16 @@
|
|||
"usage": {
|
||||
"prompt_tokens": 200,
|
||||
"completion_tokens": 10,
|
||||
"effort": {
|
||||
"requested": "xhigh",
|
||||
"sent": {
|
||||
"reasoning_effort": "xhigh"
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"provider": "openai",
|
||||
"resolved_model": "openai/gpt-5.5",
|
||||
|
|
@ -385,6 +395,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "xhigh",
|
||||
"applied_effort": "xhigh",
|
||||
"original_requested_effort": "xhigh",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
|
|||
184
tests/fixtures/llm_golden/openrouter_payload.json
vendored
184
tests/fixtures/llm_golden/openrouter_payload.json
vendored
|
|
@ -1585,6 +1585,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "high",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "high",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -1594,6 +1607,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "high",
|
||||
"applied_effort": "high",
|
||||
"original_requested_effort": "high",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "function",
|
||||
"applied_tool_dialect": "function",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -1768,6 +1786,19 @@
|
|||
"usage": {
|
||||
"prompt_tokens": 200,
|
||||
"completion_tokens": 10,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"provider": "openrouter",
|
||||
"resolved_model": "x-ai/grok-4.5",
|
||||
|
|
@ -1777,6 +1808,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -1938,6 +1974,19 @@
|
|||
"usage": {
|
||||
"prompt_tokens": 200,
|
||||
"completion_tokens": 10,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cost": 0.0077,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -1946,6 +1995,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -2097,6 +2151,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -2105,6 +2172,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -2287,6 +2359,19 @@
|
|||
"prompt_tokens": 90,
|
||||
"completion_tokens": 12,
|
||||
"cost": 0.002,
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "tool_calls",
|
||||
"provider": "openrouter",
|
||||
"resolved_model": "x-ai/grok-4.5",
|
||||
|
|
@ -2294,6 +2379,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "function",
|
||||
"applied_tool_dialect": "function",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -2467,6 +2557,19 @@
|
|||
"server_tool_use": {
|
||||
"web_search_requests": 2
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"web_search_sources": [
|
||||
{
|
||||
|
|
@ -2481,6 +2584,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -2596,6 +2704,19 @@
|
|||
"response_id": "gen-fixture-429"
|
||||
},
|
||||
"usage": {
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"provider_error": {
|
||||
"code": 429,
|
||||
"type": null,
|
||||
|
|
@ -2609,6 +2730,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -2813,6 +2939,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -2827,6 +2966,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "function",
|
||||
"applied_tool_dialect": "function",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -2967,8 +3111,8 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"id": "openrouter.dispatch.effort_clamp_disclosure",
|
||||
"route": "a learned ceiling clamp rides THIS call's usage as reasoning_effort_clamped",
|
||||
"id": "openrouter.dispatch.legacy_ceiling_does_not_clamp_xhigh",
|
||||
"route": "requested xhigh reaches the transport unchanged despite a legacy learned ceiling, with exact sent-effort usage and no clamp disclosure",
|
||||
"spec": {
|
||||
"env": {
|
||||
"OPENROUTER_API_KEY": "or-fixture-key"
|
||||
|
|
@ -3037,6 +3181,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "xhigh",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "xhigh",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -3045,6 +3202,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "xhigh",
|
||||
"applied_effort": "xhigh",
|
||||
"original_requested_effort": "xhigh",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "none",
|
||||
"applied_tool_dialect": "none",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
@ -3207,6 +3369,19 @@
|
|||
"prompt_tokens_details": {
|
||||
"cached_tokens": 40
|
||||
},
|
||||
"effort": {
|
||||
"requested": "medium",
|
||||
"sent": {
|
||||
"extra_body.reasoning": {
|
||||
"effort": "medium",
|
||||
"exclude": false
|
||||
}
|
||||
},
|
||||
"sent_state": "explicit",
|
||||
"sent_source": "host_candidate",
|
||||
"reported": null,
|
||||
"report_source": null
|
||||
},
|
||||
"response_finish_reason": "stop",
|
||||
"cached_tokens": 40,
|
||||
"provider": "openrouter",
|
||||
|
|
@ -3216,6 +3391,11 @@
|
|||
"request_wire": {
|
||||
"requested_effort": "medium",
|
||||
"applied_effort": "medium",
|
||||
"original_requested_effort": "medium",
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": null,
|
||||
"reported_effort_source": null,
|
||||
"requested_tool_dialect": "function",
|
||||
"applied_tool_dialect": "function",
|
||||
"reason_code": "requested_wire_form",
|
||||
|
|
|
|||
|
|
@ -371,6 +371,15 @@ def test_resume_consumes_grant_restores_cognition_and_never_reexecutes(tmp_path,
|
|||
|
||||
queue, state, workers = _install_queue(tmp_path, monkeypatch)
|
||||
monkeypatch.setattr(state, "budget_remaining", lambda _st, **_k: 5.0)
|
||||
from tests import _budget_pause_exact_helpers as helpers
|
||||
original_ctx = helpers._loop_ctx
|
||||
|
||||
def with_sent_disclosure(*args, **kwargs):
|
||||
context, limits = original_ctx(*args, **kwargs)
|
||||
limits.accumulated_usage["request_wire"] = {"applied_effort": "low", "original_requested_effort": "high"}
|
||||
return context, limits
|
||||
|
||||
monkeypatch.setattr(helpers, "_loop_ctx", with_sent_disclosure)
|
||||
task, row = _parked(tmp_path, monkeypatch, task_id="loop-1")
|
||||
assert queue.resume_budget_paused_task("loop-1")["ok"] is True
|
||||
handoff = task["_budget_pause_resume"]
|
||||
|
|
@ -384,6 +393,8 @@ def test_resume_consumes_grant_restores_cognition_and_never_reexecutes(tmp_path,
|
|||
model, effort, use_local, mode, round_idx, plan = budget_pause.resume_paused_loop(
|
||||
tools, state_blob, messages, trace, usage, seen, budget_remaining_usd=5.0)
|
||||
assert (model, round_idx, mode) == ("m", 4, "max")
|
||||
assert effort == "high" # restored original preference, independent of old wire disclosures
|
||||
assert usage["request_wire"]["applied_effort"] == "low"
|
||||
assert usage["cost"] == 1.25 and "execution_status" not in usage
|
||||
# The unanswered call is closed as UNKNOWN, not re-run, not declared un-run.
|
||||
unknown = [m for m in messages if m.get("role") == "tool" and m.get("tool_call_id") == "call_b"]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import base64
|
|||
from copy import deepcopy
|
||||
from dataclasses import asdict
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
|
@ -36,31 +37,44 @@ def call(client, asynchronous, **kwargs):
|
|||
return asyncio.run(value) if asynchronous else value
|
||||
|
||||
|
||||
def capture_catalog(failure, effort):
|
||||
return [{**CAPTURE_OPERATION, "parameters": [
|
||||
{"name": name, "location": "query", "enum": ["true", "false"]}
|
||||
for name, enabled in (("captureFailureEvidence", failure), ("captureEffortEvidence", effort))
|
||||
if enabled]}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("supported", [False, True])
|
||||
def test_capture_is_negotiated_once_without_changing_provider_payload(setup, asynchronous, supported):
|
||||
@pytest.mark.parametrize("failure,effort", [(False, False), (True, False), (False, True), (True, True)])
|
||||
def test_capture_is_negotiated_once_without_changing_provider_payload(setup, asynchronous, failure, effort):
|
||||
root, gateway, client = setup
|
||||
gateway.operation_catalog = [deepcopy(CAPTURE_OPERATION)] if supported else []
|
||||
call(client, asynchronous, model_role="main")
|
||||
gateway.operation_catalog = capture_catalog(failure, effort)
|
||||
_, usage = call(client, asynchronous, model_role="main")
|
||||
assert gateway.catalog_reads == 1
|
||||
assert gateway.capture_requests == ([{"capture_failure_evidence": True}] if supported else [{}])
|
||||
expected = {key: True for key, enabled in (
|
||||
("capture_failure_evidence", failure), ("capture_effort_evidence", effort)) if enabled}
|
||||
assert gateway.capture_requests == [expected]
|
||||
payload = gateway.uploads[0][0]
|
||||
assert "captureFailureEvidence" not in json.dumps(payload)
|
||||
assert "capture_failure_evidence" not in json.dumps(payload)
|
||||
for key in ("captureFailureEvidence", "capture_failure_evidence", "captureEffortEvidence", "capture_effort_evidence"):
|
||||
assert key not in json.dumps(payload)
|
||||
assert usage.get("effort_resolution") is None # Capability alone supplies no evidence.
|
||||
assert retained(root, "request") == payload
|
||||
manifests = list((root / "observability/calls/task-one").glob("*_model_request.json"))
|
||||
assert len(manifests) == 1
|
||||
manifest = json.loads(manifests[0].read_text())
|
||||
assert manifest["capture_failure_evidence"] is supported
|
||||
assert manifest["capture_failure_evidence"] is failure
|
||||
assert manifest["capture_effort_evidence"] is effort
|
||||
assert manifest["operation_id"] == "op-0"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
def test_lost_create_and_gateway_replacement_reuse_frozen_capture(setup, monkeypatch, asynchronous):
|
||||
@pytest.mark.parametrize("failure,effort", [(False, False), (True, False), (False, True), (True, True)])
|
||||
def test_lost_create_and_gateway_replacement_reuse_frozen_capture(setup, monkeypatch, asynchronous, failure, effort):
|
||||
root, gateway, client = setup
|
||||
gateway.operation_catalog = [deepcopy(CAPTURE_OPERATION)]
|
||||
gateway.operation_catalog = capture_catalog(failure, effort)
|
||||
gateway.lose_create = True
|
||||
replacement = Gateway()
|
||||
replacement.operation_catalog = capture_catalog(not failure, not effort)
|
||||
for name in ("accepted_operations", "creates", "capture_requests"):
|
||||
setattr(replacement, name, getattr(gateway, name))
|
||||
monkeypatch.setattr(transport, "read_owned_gateway", lambda: replacement)
|
||||
|
|
@ -68,10 +82,22 @@ def test_lost_create_and_gateway_replacement_reuse_frozen_capture(setup, monkeyp
|
|||
tool_context=SimpleNamespace(task_id="task-one", is_direct_chat=False), control_reason=lambda: None))
|
||||
monkeypatch.setattr(transport.config, "NETWORK_WAIT_BACKOFF_START_SEC", 0.001)
|
||||
monkeypatch.setattr(transport.config, "NETWORK_WAIT_BACKOFF_MAX_SEC", 0.001)
|
||||
answer, usage = call(client, asynchronous)
|
||||
manifests = []
|
||||
|
||||
def observe(receipt):
|
||||
manifests.append(json.loads(Path(receipt["request_manifest_ref"]["path"]).read_text()))
|
||||
|
||||
answer, usage = call(client, asynchronous, model_operation_observer=observe)
|
||||
assert answer == result()["message"]
|
||||
assert gateway.catalog_reads == 1 and replacement.catalog_reads == 0
|
||||
assert gateway.capture_requests == [{"capture_failure_evidence": True}] * 2
|
||||
expected = {key: True for key, enabled in (
|
||||
("capture_failure_evidence", failure), ("capture_effort_evidence", effort)) if enabled}
|
||||
assert gateway.capture_requests == [expected] * 2
|
||||
assert len(manifests) == 3 # Before lost create, before rejoin, then accepted custody.
|
||||
assert all(item["capture_failure_evidence"] is failure and
|
||||
item["capture_effort_evidence"] is effort for item in manifests)
|
||||
assert len(gateway.uploads) == 1 and not replacement.uploads
|
||||
assert retained(root, "request") == gateway.uploads[0][0]
|
||||
assert len(gateway.creates) == 2 and len(set(gateway.creates)) == 1
|
||||
assert len(gateway.accepted_operations) == len(usage["ledger_attempt_ids"]) == 1
|
||||
assert gateway.closed == replacement.closed == 1
|
||||
|
|
@ -141,8 +167,13 @@ def test_known_terminal_null_message_settles_then_rejects_without_private_projec
|
|||
assert retained(root)["failureEvidence"] == evidence
|
||||
|
||||
|
||||
def test_response_rejection_survives_existing_vision_ipc_reconstruction():
|
||||
capture = ua.PhysicalAttemptCapture("attempt-one", MODEL, "claudexor", "settled", "opaque")
|
||||
@pytest.mark.parametrize("effort", [None, {
|
||||
"requested": "ultra", "sent": {"options.reasoningEffort": "ultra"},
|
||||
"sent_state": "explicit", "sent_source": "host_candidate",
|
||||
"reported": None, "report_source": None,
|
||||
}])
|
||||
def test_response_rejection_survives_existing_vision_ipc_reconstruction(effort):
|
||||
capture = ua.PhysicalAttemptCapture("attempt-one", MODEL, "claudexor", "settled", "opaque", effort=effort)
|
||||
receipt = {"receipt_id": "receipt-one", "custody": None, "capture": asdict(capture),
|
||||
"kind": "model", "text": "", "usage": {"prompt_tokens": 20}, "ledger_attempt_ids": ["attempt-one"],
|
||||
"error": "", "problem": deepcopy(REJECTION), "operation_id": "operation-one", "model_role": "vision",
|
||||
|
|
@ -153,6 +184,7 @@ def test_response_rejection_survives_existing_vision_ipc_reconstruction():
|
|||
assert error.code == "response_rejected" and error.stream_rejected and error.stream_incomplete
|
||||
assert error.problem == REJECTION and error.operation_id == "operation-one" and error.route == ROUTE
|
||||
assert error.physical_attempt_capture.state == "settled" and error.usage == receipt["usage"]
|
||||
assert error.physical_attempt_capture.effort == effort
|
||||
assert classify_llm_exception(error).kind == "provider_error"
|
||||
assert not isinstance(error, ProviderNotDispatched)
|
||||
|
||||
|
|
|
|||
|
|
@ -429,30 +429,44 @@ def test_operation_identity_mismatch_is_not_an_accepted_response(gateway_factory
|
|||
assert raised.value.code == "malformed_response"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capture", [False, True])
|
||||
def test_failure_evidence_opt_in_preserves_create_body(gateway_factory, capture):
|
||||
@pytest.mark.parametrize("capture_failure", [False, True])
|
||||
@pytest.mark.parametrize("capture_effort", [False, True])
|
||||
def test_evidence_opt_ins_preserve_create_body(gateway_factory, capture_failure, capture_effort):
|
||||
wire = ModelWire()
|
||||
gateway = gateway_factory(wire)
|
||||
ref = gateway.upload_model_request(_request(), idempotency_key="evidence")
|
||||
gateway.create_model_operation(ref, idempotency_key="evidence", capture_failure_evidence=capture)
|
||||
gateway.create_model_operation(ref, idempotency_key="evidence",
|
||||
capture_failure_evidence=capture_failure, capture_effort_evidence=capture_effort)
|
||||
request = wire.calls[-1]
|
||||
assert request.url.query == (b"captureFailureEvidence=true" if capture else b"")
|
||||
expected = {}
|
||||
if capture_failure:
|
||||
expected["captureFailureEvidence"] = "true"
|
||||
if capture_effort:
|
||||
expected["captureEffortEvidence"] = "true"
|
||||
assert dict(request.url.params) == expected
|
||||
assert json.loads(request.content) == {"request": ref}
|
||||
assert request.headers["Idempotency-Key"] == "evidence"
|
||||
assert wire.payload == _bytes(_request())
|
||||
assert wire.generation_count == 1
|
||||
|
||||
|
||||
def test_query_negotiation_uses_exact_wire_descriptor():
|
||||
parameter = {"name": "captureFailureEvidence", "location": "query", "enum": ["true", "false"]}
|
||||
@pytest.mark.parametrize("parameter_name", ["captureFailureEvidence", "captureEffortEvidence"])
|
||||
def test_query_negotiation_uses_exact_wire_descriptor(parameter_name):
|
||||
def supported(operations):
|
||||
return cx.operation_query_supported(operations, method="POST", path="/v2/model-operations",
|
||||
name=parameter_name, value="true")
|
||||
|
||||
parameter = {"name": parameter_name, "location": "query", "enum": ["true", "false"]}
|
||||
operation = {"method": "POST", "path": "/v2/model-operations", "parameters": [parameter]}
|
||||
assert cx.model_failure_evidence_supported([operation])
|
||||
assert not cx.model_failure_evidence_supported([])
|
||||
assert supported([operation])
|
||||
assert cx.model_failure_evidence_supported([operation]) is (parameter_name == "captureFailureEvidence")
|
||||
assert not supported([])
|
||||
for change in ({"method": "GET"}, {"path": "/v2/model-operations/:id"}, {"parameters": []}):
|
||||
assert not cx.model_failure_evidence_supported([{**operation, **change}])
|
||||
assert not supported([{**operation, **change}])
|
||||
for change in ({"name": "other"}, {"location": "header"}, {"enum": ["false"]}, {"enum": "true"}):
|
||||
assert not cx.model_failure_evidence_supported([{**operation, "parameters": [{**parameter, **change}]}])
|
||||
assert not supported([{**operation, "parameters": [{**parameter, **change}]}])
|
||||
accounts = {"method": "GET", "path": "/v2/model-sources", "parameters": [
|
||||
{"name": "view", "location": "query", "enum": ["accounts"]}]}
|
||||
assert cx.account_catalog_supported([accounts], accounts["path"])
|
||||
assert not cx.account_catalog_supported([operation], accounts["path"])
|
||||
assert not cx.model_failure_evidence_supported([accounts])
|
||||
assert not supported([accounts])
|
||||
|
|
|
|||
|
|
@ -116,12 +116,11 @@ def test_missing_engine_run_directory_never_reads_the_working_directory(detail,
|
|||
assert final_attempt_facts(detail, "run-fixture") == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("applied,expected", [
|
||||
({"reasoningEffort": "xhigh"}, "confirmed"),
|
||||
({"reasoningEffort": "medium"}, "mismatch"),
|
||||
(None, "unknown"),
|
||||
@pytest.mark.parametrize("applied", [
|
||||
{"reasoningEffort": "xhigh"}, {"reasoningEffort": "medium"}, None, {},
|
||||
{"cacheKey": "unrelated"}, {"reasoningEffort": None},
|
||||
])
|
||||
def test_model_invocation_records_requested_and_applied_options(applied, expected):
|
||||
def test_model_invocation_records_requested_and_applied_options(applied):
|
||||
requested = {"reasoningEffort": "xhigh"}
|
||||
invocation = _ModelInvocation(
|
||||
{"usage_model": "claudexor::codex=model"}, {"options": requested}, {}
|
||||
|
|
@ -137,7 +136,8 @@ def test_model_invocation_records_requested_and_applied_options(applied, expecte
|
|||
observed = usage["claudexor"]
|
||||
assert observed["requested_options"] == requested
|
||||
assert observed["applied_options"] == applied
|
||||
assert observed["options_honored"] == expected
|
||||
assert usage["effort"]["reported"] == (applied or {}).get("reasoningEffort")
|
||||
assert usage["effort"]["report_source"] == ("provider_applied_options" if isinstance(applied, dict) and applied.get("reasoningEffort") is not None else None)
|
||||
|
||||
|
||||
def test_a_differently_echoed_cache_key_is_a_durable_mismatch_of_its_own():
|
||||
|
|
@ -152,7 +152,8 @@ def test_a_differently_echoed_cache_key_is_a_durable_mismatch_of_its_own():
|
|||
"appliedOptions": {"reasoningEffort": "xhigh", "cacheKey": "engine-b"},
|
||||
})
|
||||
|
||||
assert usage["claudexor"]["options_honored"] == "mismatch"
|
||||
assert usage["claudexor"]["requested_options"] == requested
|
||||
assert usage["claudexor"]["applied_options"] == {"reasoningEffort": "xhigh", "cacheKey": "engine-b"}
|
||||
|
||||
|
||||
def _continuation(profile="profile-a", source="codex", model="gpt-6"):
|
||||
|
|
|
|||
|
|
@ -155,7 +155,7 @@ def test_tracked_pin_names_a_cli_capable_release():
|
|||
of this proposal converged on while the pin was still pre-CLI 3.6.0)."""
|
||||
pin = runtime.load_runtime_pin()
|
||||
assert pin is not None
|
||||
assert pin.version == "3.16.0"
|
||||
assert pin.version == "3.17.1"
|
||||
assert pin.cli_entrypoint == "claudexor.bundle.cjs"
|
||||
|
||||
|
||||
|
|
|
|||
630
tests/test_effort_compatibility.py
Normal file
630
tests/test_effort_compatibility.py
Normal file
|
|
@ -0,0 +1,630 @@
|
|||
"""Effort constraints and evidence through the existing physical send drivers."""
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from ouroboros import usage_accounting as ua
|
||||
from ouroboros.llm import LLMClient
|
||||
from ouroboros.request_wire_contract import payload_effort
|
||||
from ouroboros.request_wire_recovery import (
|
||||
plan_next_wire_retry,
|
||||
prepare_wire_payload_for_send,
|
||||
request_wire_call_scope,
|
||||
)
|
||||
from tests.test_request_wire_recovery_phase2b import (
|
||||
_capture,
|
||||
_Rejected,
|
||||
_Response,
|
||||
_store,
|
||||
_target,
|
||||
_value_payload,
|
||||
)
|
||||
from tests.test_request_wire_recovery_phase2b import (
|
||||
evidence_root as _evidence_root,
|
||||
)
|
||||
|
||||
evidence_root = _evidence_root
|
||||
|
||||
ENUM_MESSAGE = ('reasoning.effort: Invalid option: expected one of '
|
||||
'"none"|"minimal"|"low"|"medium"|"high"|"xhigh"|"max"')
|
||||
|
||||
|
||||
def _error(message=ENUM_MESSAGE, **fields):
|
||||
return {"status": 400, "message": message, **fields}
|
||||
|
||||
|
||||
def _exception(error):
|
||||
rejected = _Rejected(error["message"], error.get("status"))
|
||||
rejected.body = {"error": {key: value for key, value in error.items() if key != "status"}}
|
||||
return rejected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("body_error", [False, True])
|
||||
@pytest.mark.parametrize("requested,error,applied", [
|
||||
pytest.param("ultra", _error(), "max", id="downward"),
|
||||
pytest.param("none", _error(
|
||||
"Reasoning is mandatory and cannot be disabled", param="reasoning.effort",
|
||||
enum=["high", "medium"]), "medium", id="mandatory-structured"),
|
||||
pytest.param("none", _error(
|
||||
'reasoning.effort is mandatory: supported values are "high", "medium"'),
|
||||
"medium", id="mandatory-text"),
|
||||
pytest.param("low", _error(
|
||||
"Invalid option", param="reasoning.effort", allowed_values=["high", "medium"]),
|
||||
"medium", id="all-higher-structured"),
|
||||
pytest.param("low", _error('reasoning.effort: expected one of "high"|"medium"'),
|
||||
"medium", id="all-higher-text"),
|
||||
pytest.param("medium", _error(
|
||||
"Invalid option", param="reasoning.effort", allowed_values=["minimal", "ultra"]),
|
||||
"minimal", id="advertised-minimal-below"),
|
||||
pytest.param("ultra", _error("reasoning.effort value 'ultra' is not supported. "
|
||||
"Supported values are: 'low', 'medium', or 'high'."),
|
||||
"high", id="serial-or-conjunction"),
|
||||
pytest.param("ultra", _error("reasoning.effort value 'ultra' is not supported. "
|
||||
"Supported values are: 'low', 'medium', and 'high'."),
|
||||
"high", id="serial-and-conjunction"),
|
||||
pytest.param("ultra", _error("reasoning.effort: Input should be 'low', 'medium' or 'high' "
|
||||
"and xhigh requires a pro plan"),
|
||||
"high", id="conjunction-stops-at-prose"),
|
||||
pytest.param("ultra", _error("reasoning.effort value 'ultra' is not supported. Supported values are "
|
||||
"low, medium, or high and xhigh requires a pro plan"),
|
||||
"high", id="bare-second-conjunction-is-prose"),
|
||||
pytest.param("ultra", _error("reasoning.effort value 'ultra' is not supported. Supported values are "
|
||||
"'low', 'medium', or 'high' and 'xhigh' is not supported"),
|
||||
"high", id="quoted-second-conjunction-is-prose"),
|
||||
pytest.param("ultra", _error("reasoning.effort value 'ultra' is not supported. Supported values are "
|
||||
"low, medium, high and xhigh is not supported"),
|
||||
"high", id="first-conjunction-negative-prose"),
|
||||
pytest.param("xhigh", _error("reasoning.effort value 'xhigh' is not supported. Supported values are "
|
||||
"low, medium, high and xhigh is not supported"),
|
||||
"high", id="negative-prose-does-not-suppress-retry"),
|
||||
pytest.param("xhigh", _error("reasoning.effort value 'xhigh' is not supported. Supported values are "
|
||||
"low, medium, high and xhigh is unsupported"),
|
||||
"high", id="copula-negative-prose-does-not-suppress-retry"),
|
||||
pytest.param("ultra", _error("reasoning.effort value 'ultra' is not supported. Supported values are "
|
||||
"low, medium, high, xhigh requires a pro plan"),
|
||||
"high", id="bare-comma-negative-prose"),
|
||||
pytest.param("ultra", _error("reasoning.effort value 'ultra' is not supported. Supported values are "
|
||||
"low or medium or high"),
|
||||
"high", id="chained-conjunctions"),
|
||||
pytest.param("xhigh", _error("reasoning.effort value 'xhigh' is not supported. Supported values are "
|
||||
"low, medium, or high and xhigh requires a pro plan"),
|
||||
"high", id="prose-tier-does-not-suppress-resend"),
|
||||
])
|
||||
def test_enum_without_scalar_echo_reaches_driver_and_learns_only_after_success(
|
||||
evidence_root, tmp_path, asynchronous, body_error, requested, error, applied,
|
||||
):
|
||||
target = {**_target(), "requested_reasoning_effort": requested}
|
||||
source = _value_payload("nested", target, requested)
|
||||
sent = []
|
||||
|
||||
def send(**candidate):
|
||||
sent.append(copy.deepcopy(candidate))
|
||||
if len(sent) == 1:
|
||||
if body_error:
|
||||
return _Response({"error": error, "choices": [], "usage": {}})
|
||||
raise _exception(error)
|
||||
assert _store(evidence_root) is None
|
||||
assert payload_effort(candidate) == applied
|
||||
return _Response({
|
||||
"choices": [{"message": {"role": "assistant", "content": "ok"}}],
|
||||
"usage": {"cost": 0.125, "effort": {"requested": "low", "reported": "ultra"},
|
||||
"effort_resolution": {"observed": "ultra"}},
|
||||
})
|
||||
|
||||
async def send_async(**candidate):
|
||||
return send(**candidate)
|
||||
|
||||
client = LLMClient(api_key="unused")
|
||||
|
||||
async def call_async():
|
||||
response = await client._create_chat_completion_with_retries_async(send_async, source, target)
|
||||
return client._normalize_remote_response(response.model_dump(), target, skip_cost_fetch=True)
|
||||
|
||||
with request_wire_call_scope(), ua.usage_scope(ua.UsageScope(drive_root=tmp_path, task_id="enum")):
|
||||
if asynchronous:
|
||||
_, usage = asyncio.run(call_async())
|
||||
else:
|
||||
response = client._create_chat_completion_with_retries(send, source, target)
|
||||
_, usage = client._normalize_remote_response(response.model_dump(), target, skip_cost_fetch=True)
|
||||
assert [payload_effort(p) for p in sent] == [requested, applied]
|
||||
assert all(p["extra_body"]["reasoning"]["exclude"] is False for p in sent)
|
||||
assert usage["effort"]["requested"] == requested
|
||||
assert usage["effort"]["sent"] == {"extra_body.reasoning": {"effort": applied, "exclude": False}}
|
||||
assert usage["effort"]["reported"] is None
|
||||
assert usage["request_wire"]["applied_effort_source"] == "sent_candidate"
|
||||
rows = [json.loads(line) for line in (tmp_path / ua.LEDGER_REL).read_text().splitlines()]
|
||||
assert rows[-1]["state"] == "settled"
|
||||
assert rows[-1]["effort"] == usage["effort"]
|
||||
assert "effort_resolution" not in rows[-1] and "effort_resolution" not in usage
|
||||
assert rows[-1]["cost_usd"] == 0.125 and rows[-1]["cost_final"] is True
|
||||
assert _store(evidence_root)
|
||||
with request_wire_call_scope():
|
||||
assert payload_effort(prepare_wire_payload_for_send(target, source, api_surface="chat.completions")) == applied
|
||||
supported = _value_payload("nested", target, "high")
|
||||
assert payload_effort(prepare_wire_payload_for_send(target, supported, api_surface="chat.completions")) == "high"
|
||||
changed_route = {**target, "base_url": "https://different.invalid/v1"}
|
||||
assert payload_effort(prepare_wire_payload_for_send(changed_route, source, api_surface="chat.completions")) == requested
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("body_error", [False, True])
|
||||
@pytest.mark.parametrize("carrier,requested,error,applied", [
|
||||
pytest.param("nested", "none", _error("Reasoning is mandatory and cannot be disabled"),
|
||||
"low", id="legacy-mandatory"),
|
||||
pytest.param("nested", "minimal", _error("Mandatory parameter", param="reasoning.effort"),
|
||||
"low", id="mandatory-alias"),
|
||||
pytest.param("nested", "none", _error("Unsupported parameter", param="reasoning"),
|
||||
"provider_default", id="default-alias"),
|
||||
pytest.param("nested", "none", _error("Unsupported parameter", param="extra_body.reasoning"),
|
||||
"provider_default", id="default-nested"),
|
||||
pytest.param("top", "none", _error("Unsupported parameter", param="reasoning_effort"),
|
||||
"provider_default", id="default-top"),
|
||||
pytest.param("nested", "low", _error(
|
||||
"reasoning.effort value 'low' is not supported; use 'medium' or 'high'"),
|
||||
None, id="arbitrary-quotes"),
|
||||
pytest.param("nested", "low", _error("Invalid option", param="reasoning.effort"),
|
||||
None, id="unknown-minimum"),
|
||||
pytest.param("nested", "none", _error(
|
||||
"Reasoning is mandatory", param="reasoning.effort", enum=["future"]),
|
||||
None, id="unknown-enum"),
|
||||
pytest.param("nested", "none", _error(
|
||||
"Reasoning is mandatory", param="temperature", enum=["medium", "high"]),
|
||||
None, id="other-field"),
|
||||
pytest.param("nested", "none", None, None, id="success"),
|
||||
])
|
||||
def test_minimum_recovery_preserves_legacy_aliases_and_requires_positive_evidence(
|
||||
evidence_root, tmp_path, asynchronous, body_error, carrier, requested, error, applied,
|
||||
):
|
||||
target = _target()
|
||||
source = _value_payload(carrier, target, requested)
|
||||
sent = []
|
||||
|
||||
def send(**candidate):
|
||||
sent.append(copy.deepcopy(candidate))
|
||||
if len(sent) == 1 and error is not None:
|
||||
if body_error:
|
||||
return _Response({"error": error, "choices": [], "usage": {}})
|
||||
raise _exception(error)
|
||||
return _Response()
|
||||
|
||||
async def send_async(**candidate):
|
||||
return send(**candidate)
|
||||
|
||||
client = LLMClient(api_key="unused")
|
||||
|
||||
async def call_async():
|
||||
response = await client._create_chat_completion_with_retries_async(send_async, source, target)
|
||||
if applied or error is None:
|
||||
client._normalize_remote_response(response.model_dump(), target, skip_cost_fetch=True)
|
||||
return response
|
||||
|
||||
def call():
|
||||
if asynchronous:
|
||||
return asyncio.run(call_async())
|
||||
response = client._create_chat_completion_with_retries(send, source, target)
|
||||
if applied or error is None:
|
||||
client._normalize_remote_response(response.model_dump(), target, skip_cost_fetch=True)
|
||||
return response
|
||||
|
||||
with request_wire_call_scope(), ua.usage_scope(ua.UsageScope(drive_root=tmp_path, task_id="minimum")), ua.physical_attempt_limit(2 if applied else 1):
|
||||
if error is not None and applied is None and not body_error:
|
||||
with pytest.raises(_Rejected):
|
||||
call()
|
||||
else:
|
||||
response = call()
|
||||
if error is not None and applied is None:
|
||||
assert response.model_dump()["error"] == error
|
||||
sent_effort = "" if applied == "provider_default" else applied
|
||||
assert [payload_effort(p) for p in sent] == ([requested, sent_effort] if applied else [requested])
|
||||
assert bool(_store(evidence_root)) is bool(applied)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body_error", [False, True])
|
||||
@pytest.mark.parametrize("carrier", ["top", "nested", "anthropic"])
|
||||
@pytest.mark.parametrize("supported,applied", [
|
||||
# The old parser accepted the negative 'max' as the highest prescription.
|
||||
("; 'max' is not supported. Supported values are: 'low', 'medium', 'high', 'xhigh'.", "xhigh"),
|
||||
# The old parser read the serial conjunction as a value 'or' and lost 'high'.
|
||||
(". Supported values are: 'low', 'medium', or 'high'.", "high"),
|
||||
# The old parser read a clause after the serial conjunction as one more value.
|
||||
(". Supported values are low, medium, or high and xhigh requires a pro plan", "high"),
|
||||
])
|
||||
def test_positive_constraints_are_bound_to_field_and_ignore_negative_quotes(evidence_root, carrier, body_error, supported, applied):
|
||||
target = _target("anthropic" if carrier == "anthropic" else "openrouter")
|
||||
field = {"top": "reasoning_effort", "nested": "reasoning.effort", "anthropic": "output_config.effort"}[carrier]
|
||||
source = _value_payload(carrier, target, "ultra")
|
||||
error = _error(f"{field} value 'ultra' is not supported{supported}")
|
||||
with request_wire_call_scope():
|
||||
sent = prepare_wire_payload_for_send(target, source, api_surface="messages" if carrier == "anthropic" else "chat.completions")
|
||||
retry = plan_next_wire_retry(sent, error=error if body_error else _exception(error), body_error=body_error)
|
||||
assert payload_effort(retry) == applied
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body_error", [False, True])
|
||||
@pytest.mark.parametrize("carrier", ["top", "nested"])
|
||||
@pytest.mark.parametrize("requested,error,applied", [
|
||||
pytest.param("medium", _error("Invalid option", param="{field}", enum=["minimal", "ultra"]),
|
||||
"minimal", id="strongest-at-or-below-is-minimal"),
|
||||
pytest.param("medium", _error("Invalid option", param="{field}", enum=["minimal"]),
|
||||
"minimal", id="only-minimal"),
|
||||
pytest.param("medium", _error('{field}: expected one of "minimal"|"ultra"'),
|
||||
"minimal", id="text-minimal"),
|
||||
pytest.param("none", _error("Reasoning is mandatory", param="{field}", enum=["minimal", "high"]),
|
||||
"minimal", id="mandatory-floor-minimal"),
|
||||
pytest.param("none", _error('{field} is mandatory: supported values are "high", "medium", or "minimal"'),
|
||||
"minimal", id="mandatory-text-conjunction-minimal"),
|
||||
pytest.param("medium", _error("Invalid option", param="{field}", enum=["none", "high"]),
|
||||
"high", id="none-is-not-comparable"),
|
||||
pytest.param("medium", _error("{field} value 'medium' is not supported; 'minimal' is not "
|
||||
"supported. Supported values are: 'ultra'"),
|
||||
"ultra", id="negative-minimal-not-advertised"),
|
||||
pytest.param("minimal", _error("Invalid option", param="{field}", enum=["minimal", "high"]),
|
||||
None, id="exact-minimal-accepted"),
|
||||
pytest.param("medium", _error("Invalid option", param="{field}", enum=["none"]),
|
||||
None, id="only-none"),
|
||||
pytest.param("medium", _error("Invalid option", param="{field}", enum=["future", "turbo"]),
|
||||
None, id="unknown-tiers"),
|
||||
pytest.param("medium", _error("Invalid option", param="temperature", enum=["minimal"]),
|
||||
None, id="foreign-field"),
|
||||
pytest.param("low", _error("{field} value 'low' is not supported"),
|
||||
None, id="unadvertised-walk-stops-at-low"),
|
||||
])
|
||||
def test_advertised_minimal_is_a_comparable_tier(evidence_root, carrier, body_error, requested, error, applied):
|
||||
field = {"top": "reasoning_effort", "nested": "reasoning.effort"}[carrier]
|
||||
error = {key: value.format(field=field) if isinstance(value, str) else value for key, value in error.items()}
|
||||
target = _target()
|
||||
source = _value_payload(carrier, target, requested)
|
||||
with request_wire_call_scope():
|
||||
sent = prepare_wire_payload_for_send(target, source, api_surface="chat.completions")
|
||||
failure = error if body_error else _exception(error)
|
||||
if not body_error:
|
||||
failure.physical_attempt_capture = _capture(sent, target)
|
||||
retry = plan_next_wire_retry(sent, error=failure, body_error=body_error)
|
||||
assert (payload_effort(retry) if retry is not None else None) == applied
|
||||
assert _store(evidence_root) is None # Planning alone never learns a contract.
|
||||
|
||||
|
||||
@pytest.mark.parametrize("error", [
|
||||
_error(ENUM_MESSAGE, param="temperature"),
|
||||
_error(ENUM_MESSAGE, code="insufficient_quota"),
|
||||
_error(ENUM_MESSAGE, code="context_length_exceeded"),
|
||||
_error(ENUM_MESSAGE, status=None),
|
||||
_error(ENUM_MESSAGE, status=429),
|
||||
_error("reasoning.effort: invalid option", param="reasoning.effort", code="invalid_option"),
|
||||
_error("reasoning.effort value 'ultra' invalid; temperature expected one of 'low', 'high'"),
|
||||
_error("reasoning.effort value 'high' invalid; expected one of 'low', 'medium'"),
|
||||
_error("reasoning.effort value 'ultra' invalid; temperature expected one of 'low', or 'high'"),
|
||||
_error("reasoning.effort value 'high' invalid; supported values are 'low', 'medium', or 'xhigh'"),
|
||||
_error(ENUM_MESSAGE, param="reasoning.effort", value="high"),
|
||||
])
|
||||
@pytest.mark.parametrize("body_error", [False, True])
|
||||
def test_other_fields_unknown_and_noncompatibility_errors_do_not_remove_effort(evidence_root, error, body_error):
|
||||
target = _target()
|
||||
source = _value_payload("nested", target, "ultra")
|
||||
with request_wire_call_scope():
|
||||
sent = prepare_wire_payload_for_send(target, source, api_surface="chat.completions")
|
||||
assert plan_next_wire_retry(sent, error=error if body_error else _exception(error), body_error=body_error) is None
|
||||
assert payload_effort(source) == "ultra"
|
||||
assert _store(evidence_root) is None
|
||||
|
||||
|
||||
def test_structured_enum_and_parameter_survive_sdk_exception_projection(evidence_root):
|
||||
target = _target()
|
||||
source = _value_payload("nested", target, "ultra")
|
||||
with request_wire_call_scope():
|
||||
sent = prepare_wire_payload_for_send(target, source, api_surface="chat.completions")
|
||||
retry = plan_next_wire_retry(sent, error=_exception(_error(
|
||||
"Invalid option", param="reasoning.effort", code="invalid_option", allowed_values=["low", "high"])))
|
||||
assert payload_effort(retry) == "high"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body_error", [False, True])
|
||||
@pytest.mark.parametrize("message,allowed", [
|
||||
('reasoning_effort: expected one of "and"|"or"', ("and", "or")),
|
||||
("reasoning_effort: allowed values are 'low', 'or', or 'high'", ("low", "or", "high")),
|
||||
("reasoning_effort: allowed values are low, medium, or high", ("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are 'low' or higher", ("low",)),
|
||||
("reasoning_effort: allowed values are 'or' and 'and'", ("or", "and")),
|
||||
("reasoning_effort: allowed values are low, medium, or high and xhigh requires a pro plan",
|
||||
("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are 'low', 'medium', or 'high' and 'xhigh' is not supported",
|
||||
("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are 'low', 'medium', 'high', xhigh requires a pro plan",
|
||||
("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are low, medium, high and xhigh is not supported",
|
||||
("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are low, medium, high and xhigh is unsupported",
|
||||
("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are 'low', 'medium', 'high', and 'xhigh' isn't supported",
|
||||
("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are 'low', 'medium', 'high', 'xhigh' (unsupported on this plan)",
|
||||
("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are low, medium, high, xhigh requires a pro plan",
|
||||
("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are low or medium or high",
|
||||
("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are 'high' (not recommended)", ("high",)),
|
||||
("reasoning_effort: allowed values are low, medium, or high",
|
||||
("low", "medium", "high")),
|
||||
("reasoning_effort: allowed values are 'low', 'medium', or 'high' and 'xhigh' is not supported",
|
||||
("low", "medium", "high")),
|
||||
('reasoning_effort: expected one of "low"|"medium"|"high" and "xhigh" requires a pro plan',
|
||||
("low", "medium", "high")),
|
||||
])
|
||||
def test_enum_conjunction_keeps_quoted_literals_and_stops_at_prose(body_error, message, allowed):
|
||||
from ouroboros.request_wire_recovery import _wire_rejection
|
||||
|
||||
error = _error(message)
|
||||
assert _wire_rejection(error if body_error else _exception(error)).allowed == allowed
|
||||
|
||||
|
||||
def test_rejection_from_another_physical_candidate_cannot_change_this_one(evidence_root):
|
||||
from ouroboros.request_wire_recovery import plan_wire_retry_from_exception
|
||||
|
||||
target = _target()
|
||||
source = _value_payload("nested", target, "ultra")
|
||||
with request_wire_call_scope():
|
||||
sent = prepare_wire_payload_for_send(target, source, api_surface="chat.completions")
|
||||
error = _exception(_error())
|
||||
error.physical_attempt_capture = {}
|
||||
assert plan_next_wire_retry(sent, error=error) is None
|
||||
assert plan_wire_retry_from_exception(error) is None
|
||||
error.physical_attempt_capture = _capture({**sent, "model": "other"}, target)
|
||||
assert plan_next_wire_retry(sent, error=error) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body_error", [False, True])
|
||||
def test_carrierless_repair_requires_its_prepared_logical_and_physical_input(evidence_root, body_error):
|
||||
from ouroboros.llm_attempt import _finalized_physical_candidate
|
||||
from ouroboros.send_clock import MainSendClock, SendClockPolicy
|
||||
|
||||
target = _target()
|
||||
source = {"model": target["resolved_model"], "messages": [{"role": "user", "content": "hi"}],
|
||||
"temperature": 0.7, "timeout": 123}
|
||||
rejection = _error("Unsupported parameter: temperature", param="temperature")
|
||||
error = rejection if body_error else _exception(rejection)
|
||||
with request_wire_call_scope(), MainSendClock(SendClockPolicy("UTC")).bound():
|
||||
sent = _finalized_physical_candidate(target, source, "chat.completions", fresh_clock=True)
|
||||
if not body_error:
|
||||
error.physical_attempt_capture = {}
|
||||
assert plan_next_wire_retry(source, error=error) is None
|
||||
error.physical_attempt_capture = _capture({**sent, "model": "foreign"}, target)
|
||||
assert plan_next_wire_retry(source, error=error) is None
|
||||
error.physical_attempt_capture = _capture(sent, target)
|
||||
assert plan_next_wire_retry({**source, "temperature": 0.8}, error=error, body_error=body_error) is None
|
||||
retry = plan_next_wire_retry(source, error=error, body_error=body_error)
|
||||
assert retry == {key: value for key, value in sent.items() if key != "temperature"}
|
||||
assert _store(evidence_root) is None # This path never teaches a durable contract.
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body_error", [False, True])
|
||||
@pytest.mark.parametrize("error,omitted", [
|
||||
(_error("thinking.type: Input should be 'adaptive'"), True),
|
||||
(_error("thinking.type: Input should be 'enabled', 'adaptive', or 'disabled'"), False),
|
||||
(_error("Invalid type", param="thinking.type", enum=["adaptive"], value="disabled"), True),
|
||||
(_error("Invalid type", param="thinking.type", enum=["adaptive", "disabled"]), False),
|
||||
(_error("Invalid type", param="thinking.type", enum=["adaptive"], value="enabled"), False),
|
||||
(_error("thinking.type: Input should be 'adaptive'", param="output_config.effort"), False),
|
||||
(_error("thinking value 'none' is unsupported", param="thinking"), False),
|
||||
])
|
||||
def test_disabled_carrier_type_refusal_is_not_an_individual_tier_refusal(evidence_root, body_error, error, omitted):
|
||||
target = _target("anthropic", "future")
|
||||
source = {"model": "future", "messages": [{"role": "user", "content": "hi"}],
|
||||
"thinking": {"type": "disabled"}, "max_tokens": 128}
|
||||
with request_wire_call_scope():
|
||||
sent = prepare_wire_payload_for_send(target, source, api_surface="messages")
|
||||
failure = error if body_error else _exception(error)
|
||||
if not body_error:
|
||||
failure.physical_attempt_capture = _capture(sent, target)
|
||||
retry = plan_next_wire_retry(source, error=failure, body_error=body_error)
|
||||
assert retry == ({key: value for key, value in sent.items() if key != "thinking"} if omitted else None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body_error", [False, True])
|
||||
def test_logical_input_binding_survives_preparation_but_rejects_changed_input(evidence_root, body_error):
|
||||
from ouroboros.llm_attempt import _finalized_physical_candidate
|
||||
from ouroboros.send_clock import MainSendClock, SendClockPolicy
|
||||
|
||||
target = _target()
|
||||
source = _value_payload("nested", target, "ultra")
|
||||
source["timeout"] = 123
|
||||
source["messages"][0]["_context_capsule"] = {"kind": "host-only"}
|
||||
error = _error() if body_error else _exception(_error())
|
||||
with request_wire_call_scope(), MainSendClock(SendClockPolicy("UTC")).bound():
|
||||
sent = _finalized_physical_candidate(target, source, "chat.completions", fresh_clock=True)
|
||||
assert "timeout" not in sent
|
||||
assert "_context_capsule" not in sent["messages"][0]
|
||||
assert plan_next_wire_retry({**source, "model": "other"}, error=error, body_error=body_error) is None
|
||||
retry = plan_next_wire_retry(source, error=error, body_error=body_error)
|
||||
assert payload_effort(retry) == "max"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("body_error", [False, True])
|
||||
@pytest.mark.parametrize("nano", [False, True])
|
||||
def test_real_driver_recovery_binds_timeout_capsule_and_clock(evidence_root, tmp_path, asynchronous, body_error, nano):
|
||||
from ouroboros.send_clock import MainSendClock, SendClockPolicy
|
||||
|
||||
target = {**_target(), **({"context_mode": "nano"} if nano else {})}
|
||||
source = _value_payload("nested", target, "ultra")
|
||||
if nano:
|
||||
source["max_tokens"] = 200000 # Forces real allowance reduction before sealing.
|
||||
source["timeout"] = 123
|
||||
source["messages"][0]["_context_capsule"] = {"kind": "host-only"}
|
||||
original, sent = copy.deepcopy(source), []
|
||||
|
||||
def send(**candidate):
|
||||
sent.append(copy.deepcopy(candidate))
|
||||
if len(sent) == 1:
|
||||
if body_error:
|
||||
return _Response({"error": _error(), "choices": [], "usage": {}})
|
||||
raise _exception(_error())
|
||||
return _Response()
|
||||
|
||||
client = LLMClient(api_key="unused")
|
||||
with ua.usage_scope(ua.UsageScope(drive_root=tmp_path, task_id="binding")), MainSendClock(SendClockPolicy("UTC")).bound():
|
||||
if asynchronous:
|
||||
async def async_send(**candidate):
|
||||
return send(**candidate)
|
||||
asyncio.run(client._create_chat_completion_with_retries_async(async_send, source, target))
|
||||
else:
|
||||
client._create_chat_completion_with_retries(send, source, target)
|
||||
assert source == original
|
||||
assert [payload_effort(candidate) for candidate in sent] == ["ultra", "max"]
|
||||
assert all(candidate["timeout"] == 123 for candidate in sent)
|
||||
assert all("_context_capsule" not in candidate["messages"][0] for candidate in sent)
|
||||
if nano:
|
||||
assert all(candidate["max_tokens"] < original["max_tokens"] for candidate in sent)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("control", ["budget", "stop"])
|
||||
@pytest.mark.parametrize("requested,error", [
|
||||
("ultra", _error()),
|
||||
("none", _error("Reasoning is mandatory", param="reasoning.effort", enum=["medium", "high"])),
|
||||
("low", _error("Invalid option", param="reasoning.effort", enum=["medium", "high"])),
|
||||
])
|
||||
def test_reactive_effort_repair_never_adds_a_send_allowance(evidence_root, tmp_path, monkeypatch, control, requested, error):
|
||||
from ouroboros import llm_attempt
|
||||
|
||||
target = _target()
|
||||
source = _value_payload("nested", target, requested)
|
||||
sent = []
|
||||
|
||||
def send(**candidate):
|
||||
sent.append(candidate)
|
||||
if control == "stop":
|
||||
def stopped():
|
||||
raise llm_attempt.PhysicalDispatchInterrupted("stop")
|
||||
monkeypatch.setattr(llm_attempt, "require_physical_dispatch_window", stopped)
|
||||
raise _exception(error)
|
||||
|
||||
failure = ua.PhysicalAttemptLimitExceeded if control == "budget" else llm_attempt.PhysicalDispatchInterrupted
|
||||
with request_wire_call_scope(), ua.usage_scope(ua.UsageScope(drive_root=tmp_path, task_id="limit")), ua.physical_attempt_limit(1), pytest.raises(failure):
|
||||
LLMClient(api_key="unused")._create_chat_completion_with_retries(send, source, target)
|
||||
assert len(sent) == 1
|
||||
assert _store(evidence_root) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider,requested,sent_effort", [("deepseek", "ultra", "max"), ("zai", "medium", "high"), ("anthropic", "minimal", "low")])
|
||||
def test_original_preference_survives_native_mapping_in_usage_and_ledger(evidence_root, tmp_path, monkeypatch, provider, requested, sent_effort):
|
||||
from ouroboros.llm_attempt import (
|
||||
_attempt_request,
|
||||
_candidate_before_dispatch,
|
||||
_execute_candidate,
|
||||
attach_processing_receipt,
|
||||
)
|
||||
|
||||
target = _target(provider, "future")
|
||||
client = LLMClient(api_key="unused")
|
||||
source = client._build_remote_candidate(target, [{"role": "user", "content": "hello"}], requested, 32, "auto", None, None)
|
||||
assert payload_effort(source) == sent_effort
|
||||
with ua.usage_scope(ua.UsageScope(drive_root=tmp_path, task_id="projection")):
|
||||
request = _attempt_request(target, source)
|
||||
_execute_candidate(request, lambda: _Response(), _candidate_before_dispatch(source, request))
|
||||
usage = {}
|
||||
attach_processing_receipt(target, usage)
|
||||
assert usage["effort"]["requested"] == requested
|
||||
assert usage["effort"]["reported"] is None
|
||||
rows = [json.loads(line) for line in (tmp_path / ua.LEDGER_REL).read_text().splitlines()]
|
||||
reserved = next(row for row in rows if row["state"] == "reserved")
|
||||
assert reserved["effort"] == usage["effort"]
|
||||
assert rows[-1]["state"] == "settled"
|
||||
assert rows[-1]["effort"] == usage["effort"]
|
||||
|
||||
|
||||
def test_switch_model_next_round_keeps_preference_after_native_projection(tmp_path, monkeypatch):
|
||||
from ouroboros.loop import _apply_runtime_overrides
|
||||
from ouroboros.tools.control_runtime import _switch_model
|
||||
from ouroboros.tools.registry import ToolRegistry
|
||||
|
||||
ctx = ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path)._ctx
|
||||
monkeypatch.setattr(LLMClient, "available_models", lambda *_: ["zai::future", "openai::future"])
|
||||
assert "next round" in _switch_model(ctx, model="zai::future", effort="ultra")
|
||||
model, local, effort = _apply_runtime_overrides(ctx, "openai::future", False, "medium")
|
||||
assert (model, local, effort) == ("zai::future", False, "ultra")
|
||||
target = _target("zai", "future")
|
||||
payload = LLMClient(api_key="unused")._build_remote_candidate(target, [], effort, 32, "auto", None, None)
|
||||
assert payload_effort(payload) == "max"
|
||||
assert _apply_runtime_overrides(ctx, model, local, effort) == (model, local, "ultra")
|
||||
assert "next round" in _switch_model(ctx, model="openai::future")
|
||||
assert _apply_runtime_overrides(ctx, model, local, effort) == ("openai::future", False, "ultra")
|
||||
|
||||
|
||||
def test_native_omission_is_recorded_without_inventing_provider_effort(tmp_path, monkeypatch):
|
||||
from types import SimpleNamespace
|
||||
|
||||
sent = []
|
||||
completion = SimpleNamespace(choices=[SimpleNamespace(
|
||||
message=SimpleNamespace(content="done", function_call=None), finish_reason="stop")])
|
||||
client = LLMClient(api_key="unused")
|
||||
monkeypatch.setattr(client, "_get_gigachat_client", lambda *a, **kw: SimpleNamespace(
|
||||
chat=lambda payload: sent.append(payload) or completion))
|
||||
target = _target("gigachat", "GigaChat")
|
||||
with ua.usage_scope(ua.UsageScope(drive_root=tmp_path, task_id="omitted")):
|
||||
_, usage = client._chat_gigachat(target, [], None, "ultra", 32, "auto")
|
||||
assert len(sent) == 1 and "reasoning_effort" not in sent[0]
|
||||
assert usage["effort"] == {
|
||||
"requested": "ultra", "sent": {}, "sent_state": "omitted", "sent_source": "host_candidate",
|
||||
"reported": None, "report_source": None,
|
||||
}
|
||||
rows = [json.loads(line) for line in (tmp_path / ua.LEDGER_REL).read_text().splitlines()]
|
||||
assert rows[-1]["state"] == "settled"
|
||||
assert rows[-1]["effort"] == usage["effort"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
def test_local_driver_retains_known_intent_with_omitted_effort(tmp_path, monkeypatch, asynchronous):
|
||||
from types import SimpleNamespace
|
||||
from ouroboros import llm_local, local_model
|
||||
|
||||
sent = []
|
||||
client = LLMClient(api_key="unused")
|
||||
manager = SimpleNamespace(serving_context_evidence=lambda: {})
|
||||
monkeypatch.setattr(local_model, "get_manager", lambda: manager)
|
||||
monkeypatch.setattr(llm_local, "local_context_limits", lambda maximum: (0, maximum))
|
||||
monkeypatch.setattr(client, "_get_local_client", lambda: SimpleNamespace(chat=SimpleNamespace(
|
||||
completions=SimpleNamespace(create=lambda **payload: sent.append(payload) or _Response()))))
|
||||
with ua.usage_scope(ua.UsageScope(drive_root=tmp_path, task_id="local-intent")):
|
||||
kwargs = dict(messages=[{"role": "user", "content": "hi"}], model="local", use_local=True,
|
||||
reasoning_effort="ultra", max_tokens=32)
|
||||
_, usage = asyncio.run(client.chat_async(**kwargs)) if asynchronous else client.chat(**kwargs)
|
||||
assert len(sent) == 1 and "reasoning_effort" not in sent[0] and "reasoning" not in sent[0]
|
||||
assert usage["effort"] == {
|
||||
"requested": "ultra", "sent": {}, "sent_state": "omitted", "sent_source": "host_candidate",
|
||||
"reported": None, "report_source": None,
|
||||
}
|
||||
rows = [json.loads(line) for line in (tmp_path / ua.LEDGER_REL).read_text().splitlines()]
|
||||
assert rows[-1]["state"] == "settled" and rows[-1]["effort"] == usage["effort"]
|
||||
|
||||
|
||||
def test_provider_usage_cannot_supply_host_effort_evidence(monkeypatch):
|
||||
from ouroboros import llm_attempt
|
||||
|
||||
monkeypatch.setattr(llm_attempt, "last_physical_attempt_capture", lambda: None)
|
||||
usage = {"effort": {"reported": "ultra", "sent_source": "host_candidate"},
|
||||
"effort_resolution": {"observed": "ultra"}}
|
||||
llm_attempt.attach_processing_receipt(_target(), usage)
|
||||
assert "effort" not in usage
|
||||
assert "effort_resolution" not in usage
|
||||
|
||||
|
||||
def test_late_receipt_keeps_candidate_effort_without_inheriting_an_older_report(tmp_path):
|
||||
from ouroboros.llm_attempt import _attempt_request
|
||||
|
||||
target = {**_target(), "requested_reasoning_effort": "ultra"}
|
||||
request = _attempt_request(target, _value_payload("nested", target, "max"))
|
||||
with ua.usage_scope(ua.UsageScope(drive_root=tmp_path, task_id="late")):
|
||||
reservation = ua.reserve_attempt(request)
|
||||
ua.mark_dispatched(reservation)
|
||||
# A historical administrative row may contain an older observation.
|
||||
ua._transition(reservation, "settled", settle_reason="abandoned", cost_usd=None, cost_final=False,
|
||||
effort={**request.effort, "reported": "high", "report_source": "provider_response"})
|
||||
ua.settle_attempt(reservation, {}, cost_usd=0.25, cost_final=True)
|
||||
rows = [json.loads(line) for line in (tmp_path / ua.LEDGER_REL).read_text().splitlines()]
|
||||
assert rows[-2]["effort"]["reported"] == "high"
|
||||
assert rows[-1]["effort"] == request.effort
|
||||
assert rows[-1]["settle_reason"] == "late_receipt"
|
||||
assert rows[-1]["cost_usd"] == 0.25 and rows[-1]["cost_final"] is True
|
||||
185
tests/test_effort_logs_browser.py
Normal file
185
tests/test_effort_logs_browser.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
"""Effort facts in the real Logs SPA, live and after history reload, never in Chat.
|
||||
|
||||
The provider and engine results are synthetic fixtures (the shared effort
|
||||
resolution fixtures), not vendor captures. Production code builds and persists
|
||||
the facts: ``effort_request_facts`` for API sends, ``llm_claudexor._usage`` for
|
||||
managed model operations, ``final_attempt_facts`` plus the custody writer for a
|
||||
delegated session, the supervisor ``llm_usage`` handler, and ``/api/logs``.
|
||||
Only the WebSocket transport and bootstrap/state envelope are fixtures; there
|
||||
is no supervisor process.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.routing import Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from ouroboros import delegate_custody
|
||||
from ouroboros.gateway.logs import api_logs_tail
|
||||
from ouroboros.gateways.claudexor import final_attempt_facts
|
||||
from ouroboros.llm_attempt import effort_request_facts
|
||||
from ouroboros.llm_claudexor import _usage as managed_usage
|
||||
from supervisor.events_budget import _handle_llm_usage
|
||||
from tests.test_delegated_activity_browser import engine_ui as engine_ui
|
||||
from tests.test_subscription_setup_browser import capture
|
||||
from tests.test_subscription_setup_browser import subscription_ui as subscription_ui
|
||||
from tests.test_usage_writer_view import root as root
|
||||
|
||||
pytestmark = [pytest.mark.ui_browser, pytest.mark.serial]
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
||||
PAIRED = json.loads((FIXTURES / "claudexor_effort_resolution.json").read_text(encoding="utf-8"))
|
||||
VARIANTS = json.loads((FIXTURES / "claudexor_effort_resolution_variants.json").read_text(encoding="utf-8"))
|
||||
|
||||
# Exact Logs bodies: every requested/sent/reported fact is explicit or "unknown".
|
||||
EXPECTED = {
|
||||
"effort-api-explicit": 'Effort requested ultra · host sent {"extra_body.reasoning":'
|
||||
'{"effort":"max","exclude":false}} · reported unknown',
|
||||
"effort-api-omitted": "Effort requested medium · host sent omitted · reported unknown",
|
||||
"effort-managed-prepared": 'Effort requested ultra · host sent {"options.reasoningEffort":"ultra"}'
|
||||
" · prepared reasoning.effort=xhigh (downward; account_catalog) · reported unknown",
|
||||
"effort-managed-reported": 'Effort requested ultra · host sent {"options.reasoningEffort":"ultra"}'
|
||||
" · prepared reasoning.effort=xhigh (downward; account_catalog)"
|
||||
" · reported xhigh (codex.responses.reasoning.effort)",
|
||||
"effort-managed-omitted": 'Effort requested ultra · host sent {"options.reasoningEffort":"ultra"}'
|
||||
" · prepared omitted (omitted; adapter) · reported unknown",
|
||||
"effort-session": "Engine requested effort ultra · host sent unknown"
|
||||
" · prepared reasoning.effort=low (floor; account_catalog) · reported unknown",
|
||||
}
|
||||
DISCLOSURE_MARKERS = ("Effort requested", "Engine requested effort", "host sent", "prepared ")
|
||||
|
||||
|
||||
def _usage_events(root: Path) -> list[dict]:
|
||||
api = {"provider": "openrouter", "resolved_model": "vendor/future", "requested_reasoning_effort": "ultra"}
|
||||
managed = {"provider": "claudexor", "requested_reasoning_effort": "ultra"}
|
||||
explicit = {"model": "vendor/future", "extra_body": {"reasoning": {"effort": "max", "exclude": False}}}
|
||||
managed_facts = effort_request_facts(managed, {"options": {"reasoningEffort": "ultra"}})
|
||||
# The engine's one model reporter mirrors the provider's appliedOptions echo
|
||||
# into observed/observedSource, so a reported result carries both, equal.
|
||||
reported = {"appliedOptions": {"reasoningEffort": "xhigh"}, "effortResolution": {
|
||||
**PAIRED, "observed": "xhigh", "observedSource": "codex.responses.reasoning.effort"}}
|
||||
return [
|
||||
("effort-api-explicit", "openrouter", {"effort": effort_request_facts(api, explicit)}),
|
||||
("effort-api-omitted", "openai",
|
||||
{"effort": effort_request_facts({**api, "requested_reasoning_effort": "medium"}, {"model": "future"})}),
|
||||
("effort-managed-prepared", "claudexor", managed_usage({"effortResolution": PAIRED}, managed_facts)[0]),
|
||||
("effort-managed-reported", "claudexor", managed_usage(reported, managed_facts)[0]),
|
||||
("effort-managed-omitted", "claudexor",
|
||||
managed_usage({"effortResolution": VARIANTS["known_parameter_omitted"]}, managed_facts)[0]),
|
||||
]
|
||||
|
||||
|
||||
def _settle_session(root: Path) -> dict:
|
||||
run_dir = root / "runs" / "run-effort" / "final"
|
||||
run_dir.mkdir(parents=True)
|
||||
(run_dir / "telemetry.yaml").write_text(json.dumps({
|
||||
"run_id": "run-effort", "final_attempt_id": "attempt-2",
|
||||
"attempts": [{"attempt_id": "attempt-1", "observed_model": "vendor/old"},
|
||||
{"attempt_id": "attempt-2", "harness_id": "codex", "observed_model": "vendor/future",
|
||||
"profile_id": "work", "effort_resolution": VARIANTS["engine_order_floor"]}],
|
||||
}), encoding="utf-8")
|
||||
observed = final_attempt_facts({"summary": {"runDir": str(run_dir.parent)}}, "run-effort")
|
||||
assert observed["effort_resolution"] == VARIANTS["engine_order_floor"]
|
||||
assert delegate_custody.emit(root, delegate_custody.SETTLED, {
|
||||
"run_id": "run-effort", "task_id": "effort-session", "route": "codex",
|
||||
"source": "subagent", "category": "task", "model": observed["model"],
|
||||
"observed_attempt": observed, "state": "succeeded",
|
||||
})
|
||||
return observed
|
||||
|
||||
|
||||
def _disclosed(page, selector: str) -> list[str]:
|
||||
text = page.locator(selector).inner_text() if page.locator(selector).count() else ""
|
||||
return [marker for marker in DISCLOSURE_MARKERS if marker in text]
|
||||
|
||||
|
||||
def _navigate(page, target: str):
|
||||
if page.viewport_size["width"] < 700: # The rail is an off-canvas drawer on phones.
|
||||
page.locator('[data-mobile-nav-toggle]:visible').first.click()
|
||||
page.click(f'[data-nav-page="{target}"]')
|
||||
|
||||
|
||||
def _open_logs(page):
|
||||
_navigate(page, "dashboard")
|
||||
page.wait_for_selector('#page-logs', state="visible")
|
||||
for details in page.locator('#log-entries .log-task-details').all():
|
||||
if details.get_attribute('open') is None:
|
||||
details.locator(':scope > summary').click()
|
||||
|
||||
|
||||
def _assert_logs(page, bodies: dict[str, str]):
|
||||
for task, body in bodies.items():
|
||||
page.locator('#log-entries').get_by_text(body, exact=True).first.wait_for(state="attached", timeout=10000)
|
||||
assert page.locator('#log-entries').get_by_text(body, exact=True).count() == 1, task
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [1360, 390])
|
||||
def test_effort_facts_render_only_in_logs_live_and_after_reload(engine_ui, root, monkeypatch, width):
|
||||
ui, page, sockets, published = engine_ui, engine_ui["page"], [], []
|
||||
evidence = Path(os.environ.get("OUROBOROS_UI_EVIDENCE_DIR")
|
||||
or Path(os.environ["OUROBOROS_TEST_TEMP_ROOT"]) / "effort-logs-browser")
|
||||
engine = page.context.browser.browser_type.name
|
||||
assert engine == os.environ["OUROBOROS_UI_BROWSER_ENGINE"] # Evidence names the engine that ran.
|
||||
monkeypatch.setenv("OUROBOROS_UI_EVIDENCE_DIR", str(evidence))
|
||||
page.set_viewport_size({"width": width, "height": 900})
|
||||
running = {task: {"task": {"id": task, "chat_id": 1, "type": "task"}} for task in EXPECTED}
|
||||
ctx = SimpleNamespace(RUNNING=running, DRIVE_ROOT=root, bridge=SimpleNamespace(push_log=published.append))
|
||||
page.route_web_socket("**/ws", lambda socket: sockets.append(socket))
|
||||
app = Starlette(routes=[Route("/api/logs/{name}", api_logs_tail)])
|
||||
app.state.drive_root = root
|
||||
with TestClient(app) as client:
|
||||
def forward(route):
|
||||
parsed = urlparse(route.request.url)
|
||||
response = client.get(parsed.path + ("?" + parsed.query if parsed.query else ""))
|
||||
route.fulfill(status=response.status_code, content_type="application/json", body=response.content)
|
||||
|
||||
page.route("**/api/logs/**", forward)
|
||||
page.goto(ui["url"] + "/")
|
||||
page.wait_for_selector('#chat-messages', state="attached")
|
||||
for _ in range(100):
|
||||
if sockets:
|
||||
break
|
||||
page.wait_for_timeout(50)
|
||||
assert sockets
|
||||
for task, provider, usage in _usage_events(root):
|
||||
_handle_llm_usage({"type": "llm_usage", "task_id": task, "model": f"{provider}/fixture",
|
||||
"provider": provider, "source": "loop", "usage": usage}, ctx)
|
||||
assert [row["task_id"] for row in published] == list(EXPECTED)[:-1]
|
||||
assert all(row["chat_id"] == 1 for row in published)
|
||||
for row in published:
|
||||
sockets[-1].send(json.dumps({"type": "log", "chat_id": 1, "data": row}))
|
||||
# Positive control after the effort frames: this socket does reach Chat.
|
||||
sockets[-1].send(json.dumps({"type": "chat", "role": "assistant", "chat_id": 1,
|
||||
"content": "Control reply after effort frames"}))
|
||||
page.locator('#chat-messages').get_by_text("Control reply after effort frames").wait_for()
|
||||
assert _disclosed(page, '#chat-messages') == []
|
||||
assert page.locator('#toast-stack .toast').count() == 0
|
||||
capture(page, f"{engine}-{width}-chat-after-live-effort")
|
||||
_open_logs(page)
|
||||
live = {task: body for task, body in EXPECTED.items() if task != "effort-session"}
|
||||
_assert_logs(page, live)
|
||||
capture(page, f"{engine}-{width}-logs-live")
|
||||
observed = _settle_session(root)
|
||||
page.reload()
|
||||
page.wait_for_selector('#chat-messages', state="attached")
|
||||
assert _disclosed(page, '#chat-messages') == []
|
||||
_open_logs(page)
|
||||
_assert_logs(page, EXPECTED)
|
||||
assert page.locator('#toast-stack .toast').count() == 0
|
||||
assert page.evaluate("document.documentElement.scrollWidth <= innerWidth")
|
||||
page.locator('#log-entries').get_by_text(EXPECTED["effort-session"], exact=True).scroll_into_view_if_needed()
|
||||
capture(page, f"{engine}-{width}-logs-history-reloaded")
|
||||
_navigate(page, "chat")
|
||||
assert _disclosed(page, '#chat-messages') == []
|
||||
history = client.get("/api/logs/events?limit=50").json()["entries"]
|
||||
persisted = {row["task_id"]: row for row in history if row.get("task_id") in EXPECTED}
|
||||
assert set(persisted) == set(EXPECTED)
|
||||
assert persisted["effort-session"]["observed_attempt"] == observed
|
||||
evidence.mkdir(parents=True, exist_ok=True)
|
||||
(evidence / f"{engine}-{width}-events.json").write_text(json.dumps(history, indent=2), encoding="utf-8")
|
||||
268
tests/test_effort_resolution_evidence.py
Normal file
268
tests/test_effort_resolution_evidence.py
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
"""The shared engine fixture reaches model/session consumers without false echoes."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from ouroboros.effort_evidence import validated_effort_resolution
|
||||
from ouroboros.gateways.claudexor import final_attempt_facts
|
||||
from ouroboros.llm_claudexor import _ModelInvocation
|
||||
from tests._review_session_route_shared import (
|
||||
FakeLLM,
|
||||
_agent_request,
|
||||
_agent_slot,
|
||||
_terminal_detail,
|
||||
)
|
||||
from tests._review_session_route_shared import (
|
||||
_owned_gateway_uses_each_test_transport as _owned_fixture,
|
||||
)
|
||||
from tests._review_session_route_shared import (
|
||||
fake_route as _route_fixture,
|
||||
)
|
||||
from tests.test_claudexor_observed_attempt import _write_telemetry
|
||||
from tests.test_llm_claudexor import MODEL, ledger
|
||||
from tests.test_llm_claudexor import result as _model_result
|
||||
from tests.test_llm_claudexor import setup as _model_fixture
|
||||
|
||||
_owned_gateway_uses_each_test_transport = _owned_fixture
|
||||
fake_route = _route_fixture
|
||||
model_setup = _model_fixture
|
||||
|
||||
SHARED = json.loads((Path(__file__).parent / "fixtures/claudexor_effort_resolution.json").read_text())
|
||||
VARIANTS = json.loads((Path(__file__).parent / "fixtures/claudexor_effort_resolution_variants.json").read_text())
|
||||
# Claudexor owns the ordered proof, including future tokens and native orders
|
||||
# that differ from the host preference scale. These are engine claims, not echoes.
|
||||
REPORTS = {"paired_downward": SHARED, **VARIANTS}
|
||||
|
||||
|
||||
def _finish(report, applied=None):
|
||||
invocation = _ModelInvocation(
|
||||
{"usage_model": "claudexor::codex=model", "requested_reasoning_effort": "ultra"},
|
||||
{"options": {"reasoningEffort": "ultra"}}, {})
|
||||
return invocation.finish({"outcome": "completed", "message": {"content": "done"},
|
||||
"effortResolution": report, "appliedOptions": applied})[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("applied", [None, {}, {"cacheKey": "unrelated"}])
|
||||
@pytest.mark.parametrize("report", REPORTS.values(), ids=REPORTS)
|
||||
def test_paired_fixture_is_preparation_evidence_not_dispatch_or_provider_echo(applied, report):
|
||||
usage = _finish(report, applied)
|
||||
assert usage["effort_resolution"] == report
|
||||
assert usage["effort"]["requested"] == "ultra"
|
||||
assert usage["effort"]["sent"] == {"options.reasoningEffort": "ultra"}
|
||||
assert usage["effort"]["reported"] is None
|
||||
assert usage["claudexor"]["applied_options"] == applied
|
||||
|
||||
|
||||
def test_provider_response_echo_remains_observation_when_resolution_has_none():
|
||||
usage = _finish(SHARED, {"reasoningEffort": "xhigh"})
|
||||
assert usage["effort"]["reported"] == "xhigh"
|
||||
assert usage["effort"]["report_source"] == "provider_applied_options"
|
||||
assert usage["claudexor"]["applied_options_source"] == "provider_response"
|
||||
assert usage["claudexor"]["applied_options"] == {"reasoningEffort": "xhigh"}
|
||||
|
||||
|
||||
def test_observation_reporter_is_separate_from_resolution_authority():
|
||||
report = {**SHARED, "observed": "high", "observedSource": "provider_response"}
|
||||
usage = _finish(report)
|
||||
assert usage["effort"]["reported"] == "high"
|
||||
assert usage["effort"]["report_source"] == "provider_response"
|
||||
assert usage["effort_resolution"]["source"] == "account_catalog"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("report", [None, {}, [], {"requested": "ultra"},
|
||||
{**SHARED, "submitted": None}, {**SHARED, "parameter": None},
|
||||
{**SHARED, "requested": None}, {**SHARED, "resolution": "exact"},
|
||||
{**SHARED, "resolution": "floor", "requested": None},
|
||||
{**SHARED, "resolution": "floor", "submitted": None},
|
||||
{**SHARED, "resolution": "omitted"}, {**SHARED, "resolution": "unverifiable"},
|
||||
{**SHARED, "resolution": "rejected"},
|
||||
{**SHARED, "resolution": "future-resolution"},
|
||||
{**SHARED, "resolution": []}, {**SHARED, "source": "unknown"},
|
||||
{**SHARED, "source": []}, {**SHARED, "observed": "xhigh"},
|
||||
{**SHARED, "observedSource": "provider_response"},
|
||||
{**SHARED, "reason": []},
|
||||
{**SHARED, "resolution": "rejected", "submitted": None, "parameter": None,
|
||||
"observed": "high", "observedSource": "provider_response"},
|
||||
*[{key: value for key, value in SHARED.items() if key != missing} for missing in SHARED],
|
||||
*[{**SHARED, field: invalid} for field in ("requested", "submitted", "parameter", "observed", "observedSource")
|
||||
for invalid in (" ", 1, False, [], {})],
|
||||
{**VARIANTS["known_parameter_unverifiable"], "parameter": " "},
|
||||
])
|
||||
def test_malformed_or_incoherent_reports_stay_unknown_in_both_consumers(tmp_path, report):
|
||||
assert validated_effort_resolution(report) is None
|
||||
usage = _finish(report)
|
||||
assert usage["effort_resolution"] is None
|
||||
assert usage["effort"]["reported"] is None
|
||||
assert usage["claudexor"]["applied_options"] is None
|
||||
detail = _write_telemetry(tmp_path, [{"attempt_id": "a02", "effort_resolution": report}])
|
||||
assert final_attempt_facts(detail, "run-fixture")["effort_resolution"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("report", REPORTS.values(), ids=REPORTS)
|
||||
def test_paired_fixture_binds_only_to_unique_final_session_attempt(tmp_path, report):
|
||||
detail = _write_telemetry(tmp_path, [
|
||||
{"attempt_id": "a01", "effort_resolution": {**SHARED, "observed": "wrong", "observedSource": "earlier"}},
|
||||
{"attempt_id": "a02", "effort_resolution": report},
|
||||
])
|
||||
assert final_attempt_facts(detail, "run-fixture")["effort_resolution"] == report
|
||||
detail = _write_telemetry(tmp_path, [{"attempt_id": "a01", "effort_resolution": report}, {"attempt_id": "a02"}])
|
||||
assert "effort_resolution" not in final_attempt_facts(detail, "run-fixture")
|
||||
detail = _write_telemetry(tmp_path, [{"attempt_id": "a02", "effort_resolution": report}, {"attempt_id": "a02"}])
|
||||
assert final_attempt_facts(detail, "run-fixture") == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token", ["future-token", "none"])
|
||||
def test_exact_tokens_are_not_reranked_and_literal_none_is_not_omission(token):
|
||||
report = {**SHARED, "requested": token, "submitted": token, "resolution": "exact"}
|
||||
assert validated_effort_resolution(report) == report
|
||||
|
||||
|
||||
@pytest.mark.parametrize("resolution", ["omitted", "unverifiable", "rejected"])
|
||||
@pytest.mark.parametrize("parameter", [None, "model_reasoning_effort"])
|
||||
def test_absent_knob_is_null_and_never_confirms_a_tier(resolution, parameter):
|
||||
report = {**SHARED, "submitted": None, "parameter": parameter, "resolution": resolution}
|
||||
assert validated_effort_resolution(report) == report
|
||||
assert _finish(report)["effort"]["reported"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("report", REPORTS.values(), ids=REPORTS)
|
||||
def test_final_session_report_reaches_review_ledger_and_last_execution(tmp_path, monkeypatch, fake_route, report):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from ouroboros.review_substrate import run_review_request
|
||||
from ouroboros.reviewer_slot_config import record_reviewer_slot_executions, reviewer_slot_last_executions
|
||||
from ouroboros.usage_accounting import LEDGER_REL
|
||||
|
||||
monkeypatch.setattr("ouroboros.config.DATA_DIR", tmp_path / "canonical-data")
|
||||
fake_route.detail = _terminal_detail('[]')
|
||||
fake_route.telemetry = {"run_id": "run-1", "final_attempt_id": "a02", "attempts": [
|
||||
{"attempt_id": "a02", "harness_id": "fake-review", "observed_model": "fake-small",
|
||||
"effort_resolution": report},
|
||||
]}
|
||||
slot = _agent_slot()
|
||||
result = run_review_request(_agent_request(), slots=[slot], drive_root=tmp_path, llm=FakeLLM())
|
||||
actor = result.actors[0]
|
||||
assert actor["usage"]["effort_resolution"] == report
|
||||
record_reviewer_slot_executions("scope_review", [SimpleNamespace(**actor)], {slot.slot_id: slot})
|
||||
effective = reviewer_slot_last_executions()[slot.slot_id]["effective"]
|
||||
assert effective["effort_resolution"] == report
|
||||
assert "effort" not in effective # Prepared effort must not become an applied scalar.
|
||||
rows = [json.loads(line) for line in (tmp_path / LEDGER_REL).read_text().splitlines()]
|
||||
assert any(row.get("effort_resolution") == report for row in rows)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("reported", [None, "xhigh"])
|
||||
@pytest.mark.parametrize("report", REPORTS.values(), ids=REPORTS)
|
||||
def test_shared_model_fixture_survives_real_driver_and_monetary_settlement(model_setup, asynchronous, reported, report):
|
||||
import asyncio
|
||||
|
||||
root, gateway, client = model_setup
|
||||
response = _model_result(cash=0.25, knowledge="exact")
|
||||
response.update(effortResolution=report,
|
||||
appliedOptions={"reasoningEffort": reported} if reported else {})
|
||||
gateway.results = [response]
|
||||
args = ([{"role": "user", "content": "hi"}], MODEL)
|
||||
if asynchronous:
|
||||
_, usage = asyncio.run(client.chat_async(*args, reasoning_effort="ultra"))
|
||||
else:
|
||||
_, usage = client.chat(*args, reasoning_effort="ultra")
|
||||
assert usage["effort_resolution"] == report
|
||||
assert usage["effort"]["requested"] == "ultra"
|
||||
assert usage["effort"]["reported"] == reported
|
||||
assert usage["effort"]["report_source"] == ("provider_applied_options" if reported else None)
|
||||
assert usage["claudexor"]["applied_options"] == response["appliedOptions"]
|
||||
rows = ledger(root)
|
||||
assert [row["state"] for row in rows] == ["reserved", "dispatched", "settled"]
|
||||
assert rows[-1]["effort_resolution"] == report
|
||||
assert rows[-1]["effort"] == usage["effort"]
|
||||
assert rows[-1]["cost_usd"] == 0.25
|
||||
|
||||
|
||||
def test_old_model_result_without_resolution_keeps_effort_unknown(model_setup):
|
||||
root, gateway, client = model_setup
|
||||
response = _model_result(cash=0.25, knowledge="exact")
|
||||
response.pop("appliedOptions")
|
||||
gateway.results = [response]
|
||||
_, usage = client.chat([{"role": "user", "content": "hi"}], MODEL, reasoning_effort="ultra")
|
||||
assert "effort_resolution" not in usage
|
||||
assert usage["effort"]["reported"] is None
|
||||
assert usage["effort"]["report_source"] is None
|
||||
assert usage["claudexor"]["applied_options"] is None
|
||||
assert not ledger(root)[-1].get("effort_resolution")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("online", [False, True])
|
||||
@pytest.mark.parametrize("evidence", ["missing", "prepared", "applied", "observed", "malformed", "malformed_applied"])
|
||||
def test_late_model_result_keeps_only_exact_attempt_effort_evidence(tmp_path, monkeypatch, online, evidence):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from ouroboros import usage_accounting as ua
|
||||
from ouroboros.llm_attempt import effort_request_facts
|
||||
from ouroboros.llm_claudexor import recover_model_attempt
|
||||
from ouroboros.observability import persist_call
|
||||
|
||||
target = {"usage_model": MODEL, "requested_reasoning_effort": "ultra"}
|
||||
payload = {"options": {"reasoningEffort": "ultra"}}
|
||||
facts = effort_request_facts(target, payload)
|
||||
result = _model_result(cash=0.25, knowledge="exact")
|
||||
result.pop("appliedOptions", None)
|
||||
if evidence != "missing":
|
||||
result["effortResolution"] = {} if evidence == "malformed" else dict(SHARED)
|
||||
if evidence in {"applied", "observed"}:
|
||||
result["appliedOptions"] = {"reasoningEffort": "xhigh"}
|
||||
if evidence == "observed":
|
||||
result["effortResolution"].update(observed="high", observedSource="provider_response")
|
||||
if evidence == "malformed_applied":
|
||||
result["appliedOptions"] = {"reasoningEffort": ["xhigh"]}
|
||||
with ua.usage_scope(ua.UsageScope(drive_root=tmp_path, task_id="late")):
|
||||
reservation = ua.reserve_attempt(ua.AttemptRequest(model=MODEL, provider="claudexor", effort=facts))
|
||||
ua.mark_dispatched(reservation)
|
||||
ua._transition(reservation, "settled", settle_reason="abandoned", cost_usd=None, cost_final=False,
|
||||
effort={**facts, "reported": "stale", "report_source": "older_receipt"},
|
||||
effort_resolution={**SHARED, "observed": "stale", "observedSource": "older_receipt"})
|
||||
row = ledger(tmp_path)[-1]
|
||||
attempt = reservation.attempt_id
|
||||
persist_call(tmp_path, task_id="late", call_id=f"{attempt}_model_request",
|
||||
call_type="llm_claudexor_request", payload=payload, keep_raw=True,
|
||||
manifest={"invocation_id": attempt, "operation_id": "exact-operation"})
|
||||
raw = json.dumps(result)
|
||||
calls = []
|
||||
|
||||
def get_operation(operation, **kwargs):
|
||||
assert operation == "exact-operation"
|
||||
calls.append("read_operation")
|
||||
return {"state": "succeeded", "dispatch": {"state": "response_received"},
|
||||
"response": {"state": "ready", "ref": {"sha256": "fixture-ref"}}}
|
||||
|
||||
def get_result(operation, **kwargs):
|
||||
assert operation == "exact-operation" and kwargs["expected_ref"] == {"sha256": "fixture-ref"}
|
||||
calls.append("read_result")
|
||||
return raw.encode()
|
||||
|
||||
def factory():
|
||||
assert online, "retained terminal evidence needs no gateway"
|
||||
return SimpleNamespace(get_model_operation=get_operation, get_model_result=get_result,
|
||||
acknowledge_model_result=lambda *args: calls.append("ack"),
|
||||
close=lambda: calls.append("close"))
|
||||
|
||||
if not online:
|
||||
persist_call(tmp_path, task_id="late", call_id=f"{attempt}_model_response",
|
||||
call_type="llm_claudexor_response", payload={"result_json_utf8": raw}, keep_raw=True,
|
||||
manifest={"invocation_id": attempt, "operation_id": "exact-operation",
|
||||
"operation_state": "succeeded", "dispatch_state": "response_received"})
|
||||
monkeypatch.setattr("ouroboros.llm_claudexor.read_owned_gateway", factory)
|
||||
state, usage, cost, final = recover_model_attempt(tmp_path, row)
|
||||
assert state == "settled"
|
||||
live = _ModelInvocation(target, payload, {}).extract_usage(result)[0]
|
||||
assert usage.get("effort_resolution") == live.get("effort_resolution")
|
||||
assert usage["effort"] == live["effort"]
|
||||
ua.settle_attempt(reservation, usage, cost_usd=cost, cost_final=final)
|
||||
settled = ledger(tmp_path)[-1]
|
||||
assert settled["effort"] == live["effort"]
|
||||
assert settled.get("effort_resolution") == live.get("effort_resolution")
|
||||
assert settled["cost_usd"] == 0.25 and settled["cost_final"] is True
|
||||
assert settled["settle_reason"] == "late_receipt"
|
||||
assert len({entry["attempt_id"] for entry in ledger(tmp_path)}) == 1
|
||||
assert calls == (["read_operation", "read_result", "ack", "close"] if online else [])
|
||||
53
tests/test_effort_settings_browser.py
Normal file
53
tests/test_effort_settings_browser.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
"""Rendered Settings keeps reviewer overrides and describes effort as a preference."""
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests import test_subscription_role_routes_browser as roles
|
||||
|
||||
pytestmark = [pytest.mark.ui_browser, pytest.mark.serial]
|
||||
subscription_ui = roles.subscription_ui
|
||||
role_ui = roles.role_ui
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [1360, 390])
|
||||
def test_reviewer_effort_override_caption_and_saved_preference(role_ui, tmp_path, width):
|
||||
ui = role_ui
|
||||
roles.configure_mixed(ui)
|
||||
ui["settings"]["OUROBOROS_SUBAGENTS"]["items"][0]["effort"] = "high"
|
||||
slots = ui["fixture"]["preview"]["reviewer_slots"]
|
||||
slots["scope"][0]["effort"] = "low"
|
||||
ui["settings"]["OUROBOROS_REVIEWER_SLOTS"] = json.dumps(slots)
|
||||
ui["page"].set_viewport_size({"width": width, "height": 900})
|
||||
page = roles.open_agents(ui)
|
||||
row = page.locator('[data-slot-id="scope_1"]')
|
||||
caption = row.locator('.reviewer-slot-meta')
|
||||
assert "preferred effort low" in caption.inner_text()
|
||||
row.locator('[data-slot-effort]').select_option('medium')
|
||||
assert "preferred effort medium" in caption.inner_text()
|
||||
deep = page.locator('[data-deep-review-row]')
|
||||
deep_caption = deep.locator('.reviewer-slot-meta')
|
||||
assert "preferred effort high" in deep_caption.inner_text()
|
||||
deep.locator('[data-deep-review-effort]').select_option('low')
|
||||
assert "preferred effort low" in deep_caption.inner_text()
|
||||
deep.locator('[data-deep-review-effort]').select_option('')
|
||||
assert "preferred effort high" in deep_caption.inner_text()
|
||||
deep.locator('[data-deep-review-effort]').select_option('medium')
|
||||
assert "preferred effort medium" in deep_caption.inner_text()
|
||||
row.scroll_into_view_if_needed()
|
||||
assert page.evaluate("document.documentElement.scrollWidth <= innerWidth")
|
||||
page.screenshot(path=str(tmp_path / f"effort-reviewer-{width}.png"))
|
||||
with page.expect_response('**/api/settings'):
|
||||
page.locator('#btn-save-settings').click()
|
||||
saved = [body for path, body in ui["posts"] if path == '/api/settings'][-1]
|
||||
assert json.loads(saved['OUROBOROS_REVIEWER_SLOTS'])['scope'][0]['effort'] == 'medium'
|
||||
assert json.loads(saved['OUROBOROS_REVIEWER_SLOTS'])['deep_review']['effort'] == 'medium'
|
||||
with page.expect_response('**/api/reviewer-slots'):
|
||||
page.locator('#btn-reload-settings').click()
|
||||
assert "preferred effort medium" in caption.inner_text()
|
||||
assert "preferred effort medium" in deep_caption.inner_text()
|
||||
page.locator('[data-settings-tab="behavior"]').click()
|
||||
help_text = page.get_by_text('Preferred reasoning effort per task type.', exact=False)
|
||||
assert "Requested, sent and reported effort are recorded in Logs" in help_text.inner_text()
|
||||
help_text.scroll_into_view_if_needed()
|
||||
page.screenshot(path=str(tmp_path / f"effort-settings-{width}.png"))
|
||||
|
|
@ -325,3 +325,26 @@ def test_llm_usage_serializer_carries_web_search_sources():
|
|||
src = (pathlib.Path(__file__).resolve().parent.parent
|
||||
/ "supervisor" / "events_budget.py").read_text(encoding="utf-8")
|
||||
assert "web_search_sources" in src
|
||||
|
||||
|
||||
def test_effort_facts_with_legacy_option_status_survive_logs_without_notices(tmp_path):
|
||||
"""An old option_status stays in both logs without creating a notice."""
|
||||
from types import SimpleNamespace
|
||||
from supervisor import events
|
||||
|
||||
(tmp_path / "logs").mkdir()
|
||||
frames = []
|
||||
facts = {
|
||||
"effort": {"requested": "ultra", "sent": {"reasoning_effort": "max"},
|
||||
"sent_state": "explicit", "reported": None, "report_source": None},
|
||||
"request_wire": {"requested_effort": "ultra", "applied_effort": "max",
|
||||
"applied_effort_source": "sent_candidate"},
|
||||
"claudexor": {"requested_options": {"reasoningEffort": "ultra"},
|
||||
"applied_options": {}, "option_status": {"reasoningEffort": "unknown"}},
|
||||
}
|
||||
ctx = SimpleNamespace(DRIVE_ROOT=tmp_path, bridge=SimpleNamespace(push_log=frames.append))
|
||||
events._handle_llm_usage({"type": "llm_usage", "task_id": "t", "usage": facts}, ctx)
|
||||
written = json.loads((tmp_path / "logs/events.jsonl").read_text())
|
||||
assert {key: written[key] for key in facts} == facts
|
||||
assert frames == [written]
|
||||
assert "toast_once" not in written and "task_incident" not in written
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ class TestAliasProducerFanOutSweep:
|
|||
("ouroboros/tools/search.py", "cost_usd", "_web_search"): ("ledger settlement call (web search attempt)", 1),
|
||||
# usage/observability event streams (events.jsonl, live log frames;
|
||||
# /api/logs replay converts to the honest name at the boundary)
|
||||
("ouroboros/loop_llm_call.py", "cost_usd", "call_llm_with_retry"): ("llm_round usage event rows", 2),
|
||||
("ouroboros/loop_llm_call.py", "cost_usd", "call_llm_with_retry"): ("llm_round usage event rows", 1),
|
||||
("ouroboros/post_task_synthesis.py", "cost_usd", "_run_chat_consolidation"): ("chat_block_consolidation event row", 1),
|
||||
("ouroboros/post_task_synthesis.py", "cost_usd", "_run_reflection"): ("reflection generation gate args", 1),
|
||||
("supervisor/events_evolution_done.py", "cost_usd", "_handle_evolution_task_done"): ("internal lifecycle/checkpoint call kwargs + supervisor.jsonl observability row", 3),
|
||||
|
|
|
|||
|
|
@ -274,6 +274,71 @@ def test_local_candidate_is_measured_after_existing_local_transform(data_root, m
|
|||
assert final["candidate_raw_size_bytes"] == len(raw)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
@pytest.mark.parametrize("timeout", [33.0, None, 0.0, -1.0])
|
||||
@pytest.mark.parametrize("measurement_supported", [False, True], ids=["estimated", "measured"])
|
||||
def test_local_transport_timeout_is_separate_from_physical_candidate(
|
||||
data_root, monkeypatch, asynchronous, timeout, measurement_supported,
|
||||
):
|
||||
from importlib import import_module
|
||||
|
||||
import httpx
|
||||
from openai import OpenAI
|
||||
|
||||
from ouroboros.local_model_server import input_fingerprint
|
||||
from ouroboros.observability import read_blob_ref
|
||||
|
||||
client = LLMClient(api_key="unused")
|
||||
sent, transport_timeouts, measured = [], [], []
|
||||
|
||||
def handle(request):
|
||||
sent.append(json.loads(request.content))
|
||||
transport_timeouts.append(request.extensions["timeout"])
|
||||
return httpx.Response(200, json=_Response(text="local").model_dump())
|
||||
|
||||
def measure(payload):
|
||||
measured.append(copy.deepcopy(payload))
|
||||
return {
|
||||
"supported": measurement_supported, "input_is_exact": measurement_supported,
|
||||
"input_tokens": 8, "context_window": 8192, "process_id": 123,
|
||||
"native_input_sha256": input_fingerprint(payload),
|
||||
"output_limit_enforced": True, "reasoning_included_in_limit": True,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(import_module("ouroboros.local_model"), "get_manager", lambda: SimpleNamespace(
|
||||
serving_context_evidence=lambda: {"context_window": 8192, "confirmed": True, "process_id": 123},
|
||||
measure_prepared_input=measure,
|
||||
))
|
||||
messages = [{"role": "user", "content": "hello", "_context_capsule": {"generation": 2}}]
|
||||
original = copy.deepcopy(messages)
|
||||
kwargs = dict(messages=messages, model="local", use_local=True, max_tokens=512, timeout=timeout)
|
||||
with OpenAI(
|
||||
base_url="http://127.0.0.1:8799/v1", api_key="local", max_retries=0, timeout=91.0,
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(handle)),
|
||||
) as transport, ua.usage_scope(_scope(data_root)), ua.physical_attempt_limit(1):
|
||||
monkeypatch.setattr(client, "_get_local_client", lambda: transport)
|
||||
message, usage = asyncio.run(client.chat_async(**kwargs)) if asynchronous else client.chat(**kwargs)
|
||||
|
||||
assert message["content"] == "local" and len(sent) == 1
|
||||
# Assert the SDK's actual HTTP timeout, including its unchanged default
|
||||
# when the caller supplies no positive override, independently of kwargs.
|
||||
expected_timeout = timeout if timeout is not None and timeout > 0 else 91.0
|
||||
assert transport_timeouts == [dict.fromkeys(("connect", "read", "write", "pool"), expected_timeout)]
|
||||
physical = sent[0]
|
||||
assert measured == [physical] and "timeout" not in measured[0]
|
||||
assert not _has_capsule(physical) and messages == original
|
||||
raw = _canonical_candidate_bytes(physical)
|
||||
rows = _rows(data_root)
|
||||
assert [row["state"] for row in rows] == ["reserved", "dispatched", "settled"]
|
||||
assert all(row["candidate_raw_sha256"] == hashlib.sha256(raw).hexdigest() for row in rows)
|
||||
assert all(row["candidate_raw_size_bytes"] == len(raw) for row in rows)
|
||||
assert rows[0]["reservation_upper_bound_usd"] == rows[-1]["cost_usd"] == 0.0
|
||||
assert usage["cost"] == 0.0 and usage["cost_final"] is True
|
||||
assert usage["ledger_attempt_ids"] == [rows[-1]["attempt_id"]]
|
||||
manifest = _manifest(rows[-1]["candidate_manifest_ref"])
|
||||
assert read_blob_ref(data_root, manifest["full_payload_ref"]) == physical
|
||||
|
||||
|
||||
def test_local_read_timeout_after_dispatch_is_not_retried(data_root, monkeypatch):
|
||||
import httpx
|
||||
import ouroboros.local_model as local_model
|
||||
|
|
|
|||
|
|
@ -197,41 +197,6 @@ def test_provider_terminal_text_claims_only_recorded_recovery_facts():
|
|||
assert "same-model reroute" not in unknown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("honored", ["confirmed", "unknown"])
|
||||
def test_applied_options_without_mismatch_emit_no_owner_line(honored):
|
||||
progress = []
|
||||
usage = {"_options": {"options_honored": honored}}
|
||||
|
||||
loop_transport.emit_model_effort_mismatch(
|
||||
usage, task_id="task-7",
|
||||
emit_progress=lambda text, *, incident=None: progress.append((text, incident)),
|
||||
)
|
||||
|
||||
assert progress == []
|
||||
|
||||
|
||||
def test_owner_line_speaks_only_for_a_changed_reasoning_effort():
|
||||
"""A mismatch on another submitted option is durable, never an effort claim."""
|
||||
progress = []
|
||||
route = {"credentialProfileId": "acct-a", "model": "codex=model"}
|
||||
usage = {"_model_route": dict(route), "_options": {
|
||||
"options_honored": "mismatch", "route": dict(route),
|
||||
"requested_options": {"reasoningEffort": "high", "cacheKey": "execution-a"},
|
||||
"applied_options": {"reasoningEffort": "high", "cacheKey": "engine-b"}}}
|
||||
|
||||
def emit(text, *, incident=None):
|
||||
progress.append(text)
|
||||
|
||||
loop_transport.emit_model_effort_mismatch(usage, task_id="task-7", emit_progress=emit)
|
||||
assert progress == [] and usage["_options"]["options_honored"] == "mismatch"
|
||||
|
||||
# The silent round spent no dedupe slot: a real effort change still speaks.
|
||||
usage["_options"]["applied_options"] = {"reasoningEffort": "low", "cacheKey": "engine-b"}
|
||||
loop_transport.emit_model_effort_mismatch(usage, task_id="task-7", emit_progress=emit)
|
||||
assert progress == ["⚠️ Claudexor served at low effort while high was requested"
|
||||
" (Claudexor account acct-a)."]
|
||||
|
||||
|
||||
def _mismatch_round_context(tmp_path, monkeypatch, *, emit_progress, applied_values):
|
||||
"""A Main round whose subscription answer reports a lowered effort."""
|
||||
registry = ToolRegistry(repo_dir=tmp_path, drive_root=tmp_path)
|
||||
|
|
@ -246,23 +211,22 @@ def _mismatch_round_context(tmp_path, monkeypatch, *, emit_progress, applied_val
|
|||
|
||||
route = {"credentialProfileId": "account-a", "model": "codex=model"}
|
||||
|
||||
def call(_llm, _messages, _model, _tools, _effort, _retries, _logs, _tid,
|
||||
_round, _queue, usage, *_args, **_kwargs):
|
||||
usage["_options"] = {
|
||||
"requested_options": {"reasoningEffort": "high"},
|
||||
"applied_options": {"reasoningEffort": next(applied_values)},
|
||||
"options_honored": "mismatch",
|
||||
"route": dict(route),
|
||||
def chat(**_kwargs):
|
||||
reported = next(applied_values)
|
||||
return {"role": "assistant", "content": "done"}, {
|
||||
"provider": "claudexor", "cost": 0.0,
|
||||
"effort": {"requested": "high", "sent": {"options.reasoningEffort": "high"},
|
||||
"reported": reported, "report_source": "provider_applied_options"},
|
||||
"claudexor": {"route": dict(route), "requested_options": {"reasoningEffort": "high"},
|
||||
"applied_options": {"reasoningEffort": reported}},
|
||||
}
|
||||
usage["_model_route"] = dict(route)
|
||||
return {"role": "assistant", "content": "done"}, 0.0
|
||||
|
||||
monkeypatch.setattr(loop, "call_llm_with_retry", call)
|
||||
ctx.llm = SimpleNamespace(chat=chat)
|
||||
monkeypatch.setattr(loop, "_server_web_allowed_by_task", lambda _ctx: False)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_effort_mismatch_emits_one_typed_owner_line_per_task_and_model(tmp_path, monkeypatch):
|
||||
def test_effort_mismatch_emits_no_chat_toast_or_incident_across_rounds(tmp_path, monkeypatch):
|
||||
progress = []
|
||||
ctx = _mismatch_round_context(
|
||||
tmp_path, monkeypatch, applied_values=iter(("medium", "low")),
|
||||
|
|
@ -271,29 +235,26 @@ def test_effort_mismatch_emits_one_typed_owner_line_per_task_and_model(tmp_path,
|
|||
loop._dispatch_round_model(ctx, None, attempt_cap=None)
|
||||
loop._dispatch_round_model(ctx, None, attempt_cap=None)
|
||||
|
||||
assert len(progress) == 1
|
||||
text, incident = progress[0]
|
||||
assert "served at medium effort while high was requested" in text
|
||||
assert "Claudexor account account-a" in text
|
||||
assert incident == {
|
||||
"task_incident": "model_effort_mismatch",
|
||||
"toast_once": "task-7:model_effort_mismatch:codex=model",
|
||||
}
|
||||
assert progress == []
|
||||
rows = [json.loads(line) for line in (ctx.drive_logs / "events.jsonl").read_text().splitlines()]
|
||||
rounds = [row for row in rows if row.get("type") == "llm_round"]
|
||||
assert [row["effort"]["reported"] for row in rounds] == ["medium", "low"]
|
||||
assert "_options" not in ctx.accumulated_usage
|
||||
|
||||
|
||||
def test_failed_round_route_never_borrows_the_previous_applied_options(tmp_path, monkeypatch):
|
||||
"""Only the route that reported applied options can be named in its line."""
|
||||
"""Failed rounds cannot borrow previous options into a fresh success log."""
|
||||
from ouroboros.llm_claudexor import ClaudexorModelError
|
||||
|
||||
logs = tmp_path / "logs"
|
||||
logs.mkdir(parents=True, exist_ok=True)
|
||||
usage, progress = {}, []
|
||||
usage = {}
|
||||
route_a = {"model": "MODEL-A", "credentialProfileId": "acct-a", "source": "codex"}
|
||||
route_b = {"model": "MODEL-B", "credentialProfileId": "acct-b", "source": "claude"}
|
||||
|
||||
def served(route, requested, applied):
|
||||
return {"role": "assistant", "content": "done"}, {"claudexor": {
|
||||
"route": dict(route), "options_honored": "mismatch",
|
||||
"route": dict(route), "applied_options_source": "provider_response",
|
||||
"requested_options": {"reasoningEffort": requested},
|
||||
"applied_options": {"reasoningEffort": applied}}}
|
||||
|
||||
|
|
@ -303,7 +264,6 @@ def test_failed_round_route_never_borrows_the_previous_applied_options(tmp_path,
|
|||
|
||||
rounds = [lambda: served(route_a, "xhigh", "low"), lambda: failed(route_b),
|
||||
lambda: served(route_b, "high", "medium")]
|
||||
lines_after = []
|
||||
|
||||
for index, step in enumerate(rounds):
|
||||
monkeypatch.setattr(loop_llm_call, "_send_main_candidate",
|
||||
|
|
@ -311,18 +271,16 @@ def test_failed_round_route_never_borrows_the_previous_applied_options(tmp_path,
|
|||
loop_llm_call.call_llm_with_retry(
|
||||
SimpleNamespace(), [], "MODEL", [], "xhigh", 1, logs, "task-1", index, None, usage,
|
||||
"task", attempt_cap=1, initial_messages=[])
|
||||
loop_transport.emit_model_effort_mismatch(
|
||||
usage, task_id="task-1",
|
||||
emit_progress=lambda text, *, incident=None: progress.append((text, incident)))
|
||||
lines_after.append(len(progress))
|
||||
if index == 1: # the failed round names its own route and carries no applied options
|
||||
assert usage["_model_route"] == route_b and usage["_options"]["route"] == route_a
|
||||
assert "_options" not in usage
|
||||
if index == 1:
|
||||
assert usage["_model_route"] == route_b
|
||||
|
||||
assert lines_after == [1, 1, 2] # the failed round adds nothing; MODEL-B speaks for itself
|
||||
assert [incident["toast_once"] for _text, incident in progress] == [
|
||||
"task-1:model_effort_mismatch:MODEL-A", "task-1:model_effort_mismatch:MODEL-B"]
|
||||
assert progress[1][0] == ("⚠️ Claudexor served at medium effort while high was requested"
|
||||
" (Claudexor account acct-b).")
|
||||
rows = [json.loads(line) for line in (logs / "events.jsonl").read_text().splitlines()]
|
||||
rounds = [row for row in rows if row.get("type") == "llm_round"]
|
||||
assert [row["claudexor"]["route"] for row in rounds] == [route_a, route_b]
|
||||
assert [row["claudexor"]["requested_options"]["reasoningEffort"] for row in rounds] == ["xhigh", "high"]
|
||||
assert [row["claudexor"]["applied_options"]["reasoningEffort"] for row in rounds] == ["low", "medium"]
|
||||
assert all(row["claudexor"]["applied_options_source"] == "provider_response" for row in rounds)
|
||||
|
||||
|
||||
def test_mismatch_round_never_calls_the_one_argument_tool_context_emitter(tmp_path, monkeypatch):
|
||||
|
|
@ -334,7 +292,7 @@ def test_mismatch_round_never_calls_the_one_argument_tool_context_emitter(tmp_pa
|
|||
|
||||
loop._dispatch_round_model(ctx, None, attempt_cap=None)
|
||||
|
||||
assert seen == [] and ctx.accumulated_usage["_options"]["options_honored"] == "mismatch"
|
||||
assert seen == [] and "_options" not in ctx.accumulated_usage
|
||||
|
||||
|
||||
def test_body_error_diagnostic_is_masked_before_terminal_publication(tmp_path, monkeypatch):
|
||||
|
|
|
|||
|
|
@ -671,3 +671,22 @@ def test_the_terminal_hint_claims_every_account_only_from_the_pool_verdict():
|
|||
assert "every compatible account blocked" in proven and FALLBACK in proven
|
||||
assert "every compatible account" not in unproven and "unproven" in unproven
|
||||
assert "scheduled" not in proven + unproven and "nothing sleeps" in unproven
|
||||
|
||||
|
||||
def test_fallback_keeps_original_effort_instead_of_previous_sent_or_reported(main_call, one_fallback, monkeypatch):
|
||||
ctx, _gateway, _owner, events, _decide, _observations = main_call
|
||||
ctx.accumulated_usage["request_wire"] = {"original_requested_effort": "ultra", "applied_effort": "low"}
|
||||
seen = []
|
||||
|
||||
def candidate(round_call):
|
||||
seen.append(round_call.active_effort)
|
||||
return {"role": "assistant", "content": "done"}, 0.0, "max"
|
||||
|
||||
monkeypatch.setattr(loop, "_call_round_model", candidate)
|
||||
message, *_rest = loop._run_cross_model_fallback_chain(
|
||||
llm=ctx.llm, ctx=ctx.tools._ctx, tools=ctx.tools, messages=ctx.messages, active_model=ctx.active_model,
|
||||
active_use_local=False, tool_schemas=[], active_effort="ultra", max_retries=1, drive_logs=ctx.drive_logs,
|
||||
task_id=ctx.task_id, round_idx=1, event_queue=events, accumulated_usage=ctx.accumulated_usage,
|
||||
task_type="task", emit_progress=lambda *_a, **_kw: None, context_fit_plan=ctx.context_fit_plan,
|
||||
active_context_mode="max")
|
||||
assert message["content"] == "done" and seen == ["ultra"]
|
||||
|
|
|
|||
|
|
@ -253,6 +253,11 @@ def test_returned_response_discloses_its_exact_wire_candidate(
|
|||
|
||||
disclosure = usage["request_wire"]
|
||||
assert disclosure == {
|
||||
"original_requested_effort": None,
|
||||
"requested_effort_source": "provider_projection",
|
||||
"applied_effort_source": "sent_candidate",
|
||||
"reported_effort": None,
|
||||
"reported_effort_source": None,
|
||||
"requested_effort": "high",
|
||||
"applied_effort": "medium",
|
||||
"requested_tool_dialect": "function",
|
||||
|
|
|
|||
|
|
@ -379,7 +379,7 @@ def test_prescribed_lower_effort_tier_wins_over_the_one_rung_walk(evidence_root)
|
|||
assert upward is not None and payload_effort(upward) == "medium"
|
||||
|
||||
|
||||
def test_prescribed_jump_needs_a_quoted_tier_inside_the_retry_floor(evidence_root):
|
||||
def test_prose_and_unbound_quotes_keep_the_existing_one_rung_floor(evidence_root):
|
||||
target = _target(provider="openai")
|
||||
|
||||
with request_wire_call_scope():
|
||||
|
|
|
|||
|
|
@ -59,7 +59,8 @@ def test_main_loop_shares_one_install_affinity_across_executions_on_claudexor(tm
|
|||
assert captured[2]["cache_affinity"] == ""
|
||||
|
||||
|
||||
def test_main_loop_projects_claudexor_options_outside_the_route(tmp_path):
|
||||
def test_main_loop_keeps_legacy_claudexor_options_in_log_without_notice_state(tmp_path):
|
||||
"""Legacy options_honored is retained as an opaque log fact, never notice state."""
|
||||
class LLM:
|
||||
def chat(self, **_kwargs):
|
||||
return ({"content": "done", "tool_calls": [], "finish_reason": "stop"}, {
|
||||
|
|
@ -79,11 +80,11 @@ def test_main_loop_projects_claudexor_options_outside_the_route(tmp_path):
|
|||
logs, "task", 1, queue.Queue(), usage)
|
||||
|
||||
assert usage["_model_route"] == {"credentialProfileId": "account-a"}
|
||||
# The options carry the route that reported them, so a later round that
|
||||
# rewrites `_model_route` alone cannot be paired with these values.
|
||||
assert usage["_options"] == {
|
||||
"requested_options": {"reasoningEffort": "high"},
|
||||
"applied_options": {"reasoningEffort": "medium"},
|
||||
"options_honored": "mismatch",
|
||||
"route": {"credentialProfileId": "account-a"},
|
||||
}
|
||||
assert "_options" not in usage # No second, stale copy for automatic chat notices.
|
||||
import json
|
||||
|
||||
rounds = [json.loads(line) for line in (logs / "events.jsonl").read_text().splitlines()
|
||||
if json.loads(line).get("type") == "llm_round"]
|
||||
assert rounds[-1]["claudexor"]["requested_options"] == {"reasoningEffort": "high"}
|
||||
assert rounds[-1]["claudexor"]["applied_options"] == {"reasoningEffort": "medium"}
|
||||
assert rounds[-1]["claudexor"]["options_honored"] == "mismatch"
|
||||
|
|
|
|||
|
|
@ -994,8 +994,8 @@ def test_local_and_model_operation_transports_receive_no_stream_option(isolated,
|
|||
import ouroboros.llm_claudexor as operations
|
||||
client = LLMClient()
|
||||
seen = []
|
||||
def local(messages, tools, max_tokens, choice, *, timeout=None):
|
||||
seen.append({"timeout": timeout})
|
||||
def local(messages, tools, max_tokens, choice, *, timeout=None, reasoning_effort=None):
|
||||
seen.append({"timeout": timeout, "reasoning_effort": reasoning_effort})
|
||||
return {"content": "local"}, {}
|
||||
def operation(*args, **kwargs):
|
||||
seen.append(kwargs)
|
||||
|
|
|
|||
|
|
@ -436,6 +436,7 @@ class TestWrapupAffordability:
|
|||
def test_wire_recovery_matches_physical_candidate(self, monkeypatch, tmp_path):
|
||||
from ouroboros import llm as llm_module
|
||||
from ouroboros.llm import LLMClient
|
||||
from ouroboros.request_wire_contract import physical_candidate_bytes, physical_candidate_sha256
|
||||
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "unused")
|
||||
client = LLMClient(api_key="unused")
|
||||
|
|
@ -445,9 +446,13 @@ class TestWrapupAffordability:
|
|||
captured = {}
|
||||
real_prepare = llm_module.prepare_wire_payload_for_send
|
||||
|
||||
def prepare(target_, payload, *, api_surface):
|
||||
prepared = real_prepare(target_, payload, api_surface=api_surface)
|
||||
def prepare(target_, payload, *, api_surface, logical_payload=None):
|
||||
prepared = real_prepare(
|
||||
target_, payload, api_surface=api_surface, logical_payload=logical_payload,
|
||||
)
|
||||
prepared["recovered_wire_field"] = True
|
||||
captured.setdefault("prepared", []).append(prepared)
|
||||
captured.setdefault("logical", []).append(logical_payload)
|
||||
return prepared
|
||||
|
||||
def execute(request, _send, _before_dispatch):
|
||||
|
|
@ -472,10 +477,20 @@ class TestWrapupAffordability:
|
|||
target, messages, "high", prospective.max_completion_tokens, "auto", None, None,
|
||||
skip_capability_fetch=True, **_MAIN_LOOP_OPTIONS,
|
||||
)
|
||||
candidate["timeout"] = 33.0
|
||||
client._normalize_payload_cache_ttl(target, candidate)
|
||||
client._create_chat_completion_with_retries(lambda **_kwargs: None, candidate, target)
|
||||
|
||||
actual = captured["request"]
|
||||
assert len(captured["prepared"]) == len(captured["logical"]) == 2
|
||||
assert captured["logical"][-1] is candidate
|
||||
assert captured["logical"][-1]["timeout"] == 33.0
|
||||
assert all("timeout" not in payload for payload in captured["prepared"])
|
||||
prepared = captured["prepared"][-1]
|
||||
assert prepared["recovered_wire_field"] is True
|
||||
assert actual.candidate_raw_size_bytes == len(physical_candidate_bytes(prepared))
|
||||
assert actual.candidate_raw_sha256 == physical_candidate_sha256(prepared)
|
||||
assert prospective.prompt_tokens_estimate == actual.prompt_tokens_estimate
|
||||
assert prospective.candidate_raw_size_bytes == actual.candidate_raw_size_bytes
|
||||
assert prospective.candidate_raw_sha256 == actual.candidate_raw_sha256
|
||||
|
||||
|
|
|
|||
|
|
@ -290,6 +290,10 @@ def test_strict_child_schema_rejects_false_capture_and_wrong_receipt():
|
|||
vision_process._read_receipt(receipt, "ours")
|
||||
with pytest.raises(ValueError, match="another invocation"):
|
||||
vision_process._read_receipt(receipt, "theirs")
|
||||
capture["effort"] = ["not", "an", "evidence", "object"]
|
||||
with pytest.raises(Exception):
|
||||
vision_process._read_receipt(receipt, "ours")
|
||||
capture["effort"] = None
|
||||
capture["state"] = "free"
|
||||
with pytest.raises(Exception):
|
||||
vision_process._read_receipt(receipt, "ours")
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ def test_responses_flex_refusal_reprices_standard_and_keeps_original_intent(ctx,
|
|||
error.body = {"error": {"code": "resource_unavailable", "param": "service_tier"}}
|
||||
mock_openai.responses.create.side_effect = [error, _FakeStream([
|
||||
_make_event("response.output_text.delta", delta="answer"), _make_completed_event()])]
|
||||
result = json.loads(_web_search(ctx, "same query"))
|
||||
result = json.loads(_web_search(ctx, "same query", reasoning_effort="high"))
|
||||
assert result["answer"] == "answer"
|
||||
assert [call.kwargs["service_tier"] for call in mock_openai.responses.create.call_args_list] == ["flex", "default"]
|
||||
assert [request.processing_preference for request in prepared] == ["economy", "economy"]
|
||||
|
|
@ -132,6 +132,14 @@ def test_responses_flex_refusal_reprices_standard_and_keeps_original_intent(ctx,
|
|||
records = ua._read_records_locked_cached(holds[0].drive_root)
|
||||
matching = [row for row in records if row.get("candidate_raw_sha256") == prepared[0].candidate_raw_sha256]
|
||||
assert matching[-1]["state"] == "released"
|
||||
for request in prepared:
|
||||
assert request.effort == {
|
||||
"requested": "high", "sent": {"reasoning": {"effort": "high"}},
|
||||
"sent_state": "explicit", "sent_source": "host_candidate",
|
||||
"reported": None, "report_source": None,
|
||||
}
|
||||
attempt_rows = [row for row in records if row.get("candidate_raw_sha256") == request.candidate_raw_sha256]
|
||||
assert all(row["effort"] == request.effort for row in attempt_rows)
|
||||
|
||||
|
||||
def test_web_search_uses_official_openai_responses(monkeypatch):
|
||||
|
|
|
|||
23
web/modules/effort_evidence.js
Normal file
23
web/modules/effort_evidence.js
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
// Presentation only: resolution describes a prepared option, never dispatch.
|
||||
export function effortEvidenceText(evt) {
|
||||
const raw = evt.effort || evt.usage?.effort;
|
||||
const fact = raw && typeof raw === 'object' && !Array.isArray(raw) ? raw : null;
|
||||
const resolution = evt.effort_resolution || evt.usage?.effort_resolution
|
||||
|| evt.observed_attempt?.effort_resolution;
|
||||
if (!fact && !resolution) return '';
|
||||
const parts = [fact ? `Effort requested ${fact.requested ?? 'unknown'}`
|
||||
: `Engine requested effort ${resolution.requested ?? 'unknown'}`];
|
||||
const sent = fact?.sent_state === 'omitted' ? 'omitted'
|
||||
: fact?.sent && typeof fact.sent === 'object' ? JSON.stringify(fact.sent) : 'unknown';
|
||||
parts.push(`host sent ${sent}`);
|
||||
if (resolution) {
|
||||
const prepared = resolution.submitted == null ? 'omitted'
|
||||
: `${resolution.parameter}=${resolution.submitted}`;
|
||||
parts.push(`prepared ${prepared} (${resolution.resolution}; ${resolution.source})`);
|
||||
}
|
||||
const hasObserved = resolution?.observed != null && resolution?.observedSource;
|
||||
const observed = hasObserved ? resolution.observed : fact?.report_source ? fact.reported : null;
|
||||
const source = hasObserved ? resolution.observedSource : fact?.report_source;
|
||||
parts.push(`reported ${observed == null ? 'unknown' : `${observed}${source ? ` (${source})` : ''}`}`);
|
||||
return parts.join(' · ');
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ export { taskCheckpointLabel } from './task_checkpoints.js';
|
|||
import { harnessPresentation } from './harness_presentation.js';
|
||||
import { acceptanceIncidentClauses } from './acceptance_incident_presentation.js';
|
||||
import { historyRetentionView } from './history_retention.js';
|
||||
import { effortEvidenceText } from './effort_evidence.js';
|
||||
import { delegatedActivityView } from './delegated_activity.js';
|
||||
import {
|
||||
classifyReviewLifecycle,
|
||||
|
|
@ -27,11 +28,7 @@ export const LOG_CATEGORIES = {
|
|||
consciousness: { label: 'Consciousness', color: 'var(--accent)' },
|
||||
};
|
||||
|
||||
// Logs phases that file a row under the Errors filter (#323). They are the
|
||||
// typed failure outcomes summarizeLogEvent already derives — a failed task_done,
|
||||
// a tool that errored/was killed/timed out, an LLM call failure, a review
|
||||
// lifecycle error — so the category and the phase pill of one row can never
|
||||
// disagree, live or on replay.
|
||||
// The Errors filter follows the same typed failure phases on live and replay rows.
|
||||
const ERROR_LOG_PHASES = new Set(['error', 'timeout', 'lifecycle_error']);
|
||||
|
||||
export function categorizeLogEvent(evt, view = summarizeLogEvent(evt)) {
|
||||
|
|
@ -821,12 +818,11 @@ export function summarizeLogEvent(evt) {
|
|||
|
||||
if (t === 'llm_round_finished' || t === 'llm_round') {
|
||||
return view('done', `LLM round ${evt.round || ''} finished`.trim(), {
|
||||
body: effortEvidenceText(evt),
|
||||
meta: taskMeta(
|
||||
evt.model || '',
|
||||
formatLogTokens(evt),
|
||||
// ABI-3: /api/logs backfill rows carry the honest name; live
|
||||
// frames still say cost_usd/cost — resolve the pair via the
|
||||
// SSOT helper, then the live-frame `cost` spelling.
|
||||
// Backfill uses accounted upper bounds; live frames may carry cost.
|
||||
formatLogMoney(accountedUpperBound(evt) ?? evt.cost),
|
||||
evt.response_kind === 'tool_calls' ? `${evt.tool_call_count || 0} tool calls` : evt.response_kind || '',
|
||||
),
|
||||
|
|
@ -848,6 +844,7 @@ export function summarizeLogEvent(evt) {
|
|||
|
||||
if (t === 'llm_usage') {
|
||||
return view('usage', 'LLM usage recorded', {
|
||||
body: effortEvidenceText(evt),
|
||||
meta: taskMeta(
|
||||
evt.model || '',
|
||||
formatLogTokens(evt),
|
||||
|
|
@ -1081,7 +1078,7 @@ export function summarizeLogEvent(evt) {
|
|||
// only a name visible under Errors, and it is pinned as a remainder, not a taxonomy.
|
||||
const level = String(evt.level || '').toLowerCase();
|
||||
const body = shortText(
|
||||
evt.error || evt.message || evt.text || evt.result_preview
|
||||
evt.error || evt.message || evt.text || evt.result_preview || effortEvidenceText(evt)
|
||||
|| compactJson(evt.args || evt.task || evt.checks, 260), 260,
|
||||
);
|
||||
if (evt.ok === true) {
|
||||
|
|
|
|||
|
|
@ -331,10 +331,9 @@ export function subagentOptionsFor(roster, savedId, { rosterKnown = true, proces
|
|||
return options;
|
||||
}
|
||||
|
||||
export function describeSubagentReference(subagentId, roster, { rosterKnown = true, processingPreference = '' } = {}) {
|
||||
// The DERIVED facts, disclosed read-only (never editable knobs): the
|
||||
// roster row is the SSOT for a referenced reviewer's route/model/effort/
|
||||
// account, so this line only reports what that row says.
|
||||
export function describeSubagentReference(subagentId, roster, { rosterKnown = true, processingPreference = '', effort = '' } = {}) {
|
||||
// Route/model/account come from the roster; the reviewer row may override
|
||||
// the separate effort preference. Compound slugs retain their identity.
|
||||
const row = (roster || []).find(
|
||||
(item) => String(item.subagent_id || '') === String(subagentId || ''));
|
||||
if (!row) {
|
||||
|
|
@ -354,12 +353,14 @@ export function describeSubagentReference(subagentId, roster, { rosterKnown = tr
|
|||
: (route.target_id ? `API model ${route.target_id}` : 'API model (unset)'));
|
||||
}
|
||||
if (routeEditor.routeSupportsAccount(route) && route.credential_profile_id) parts.push(`account ${route.credential_profile_id}`);
|
||||
if (row.effort) parts.push(`effort ${row.effort}`);
|
||||
const preferredEffort = (route.kind === ROUTE_KIND_SESSION
|
||||
? routeEditor.compoundSessionEffort(route.target_id) : '') || effort || row.effort;
|
||||
if (preferredEffort) parts.push(`preferred effort ${preferredEffort}`);
|
||||
parts.push(`processing ${routeEditor.processingIntentLabel(row.processing_preference, processingPreference)}`);
|
||||
const off = row.enabled === false
|
||||
? '. That row is switched off, so this reference is refused at save rather than rerouted — choose another reviewer or turn the row back on'
|
||||
: '';
|
||||
return `Runs as ${parts.join(' · ')} — from its roster row under Available subagents${off}`;
|
||||
return `Runs as ${parts.join(' · ')} — route from its roster row under Available subagents${off}`;
|
||||
}
|
||||
|
||||
export function describeLastExecution(entry) {
|
||||
|
|
@ -382,7 +383,7 @@ export function describeLastExecution(entry) {
|
|||
}
|
||||
if (effective.profile_id) parts.push(`account ${effective.profile_id}`);
|
||||
if (effective.access) parts.push(`access ${effective.access}`);
|
||||
// No applied effort is rendered: none exists upstream, so the key is not emitted.
|
||||
// Effort reports stay structured and in Logs; prepared options are not applied facts.
|
||||
if (effective.verdict_method && effective.verdict_method !== 'structured'
|
||||
&& effective.verdict_method !== 'strict_parse') {
|
||||
parts.push(`verdict via ${effective.verdict_method.replace(/_/g, ' ')}`);
|
||||
|
|
@ -935,7 +936,7 @@ function rowHtml(row, group) {
|
|||
if (row.subagent_id) {
|
||||
const last = state.lastExecutions[row.slot_id];
|
||||
const metaParts = [
|
||||
describeSubagentReference(row.subagent_id, state.roster, { rosterKnown: state.rosterKnown, processingPreference: state.processingPreference }),
|
||||
describeSubagentReference(row.subagent_id, state.roster, { rosterKnown: state.rosterKnown, processingPreference: state.processingPreference, effort: row.effort }),
|
||||
];
|
||||
return `
|
||||
<div class="reviewer-slot-row" data-slot-group="${group}" data-slot-id="${escapeHtml(row.slot_id)}">
|
||||
|
|
@ -999,7 +1000,7 @@ function singletonHtml(spec) {
|
|||
const meta = (parts) => metaLineHtml(parts, row, last);
|
||||
if (row.subagent_id) {
|
||||
const metaParts = [
|
||||
describeSubagentReference(row.subagent_id, state.roster, { rosterKnown: state.rosterKnown, processingPreference: state.processingPreference }),
|
||||
describeSubagentReference(row.subagent_id, state.roster, { rosterKnown: state.rosterKnown, processingPreference: state.processingPreference, effort: row.effort }),
|
||||
...(spec.badgeOnReference ? [spec.badge(row)] : []),
|
||||
...spec.extraMeta(row),
|
||||
];
|
||||
|
|
@ -1255,6 +1256,7 @@ function bindRowEvents() {
|
|||
});
|
||||
rowEl.querySelector('[data-slot-effort]')?.addEventListener('change', (event) => {
|
||||
row.effort = String(event.target.value || '');
|
||||
renderRows({ discoveryOnly: true });
|
||||
state.onChange();
|
||||
});
|
||||
// Only an explicit switch writes delivery; a model or account change never does.
|
||||
|
|
@ -1348,6 +1350,7 @@ function bindSingletonEvents(section, spec) {
|
|||
// route/roster-row default on a session or subagent reference.
|
||||
row.effort = selected
|
||||
|| (row.subagent_id || row.route?.kind === ROUTE_KIND_SESSION ? '' : spec.apiEffortDefault);
|
||||
renderRows({ discoveryOnly: true });
|
||||
edited();
|
||||
});
|
||||
el.querySelector(`[data-${a}-processing]`)?.addEventListener('change', (event) => {
|
||||
|
|
|
|||
|
|
@ -579,7 +579,7 @@ export function effortSelectHtml(attrs, selected, surfaceDefault = 'route defaul
|
|||
...EFFORT_CHOICES.map((effort) => ({ value: effort, label: effort })),
|
||||
];
|
||||
return selectHtml(
|
||||
`${attrs} title="Reasoning effort — default: ${escapeHtml(surfaceDefault)}"`,
|
||||
`${attrs} title="Preferred reasoning effort — default: ${escapeHtml(surfaceDefault)}"`,
|
||||
[{ label: '', options }],
|
||||
selected || '',
|
||||
);
|
||||
|
|
|
|||
|
|
@ -410,7 +410,7 @@ export function renderSettingsPage() {
|
|||
<section class="settings-panel" data-settings-panel="behavior">
|
||||
<div class="form-section">
|
||||
<h3>Reasoning Effort</h3>
|
||||
<div class="settings-section-copy">Controls how deeply the model thinks per task type. Higher effort = slower but more thorough.</div>
|
||||
<div class="settings-section-copy">Preferred reasoning effort per task type. Unsupported levels adapt to the route; native mappings, required minimums or provider defaults may apply. Requested, sent and reported effort are recorded in Logs.</div>
|
||||
<div class="settings-effort-grid">
|
||||
${EFFORT_FIELDS.map(([id, label, defaultValue]) => effortField({ id, label, defaultValue })).join('')}
|
||||
</div>
|
||||
|
|
|
|||
79
web/tests/effort_evidence.test.js
Normal file
79
web/tests/effort_evidence.test.js
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { summarizeLogEvent } from '../modules/log_events.js';
|
||||
|
||||
const paired = JSON.parse(readFileSync(new URL('../../tests/fixtures/claudexor_effort_resolution.json', import.meta.url)));
|
||||
const variants = JSON.parse(readFileSync(new URL('../../tests/fixtures/claudexor_effort_resolution_variants.json', import.meta.url)));
|
||||
|
||||
for (const type of ['llm_usage', 'llm_round', 'llm_round_finished']) {
|
||||
test(`${type} separates effort preference, host carrier and absent report`, () => {
|
||||
const summary = summarizeLogEvent({ type, effort: {
|
||||
requested: 'ultra', sent: { reasoning_effort: 'max' }, sent_state: 'explicit', reported: null,
|
||||
} });
|
||||
assert.match(summary.body, /requested ultra/);
|
||||
assert.match(summary.body, /host sent .*max/);
|
||||
assert.match(summary.body, /reported unknown/);
|
||||
assert.doesNotMatch(summary.body, /applied/);
|
||||
});
|
||||
}
|
||||
|
||||
test('Logs labels omission and a provider response echo with its source', () => {
|
||||
const summary = summarizeLogEvent({ type: 'llm_usage', effort: {
|
||||
requested: 'low', sent: {}, sent_state: 'omitted', reported: 'high', report_source: 'provider_applied_options',
|
||||
} });
|
||||
assert.match(summary.body, /host sent omitted/);
|
||||
assert.match(summary.body, /reported high \(provider_applied_options\)/);
|
||||
assert.equal(summarizeLogEvent({ type: 'llm_usage' }).body, '');
|
||||
});
|
||||
|
||||
test('paired resolution fixture discloses preparation and keeps provider observation unknown', () => {
|
||||
const { body } = summarizeLogEvent({ type: 'llm_usage', effort_resolution: paired });
|
||||
assert.match(body, /prepared reasoning\.effort=xhigh \(downward; account_catalog\)/);
|
||||
assert.match(body, /reported unknown/);
|
||||
assert.match(body, /host sent unknown/);
|
||||
assert.doesNotMatch(body, /reported xhigh/);
|
||||
});
|
||||
|
||||
test('provider observation uses its own reporter, not the resolution authority', () => {
|
||||
const { body } = summarizeLogEvent({ type: 'llm_usage', effort_resolution: {
|
||||
...paired, observed: 'high', observedSource: 'provider_response',
|
||||
} });
|
||||
assert.match(body, /reported high \(provider_response\)/);
|
||||
assert.doesNotMatch(body, /reported high \(account_catalog\)/);
|
||||
});
|
||||
|
||||
test('session settlement Logs use the final-attempt report and keep missing observation unknown', () => {
|
||||
const { body } = summarizeLogEvent({ type: 'delegate_run_settled', observed_attempt: { effort_resolution: paired } });
|
||||
assert.match(body, /Engine requested effort ultra/);
|
||||
assert.match(body, /prepared reasoning\.effort=xhigh/);
|
||||
assert.match(body, /reported unknown/);
|
||||
});
|
||||
|
||||
for (const [name, report] of Object.entries(variants)) {
|
||||
for (const event of [
|
||||
{ type: 'llm_usage', effort_resolution: report },
|
||||
{ type: 'llm_round', usage: { effort_resolution: report } },
|
||||
{ type: 'delegate_run_settled', observed_attempt: { effort_resolution: report } },
|
||||
]) {
|
||||
test(`${event.type} preserves ${name} as engine preparation, without a provider echo`, () => {
|
||||
const { body } = summarizeLogEvent(event);
|
||||
const prepared = report.submitted === null ? 'omitted' : `${report.parameter}=${report.submitted}`;
|
||||
assert.equal(body, `Engine requested effort ${report.requested} · host sent unknown`
|
||||
+ ` · prepared ${prepared} (${report.resolution}; ${report.source}) · reported unknown`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const report of [undefined, null]) {
|
||||
test(`Logs keep absent or decoder-rejected evidence unknown (${report})`, () => {
|
||||
const { body } = summarizeLogEvent({ type: 'llm_usage', effort_resolution: report, effort: {
|
||||
requested: 'ultra', sent: { 'options.reasoningEffort': 'ultra' }, sent_state: 'explicit',
|
||||
reported: null, report_source: null,
|
||||
} });
|
||||
assert.match(body, /reported unknown/);
|
||||
assert.doesNotMatch(body, /prepared|reported ultra/);
|
||||
assert.equal(summarizeLogEvent({ type: 'delegate_run_settled',
|
||||
observed_attempt: { effort_resolution: report } }).body, '');
|
||||
});
|
||||
}
|
||||
|
|
@ -1217,3 +1217,19 @@ test('an owner-disabled roster row leaves the picker but a saved reference to it
|
|||
assert.match(described, /turn the row back on/);
|
||||
assert.doesNotMatch(describeSubagentReference('deep', roster), /switched off/);
|
||||
});
|
||||
|
||||
|
||||
test('referenced captions follow row effort overrides while compound routes stay immutable', () => {
|
||||
const roster = [
|
||||
{ subagent_id: 'ordinary', effort: 'high', route: { kind: ROUTE_KIND_SESSION, target_id: 'codex=gpt-model' } },
|
||||
{ subagent_id: 'compound', effort: 'xhigh', route: { kind: ROUTE_KIND_SESSION, target_id: 'cursor=cursor-grok-xhigh' } },
|
||||
];
|
||||
assert.match(describeSubagentReference('ordinary', roster, { effort: 'low' }), /preferred effort low/);
|
||||
assert.doesNotMatch(describeSubagentReference('ordinary', roster, { effort: 'low' }), /effort high/);
|
||||
assert.match(describeSubagentReference('ordinary', roster), /preferred effort high/);
|
||||
const compound = describeSubagentReference('compound', roster, { effort: 'low' });
|
||||
assert.match(compound, /cursor-grok-xhigh/);
|
||||
assert.match(compound, /preferred effort xhigh/);
|
||||
assert.doesNotMatch(compound, /effort low/);
|
||||
assert.equal(roster[1].route.target_id, 'cursor=cursor-grok-xhigh');
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue