diff --git a/docs/architecture/01-high-level-architecture.md b/docs/architecture/01-high-level-architecture.md index a2930cbb5..9d2935da9 100644 --- a/docs/architecture/01-high-level-architecture.md +++ b/docs/architecture/01-high-level-architecture.md @@ -175,7 +175,7 @@ server.py (Starlette+uvicorn) ← HTTP + WebSocket on configurable host:port (de ├── openrouter_attribution.py ← Canonical OpenRouter application attribution, centralized so forks do not compete under the same external application identity (§7) ├── openai_chat_custom.py ← Pure direct-OpenAI Chat function→custom codec: compact schemas, exact catalog binding, tool-choice projection, prior-call replay, canonical response normalization; no Responses transcript or second stored history ├── openai_chat_dispatch.py ← Direct-OpenAI Chat policy leaf: custom+requested reasoning first, exact-dialect fallback with the same reasoning, then task-local explicit `none` only when the physical-attempt rail still permits it; owns the bounded schema-error continuation - ├── request_wire_contract.py ← Exact-route request-profile actions and 14-day success-only evidence store; task-local `none` never teaches future dispatch (§6 Context fitting, retry, and compaction; §10 invariant 13) + ├── request_wire_contract.py ← Exact-route request-profile actions and 14-day success-only evidence store; profiles keep each function tool's absent/false/true/malformed `strict` state apart, so older collapsed-strictness evidence matches none; task-local `none` never teaches future dispatch (§6 Context fitting, retry, and compaction; §10 invariant 13) ├── request_wire_resolution.py ← Deterministic request-profile composition with source-predicated effort bounds/transitions; contradictions fail open to the requested effort with `conflict=True` ├── effort_evidence.py ← Live/recovered effort evidence: validate requested, prepared and reported; missing observations stay unknown, native ranking stays with the engine (§6) ├── request_wire_receipts.py ← Factory-bound wire candidates + semantic-success receipts (exact serializer digests; tool-choice semantics cannot change) diff --git a/docs/architecture/06-agent-core.md b/docs/architecture/06-agent-core.md index c5e810c7d..93bcf60fe 100644 --- a/docs/architecture/06-agent-core.md +++ b/docs/architecture/06-agent-core.md @@ -152,7 +152,7 @@ The one path that trims tier-0 is the local-model overflow compactor (`llm_local Main also uses this materializer for an actor-authored working view. At the existing successful-response boundary, after retry/reprepare/fallback and before returned tools execute, `compact_context.record_context_view` retains the canonical source and active schemas; the acceptance observer still sees the separate send projection. `compact_context` can retain exact inspected unit IDs, restore checkpoint sources, or pair an authored note with an explicit `keep_last_n` count. IDs take precedence; note-only keeps raw units, and count-only uses the helper path. Original source bytes remain retrievable, newer owner/tool messages stay untouched, and no helper model rewrites the authored note. The completed tool boundary measures the full proposed messages, schemas and receipt through Main's existing fit arithmetic; estimates are disclosed, not new capacity or economy-target admission gates. Actual overflow retains its existing recovery. Publication updates the view and schemas together; no-op observation leaves cache identity and the live turn unchanged. -Ouroboros stores one provider-neutral, function-shaped conversation. OpenRouter omits the neutral `tool_choice="auto"` while building the physical request, before request-wire identity is bound, so an implicit API default cannot become an unsupported parameter under strict routing. Direct OpenAI agent/tool traffic stays on Chat Completions: `openai_chat_custom.py` translates only the physical copy into Chat custom tools and normalizes returned custom calls back into canonical function calls; the dialect ladder is fixed at custom/function/none (ordinals 1/2/3), only an exact dialect rejection advances a rung, the custom→function control-flow fallback is never learned as a durable dialect action, and the compatibility layer never migrates the conversation to Responses or changes the model/provider/API surface. Direct Anthropic tool turns keep a private route-bound receipt with the complete native assistant `content`, replayed byte-for-byte on the immediately matching same-route continuation and scrubbed on any provider, endpoint, API-surface or model change; opaque values never reach summarizer or public projections. Owner-requested `none` is sent as `thinking.type=disabled`; no guessed legacy `budget_tokens` mapping is invented. +Ouroboros stores one provider-neutral, function-shaped conversation. Before request-wire binding, OpenRouter's physical request omits the neutral `tool_choice="auto"` (so strict routing never rejects an implicit default) and sets `strict: false` on function tools lacking it: the Responses API serving OpenAI models there otherwise attempts strict mode, requiring every property (https://developers.openai.com/api/docs/guides/migrate-to-responses §5, 2026-09-29). Direct OpenAI tool traffic stays on Chat Completions; `openai_chat_custom.py` maps only the physical copy to Chat custom tools and returned custom calls to canonical function calls. The fixed custom/function/none ladder (ordinals 1/2/3) advances only on an exact dialect rejection, never learns the custom→function fallback durably and never moves the conversation to Responses or another model/provider/API surface. Direct Anthropic tool turns keep a private route-bound receipt of the full native assistant `content`, replayed byte-for-byte on the next matching same-route continuation, scrubbed on any provider/endpoint/API-surface/model change, hidden from summarizer/public projections. Owner-requested `none` becomes `thinking.type=disabled`, never a guessed legacy `budget_tokens`. `request_wire_recovery.py` adapts exception/body errors for one credential-free exact route/profile with `set_value`, `drop_field` or registered `replace_dialect`: at most eight actions, existing send/pricing/Stop/deadline limits. Requested effort follows native mapping, never catalog ceilings. Positive field-bound enums need no scalar echo: highest comparable lower tier, or known minimum when all are higher. Negative quotes prescribe nothing; value rejection cannot drop the carrier. Mandatory floors/unsupported-carrier repairs remain. Only valid normalized success bound to the exact settled physical capture teaches the 14-day `state/request_wire_compatibility.json`; prose cannot reroute. Returned responses disclose `usage.request_wire` via `request_wire_attempt.validate_wire_attempt_identity` despite failed settlement; unsettled attempts never teach. Ordered, bounded `request_wire_history` is disclosure; `state/usage_attempts.jsonl` owns money. Private custom-argument receipts stay out of history/public views. Physical `effort` separates original preference, sent fields/omission and sourced reports (absent = unknown). Settlement keeps candidate facts; late receipts replace observations. Provider usage supplies neither effort nor engine reports. Legacy `requested_effort`/`applied_effort` mean provider projection/sent candidate with source labels, never provider-execution proof. Facts reach usage/events/Logs without automatic effort chat, toast or incident. diff --git a/docs/architecture/08-git-branching-ci-and-build.md b/docs/architecture/08-git-branching-ci-and-build.md index 0f2568f9d..5c66538e7 100644 --- a/docs/architecture/08-git-branching-ci-and-build.md +++ b/docs/architecture/08-git-branching-ci-and-build.md @@ -26,7 +26,7 @@ Quick and full jobs each run a dedicated blocking `size_ratchet` pytest step — Shared UI and keyless wait/repair browser fixtures run the dirty Git candidate in an independent checkout (`tests/candidate_checkout.py`): source and copy identities are verified, and each boot and restart binds the serving PID to unique static sentinel bytes and a VERSION suffix reported by `/api/health`, proving both served assets and imported Python came from this copy (inputs and safe recipe: DEVELOPMENT "Safe local verification and dirty browser candidates"). `ouroboros/test_environment.py` gives pytest, preflight and server children scrubbed temporary roots before imports; dependency sync installs nothing under the test marker. Both `direct_server_with_data` and `KeylessIsolatedServer` with a `CandidateCheckout` use one `ProcessContainer` per incarnation. Teardown reaps and closes it; restart receives a fresh container, and unconfirmed cleanup fails and retains the checkout. Plain-path keyless E2E callers keep their mutable HEAD clones. A POSIX pytest process killed before `finally` cannot perform cleanup. -Tool-schema compatibility has two layers. Fork-safe PR tests build the complete shipped catalog without MCP or extensions, validate every schema as general JSON Schema plus the cross-provider subset (no empty enum, no root `anyOf`/`oneOf`/`allOf`), and require the OpenRouter/function, direct-Anthropic, GigaChat and direct-OpenAI projections to preserve the complete tool-name set. The trusted `integration-test` lane sends that same full registry in one bounded `delegate_start` canary per physical route without executing the returned call — the OpenRouter families, the three shipped direct-OpenAI defaults, direct Anthropic, and optional MiniMax/DeepSeek/Cloud.ru/GigaChat; direct-OpenAI Main and DeepSeek also keep a second-turn continuation (DeepSeek because its tool contract is the `reasoning_content` replay). Each canary requires positive usage, exact provider/model identity and a normalized schema-valid tool call; quota/billing, 429, 5xx and timeout outcomes stay typed inconclusive while contract/auth/model/tool/reasoning 4xx are red, and one same-route resend is allowed only for a runtime-classified semantic-empty response. `response_finish_reason` is retained in host usage for bounded diagnostics only. +Tool-schema compatibility has two layers. Fork-safe PR tests build the complete shipped catalog without MCP or extensions, validate every schema as general JSON Schema plus the cross-provider subset (no empty enum, no root `anyOf`/`oneOf`/`allOf`), and require the OpenRouter/function, direct-Anthropic, GigaChat and direct-OpenAI projections to preserve the complete tool-name set. The trusted `integration-test` lane sends that same full registry in one bounded `delegate_start` canary per physical route without executing the returned call — the OpenRouter families, the three shipped direct-OpenAI defaults, direct Anthropic, and optional MiniMax/DeepSeek/Cloud.ru/GigaChat; direct-OpenAI Main and DeepSeek also keep a second-turn continuation (DeepSeek because its tool contract is the `reasoning_content` replay). Each canary requires positive usage, exact provider/model identity and a normalized schema-valid call carrying exactly the requested arguments; quota/billing, 429, 5xx and timeout outcomes stay typed inconclusive while contract/auth/model/tool/reasoning 4xx are red, and one same-route resend is allowed only for a runtime-classified semantic-empty response. `response_finish_reason` is retained in host usage for bounded diagnostics only. `claudexor-platform-gate.yml` proves the managed Claudexor runtime on three OSes: fixture (fake harness, offline, $0) and toolchain (`scripts/claudexor_toolchain_smoke.py`: production `ensure_cli_command()` installs the pinned official Node/npm with no ambient Node/npm or harness override; no harness install is claimed) lanes always, a Windows consumer lane (same script, `--harness-install codex`: the production `install_missing_harness_cli` really installs the engine-pinned Codex, whose native image must resolve directly, by bare name and in doctor; no login or task; red until the pin carries Windows native install), and a live lane only on explicit API keys — subscription auth stays deliberately out of CI, because an interactive machine-bound token must not enter CI secrets. `dependency-graph.yml` reads `ouroboros/claudexor_runtime_pin.json` and submits that direct runtime relationship to GitHub's dependency graph (runs only on pin/workflow changes on `main`/`ouroboros`, plus manual dispatch; `contents: write` only) without presenting Claudexor as a Python or Node package dependency. The Scorecard workflow (`.github/workflows/scorecard.yml`) runs on `main` pushes and weekly, pins every action by full commit SHA, defaults permissions to read-only, and adds only `security-events: write` + `id-token: write` for SARIF upload and OpenSSF publication. `CODE_OF_CONDUCT.md` owns community rules; `CITATION.cff` owns the software and preferred technical-report citations; `site/paper/index.html` owns the canonical paper landing page; `docs/benchmarks/evidence.json` holds the release-bound public benchmark projection; README remains the claim SSOT (guarded by `tests/test_trust_metadata.py` and `tests/test_public_site_metadata.py`). diff --git a/docs/development/02-naming-and-boundaries.md b/docs/development/02-naming-and-boundaries.md index e1ab17ee0..bbf18f6e3 100644 --- a/docs/development/02-naming-and-boundaries.md +++ b/docs/development/02-naming-and-boundaries.md @@ -113,21 +113,22 @@ its before/after byte size in the commit or PR. Recoverable tool failures are evidence for the next LLM turn, not triggers for a host-authored recovery workflow: return a typed, redacted result naming the -failed stage, the completed external effects and an actionable repair hint, and +failed stage, completed external effects and an actionable repair hint, and let the LLM decide. Host code owns deterministic integrity, authority boundaries and truthful receipts only — no task-specific auto-retry, fallback, cleanup, resume or terminal-flow state machines. Explicitly naming a documented default is never a different request: an argument whose value is what omitting it already means — `directory_strategy="direct"` with no `scope_paths` on a shape that cannot -serve the argument, or `workspace_root` naming the Ouroboros repository itself — +serve the argument, or `workspace_root` naming the Ouroboros repository — takes the omitted path, disclosed in the result; only a value that genuinely asks -for something is refused there, typed, at the earliest layer holding the authority -to judge it, with the repair named. Models fill every key of a tool schema, so an +for something is refused there, typed, at the earliest layer with authority to +judge it, repair named. Models may fill optional keys (OpenAI's Responses API +tries strict mode if unset; the OpenRouter lane sends `strict:false`), so an optional argument's empty or mode-irrelevant form (`max_wait_minutes` on a quiz that does not wait, `0` with `""`, a range end past a knowledge note, a zone beside an offset-carrying instant) is that same omitted path, and a refusal that only restates its rule is retried unchanged: it names the field, the value received and the repair in ONE -reply (house helpers for both sentences: `tools/arg_feedback`). +reply (house helpers for both: `tools/arg_feedback`). A producer that knows its call failed publishes that fact typed (`tool_result._publish_tool_result`, or a first-line `⚠️ IDENTIFIER` the legacy diff --git a/docs/inventories/DATA_LAYOUT_INVENTORY.md b/docs/inventories/DATA_LAYOUT_INVENTORY.md index 27d413cbe..45808885e 100644 --- a/docs/inventories/DATA_LAYOUT_INVENTORY.md +++ b/docs/inventories/DATA_LAYOUT_INVENTORY.md @@ -2,7 +2,7 @@ Machine extraction of the `docs/ARCHITECTURE.md` "Data layout (`~/Ouroboros/`)" tree — the durable-file orientation carrier (this tree's counterpart of the reference PERSISTENCE_OWNERS derivation checklist) — regenerated by `python scripts/regenerate_inventories.py`. Do not edit. Every entry is probed against reality: repo entries must exist as tracked paths; data-plane entries must appear as a literal in the runtime sources that construct them. A durable file renamed or removed in code while its tree row survives = red (`tests/test_generated_inventories.py`). -Source: `docs/architecture/01-high-level-architecture.md`, physical LF lines 618-710; UTF-8 SHA-256 `db2ce77fc5dba70b4680f224de0732748955b0a80ee03e9e230793263ea252f4`. +Source: `docs/architecture/01-high-level-architecture.md`, physical LF lines 618-710; UTF-8 SHA-256 `b5d06de2164ac32a6dff26bdf9e0bd7f0b2bd7a5e229e20648a7b4ef784ca344`. - entries: **82** (code-ref: 75, repo-dir: 6, repo-path: 1) diff --git a/ouroboros/llm_openai_compatible.py b/ouroboros/llm_openai_compatible.py index d08a7bc70..cc6dc184a 100644 --- a/ouroboros/llm_openai_compatible.py +++ b/ouroboros/llm_openai_compatible.py @@ -365,6 +365,10 @@ class _OpenAICompatibleLaneMixin: {k: v for k, v in tool.items() if k != "cache_control"} for tool in self._sanitize_chat_completion_tools(tools) ] + for tool in prepared_tools: # each "function" is the sanitizer's own copy + # The Responses API serving OpenAI models here tries strict mode for a + # tool WITHOUT `strict`, requiring every property; a set value stays. + tool["function"].setdefault("strict", False) if server_web_tool: prepared_tools.append(server_web_tool) # Tool cache markers are placed once, at the send-time payload finalizer diff --git a/ouroboros/request_wire_contract.py b/ouroboros/request_wire_contract.py index b401c981e..0683e345a 100644 --- a/ouroboros/request_wire_contract.py +++ b/ouroboros/request_wire_contract.py @@ -10,6 +10,7 @@ free-form provider prose. from __future__ import annotations import hashlib +import itertools import json import logging import pathlib @@ -61,7 +62,15 @@ TOOL_DIALECTS = frozenset({ TOOL_CHOICE_FAMILIES = frozenset({ "omitted", "auto", "none", "required", "any", "allowed_tools", "named", "other", }) -FUNCTION_STRICTNESS_VALUES = frozenset({"none", "plain", "strict", "mixed"}) +# Per-tool `strict` states in canonical order; a profile names the set its function tools +# carry, or "none". Historical schema-1 "plain"/"strict"/"mixed" (absent, false and +# malformed collapsed) stay legal to decode, but no payload produces them, so exact +# profile equality keeps their stored evidence from authorizing any current profile. +FUNCTION_STRICT_STATES = ("absent", "false", "true", "malformed") +LEGACY_FUNCTION_STRICTNESS_VALUES = frozenset({"plain", "strict", "mixed"}) +FUNCTION_STRICTNESS_VALUES = LEGACY_FUNCTION_STRICTNESS_VALUES | {"none", *( + "+".join(combo) for size in range(1, len(FUNCTION_STRICT_STATES) + 1) + for combo in itertools.combinations(FUNCTION_STRICT_STATES, size))} _REGISTERED_DIALECT_TRANSITIONS = frozenset({ ("tool", "function", "openai_chat_custom"), ("tool", "openai_chat_custom", "function"), @@ -298,18 +307,10 @@ def tool_choice_names(choice: Any) -> Tuple[str, ...]: def tool_choice_requires_call(choice: Any) -> bool: - family = tool_choice_family(choice) - if family in {"required", "any", "named"}: - return True - if family != "allowed_tools" or not isinstance(choice, Mapping): - return False - allowed = choice.get("allowed_tools") - mode = str( - (allowed.get("mode") if isinstance(allowed, Mapping) else None) - or choice.get("mode") - or "auto" - ).strip().lower() - return mode == "required" + return ( + tool_choice_family(choice) in {"required", "any", "named"} + or tool_choice_allowed_mode(choice) == "required" + ) def tool_choice_allowed_mode(choice: Any) -> str: @@ -379,18 +380,16 @@ def _tool_choice_digest(choice: Any) -> str: def function_strictness(payload: Mapping[str, Any]) -> str: - strict_values = [] + """Name the exact `strict` states of the function tools: a Responses-backed route + tries strict mode for an ABSENT key, so absent is not an explicit ``false``.""" + states = set() for tool in payload.get("tools") or []: - if not isinstance(tool, Mapping): - continue - function = tool.get("function") + function = tool.get("function") if isinstance(tool, Mapping) else None if isinstance(function, Mapping): - strict_values.append(bool(function.get("strict"))) - if not strict_values: - return "none" - if all(strict_values): - return "strict" - return "mixed" if any(strict_values) else "plain" + value = function.get("strict") + states.add("absent" if "strict" not in function else "true" if value is True + else "false" if value is False else "malformed") + return "+".join(state for state in FUNCTION_STRICT_STATES if state in states) or "none" def _routing_digest(payload: Mapping[str, Any]) -> str: diff --git a/ouroboros/tools/arg_feedback.py b/ouroboros/tools/arg_feedback.py index 9cf4ad313..baa25932e 100644 --- a/ouroboros/tools/arg_feedback.py +++ b/ouroboros/tools/arg_feedback.py @@ -1,9 +1,11 @@ """What a tool says about an argument it did not obey (DEVELOPMENT "LLM-first affordances"). -Models fill every key of a tool schema. A value that asks for nothing takes the -omitted path and the result says so in one line; a value that asks for something -the call cannot serve is refused ONCE, typed, naming the field, the value received -and the repair — a refusal that only restates the rule is retried unchanged. +Models may fill optional keys (OpenAI's Responses API tries strict mode when a +function tool leaves `strict` unset, so the OpenRouter lane sends `strict: false`). A value +that asks for nothing takes the omitted path and the result says so in one line; a +value that asks for something the call cannot serve is refused ONCE, typed, naming +the field, the value received and the repair — a refusal that only restates the +rule is retried unchanged. """ from __future__ import annotations diff --git a/tests/fixtures/llm_golden/fallback_ladder.json b/tests/fixtures/llm_golden/fallback_ladder.json index 9c1aab6cb..a34b9efba 100644 --- a/tests/fixtures/llm_golden/fallback_ladder.json +++ b/tests/fixtures/llm_golden/fallback_ladder.json @@ -3042,9 +3042,9 @@ "requested_tool_dialect": "function", "applied_tool_dialect": "function", "reason_code": "requested_wire_form", - "source_profile_fingerprint": "b3665fdf2866b600747bc031a414a4bb341b2d4663904da97f5ffebcbeabe844", - "accepted_profile_fingerprint": "b3665fdf2866b600747bc031a414a4bb341b2d4663904da97f5ffebcbeabe844", - "candidate_sha256": "41e784c12495bc22d0d006fb393905cbb6d9b82e05563fee57a175e3698a50f8", + "source_profile_fingerprint": "1efc6fdc0d589dce36f8cc63ac75ecd15e154c1c2fe05407b0e37fbabc3efce5", + "accepted_profile_fingerprint": "1efc6fdc0d589dce36f8cc63ac75ecd15e154c1c2fe05407b0e37fbabc3efce5", + "candidate_sha256": "18aa195e409cc792cf5212e3f6d3ba66df62579ee1b17d88f7f74390f2363466", "ladder_ordinal": 1, "applied_actions": [], "task_local": false, @@ -3093,7 +3093,8 @@ "name": "apply_patch", "parameters": { "type": "object" - } + }, + "strict": false }, "type": "function" }, @@ -3108,13 +3109,14 @@ } }, "type": "object" - } + }, + "strict": false }, "type": "function" } ] }, - "payload_sha256": "41e784c12495bc22d0d006fb393905cbb6d9b82e05563fee57a175e3698a50f8" + "payload_sha256": "18aa195e409cc792cf5212e3f6d3ba66df62579ee1b17d88f7f74390f2363466" } ], "physical_attempts": [ @@ -3122,8 +3124,8 @@ "source": "llm.chat", "model": "x-ai/grok-4.5", "provider": "openrouter", - "candidate_raw_sha256": "41e784c12495bc22d0d006fb393905cbb6d9b82e05563fee57a175e3698a50f8", - "candidate_raw_size_bytes": 536, + "candidate_raw_sha256": "18aa195e409cc792cf5212e3f6d3ba66df62579ee1b17d88f7f74390f2363466", + "candidate_raw_size_bytes": 566, "candidate_measurement_kind": "canonical_json_v1", "states": [ "reserved", diff --git a/tests/fixtures/llm_golden/openrouter_payload.json b/tests/fixtures/llm_golden/openrouter_payload.json index 0b40b731d..5c3182e0a 100644 --- a/tests/fixtures/llm_golden/openrouter_payload.json +++ b/tests/fixtures/llm_golden/openrouter_payload.json @@ -158,7 +158,8 @@ "description": "two parts", "parameters": { "type": "object" - } + }, + "strict": false } }, { @@ -173,12 +174,13 @@ "type": "string" } } - } + }, + "strict": false } } ] }, - "value_sha256": "62f5abe098c360c7e3fde2a71518f98fdb88953d58f86f96f42edc5181b85660" + "value_sha256": "74b2caa648f8a1ebbeedb73cd6be0e4722cf2436ec3011c72cd52c72f5bbee74" }, "sends": [], "unused_script_steps": 0 @@ -284,7 +286,8 @@ "description": "two parts", "parameters": { "type": "object" - } + }, + "strict": false } }, { @@ -299,12 +302,13 @@ "type": "string" } } - } + }, + "strict": false } } ] }, - "value_sha256": "e4804aeb77658b2d38a8e81e603478dda0d5b1ca38d040998025a48803d03ee5" + "value_sha256": "47aeeafb94e9475dd0ab01125fae8c6832f544e6d7e63de287203fa1794efe15" }, "sends": [], "unused_script_steps": 0 @@ -948,7 +952,8 @@ "description": "two parts", "parameters": { "type": "object" - } + }, + "strict": false } }, { @@ -963,7 +968,8 @@ "type": "string" } } - } + }, + "strict": false } }, { @@ -975,7 +981,7 @@ } ] }, - "value_sha256": "8ce93e0e8ce4b3478c4980a4b9844f4b02429571f5a97837514a6656bea470f1" + "value_sha256": "6d0792b8ff843846e03575a2f1e14633e0748ea0ebd84a3ca09848fce1fcac2d" }, "sends": [], "unused_script_steps": 0 @@ -1615,9 +1621,9 @@ "requested_tool_dialect": "function", "applied_tool_dialect": "function", "reason_code": "requested_wire_form", - "source_profile_fingerprint": "760c6157eb99a45072ed50ff3f02dda60fc887c3534865f85fbad14fcca919d1", - "accepted_profile_fingerprint": "760c6157eb99a45072ed50ff3f02dda60fc887c3534865f85fbad14fcca919d1", - "candidate_sha256": "dfa3b4af80ed25e9ddc9fb9e71f7eb7b15b354522b071a86f2c8556d4644e644", + "source_profile_fingerprint": "5123c189414df91b972f8d20078a251c6b26563d52feb572364a70836d591302", + "accepted_profile_fingerprint": "5123c189414df91b972f8d20078a251c6b26563d52feb572364a70836d591302", + "candidate_sha256": "e430a01d4e8363513e703af1967143deed81b76fca8b96d197c471e98a745d51", "ladder_ordinal": 1, "applied_actions": [], "task_local": false, @@ -1678,7 +1684,8 @@ "name": "apply_patch", "parameters": { "type": "object" - } + }, + "strict": false }, "type": "function" }, @@ -1697,13 +1704,14 @@ } }, "type": "object" - } + }, + "strict": false }, "type": "function" } ] }, - "payload_sha256": "dfa3b4af80ed25e9ddc9fb9e71f7eb7b15b354522b071a86f2c8556d4644e644" + "payload_sha256": "e430a01d4e8363513e703af1967143deed81b76fca8b96d197c471e98a745d51" } ], "physical_attempts": [ @@ -1711,8 +1719,8 @@ "source": "llm.chat", "model": "anthropic/claude-sonnet-4.6", "provider": "openrouter", - "candidate_raw_sha256": "dfa3b4af80ed25e9ddc9fb9e71f7eb7b15b354522b071a86f2c8556d4644e644", - "candidate_raw_size_bytes": 707, + "candidate_raw_sha256": "e430a01d4e8363513e703af1967143deed81b76fca8b96d197c471e98a745d51", + "candidate_raw_size_bytes": 737, "candidate_measurement_kind": "canonical_json_v1", "states": [ "reserved", @@ -2387,9 +2395,9 @@ "requested_tool_dialect": "function", "applied_tool_dialect": "function", "reason_code": "requested_wire_form", - "source_profile_fingerprint": "b3665fdf2866b600747bc031a414a4bb341b2d4663904da97f5ffebcbeabe844", - "accepted_profile_fingerprint": "b3665fdf2866b600747bc031a414a4bb341b2d4663904da97f5ffebcbeabe844", - "candidate_sha256": "41e784c12495bc22d0d006fb393905cbb6d9b82e05563fee57a175e3698a50f8", + "source_profile_fingerprint": "1efc6fdc0d589dce36f8cc63ac75ecd15e154c1c2fe05407b0e37fbabc3efce5", + "accepted_profile_fingerprint": "1efc6fdc0d589dce36f8cc63ac75ecd15e154c1c2fe05407b0e37fbabc3efce5", + "candidate_sha256": "18aa195e409cc792cf5212e3f6d3ba66df62579ee1b17d88f7f74390f2363466", "ladder_ordinal": 1, "applied_actions": [], "task_local": false, @@ -2438,7 +2446,8 @@ "name": "apply_patch", "parameters": { "type": "object" - } + }, + "strict": false }, "type": "function" }, @@ -2453,13 +2462,14 @@ } }, "type": "object" - } + }, + "strict": false }, "type": "function" } ] }, - "payload_sha256": "41e784c12495bc22d0d006fb393905cbb6d9b82e05563fee57a175e3698a50f8" + "payload_sha256": "18aa195e409cc792cf5212e3f6d3ba66df62579ee1b17d88f7f74390f2363466" } ], "physical_attempts": [ @@ -2467,8 +2477,8 @@ "source": "llm.chat", "model": "x-ai/grok-4.5", "provider": "openrouter", - "candidate_raw_sha256": "41e784c12495bc22d0d006fb393905cbb6d9b82e05563fee57a175e3698a50f8", - "candidate_raw_size_bytes": 536, + "candidate_raw_sha256": "18aa195e409cc792cf5212e3f6d3ba66df62579ee1b17d88f7f74390f2363466", + "candidate_raw_size_bytes": 566, "candidate_measurement_kind": "canonical_json_v1", "states": [ "reserved", @@ -2974,9 +2984,9 @@ "requested_tool_dialect": "function", "applied_tool_dialect": "function", "reason_code": "requested_wire_form", - "source_profile_fingerprint": "760c6157eb99a45072ed50ff3f02dda60fc887c3534865f85fbad14fcca919d1", - "accepted_profile_fingerprint": "760c6157eb99a45072ed50ff3f02dda60fc887c3534865f85fbad14fcca919d1", - "candidate_sha256": "9451eeb0c00429e0c3b5e88d3ec0795aa56ed2818e36b1f238ff30669cd22b22", + "source_profile_fingerprint": "5123c189414df91b972f8d20078a251c6b26563d52feb572364a70836d591302", + "accepted_profile_fingerprint": "5123c189414df91b972f8d20078a251c6b26563d52feb572364a70836d591302", + "candidate_sha256": "a37d8eb2c30f67ed9d5185ce8da8bd78785e76aa72832ad4f71416d0aabc7743", "ladder_ordinal": 1, "applied_actions": [], "task_local": false, @@ -3081,13 +3091,14 @@ } }, "type": "object" - } + }, + "strict": false }, "type": "function" } ] }, - "payload_sha256": "9451eeb0c00429e0c3b5e88d3ec0795aa56ed2818e36b1f238ff30669cd22b22" + "payload_sha256": "a37d8eb2c30f67ed9d5185ce8da8bd78785e76aa72832ad4f71416d0aabc7743" } ], "physical_attempts": [ @@ -3095,8 +3106,8 @@ "source": "llm.chat", "model": "anthropic/claude-sonnet-4.6", "provider": "openrouter", - "candidate_raw_sha256": "9451eeb0c00429e0c3b5e88d3ec0795aa56ed2818e36b1f238ff30669cd22b22", - "candidate_raw_size_bytes": 870, + "candidate_raw_sha256": "a37d8eb2c30f67ed9d5185ce8da8bd78785e76aa72832ad4f71416d0aabc7743", + "candidate_raw_size_bytes": 885, "candidate_measurement_kind": "canonical_json_v1", "states": [ "reserved", @@ -3399,9 +3410,9 @@ "requested_tool_dialect": "function", "applied_tool_dialect": "function", "reason_code": "requested_wire_form", - "source_profile_fingerprint": "760c6157eb99a45072ed50ff3f02dda60fc887c3534865f85fbad14fcca919d1", - "accepted_profile_fingerprint": "760c6157eb99a45072ed50ff3f02dda60fc887c3534865f85fbad14fcca919d1", - "candidate_sha256": "59c91042eda38434e8d30fa8df23731bb5d0d79cef37adef46d3c3b1178c7f0f", + "source_profile_fingerprint": "5123c189414df91b972f8d20078a251c6b26563d52feb572364a70836d591302", + "accepted_profile_fingerprint": "5123c189414df91b972f8d20078a251c6b26563d52feb572364a70836d591302", + "candidate_sha256": "466313bb0d49bcab44fc6dbe525dd551a4cb061e004e207a2ddbbf5115a9338a", "ladder_ordinal": 1, "applied_actions": [], "task_local": false, @@ -3470,13 +3481,14 @@ } }, "type": "object" - } + }, + "strict": false }, "type": "function" } ] }, - "payload_sha256": "59c91042eda38434e8d30fa8df23731bb5d0d79cef37adef46d3c3b1178c7f0f" + "payload_sha256": "466313bb0d49bcab44fc6dbe525dd551a4cb061e004e207a2ddbbf5115a9338a" } ], "physical_attempts": [ @@ -3484,8 +3496,8 @@ "source": "llm.chat", "model": "anthropic/claude-sonnet-4.6", "provider": "openrouter", - "candidate_raw_sha256": "59c91042eda38434e8d30fa8df23731bb5d0d79cef37adef46d3c3b1178c7f0f", - "candidate_raw_size_bytes": 571, + "candidate_raw_sha256": "466313bb0d49bcab44fc6dbe525dd551a4cb061e004e207a2ddbbf5115a9338a", + "candidate_raw_size_bytes": 586, "candidate_measurement_kind": "canonical_json_v1", "states": [ "reserved", diff --git a/tests/provider_contract_ci.py b/tests/provider_contract_ci.py index c1ba1a5b2..2f785911b 100644 --- a/tests/provider_contract_ci.py +++ b/tests/provider_contract_ci.py @@ -350,7 +350,8 @@ def _text_facts(value): return len(encoded), hashlib.sha256(encoded).hexdigest() -def _canary_evidence(canary_or_model, message, usage, *, call=None, call_index=None, parse_error=None): +def _canary_evidence(canary_or_model, message, usage, *, call=None, call_index=None, parse_error=None, + key_diff=None): """Return bounded structural evidence; never copy provider payloads.""" canary_id = canary_or_model.canary_id if isinstance(canary_or_model, ProviderCanary) else "" model = canary_or_model.model if isinstance(canary_or_model, ProviderCanary) else canary_or_model @@ -405,6 +406,10 @@ def _canary_evidence(canary_or_model, message, usage, *, call=None, call_index=N evidence["arguments_type"] = type(raw_arguments).__name__ else: evidence["function_type"] = type(function).__name__ + if key_diff is not None: + extra, missing = key_diff + evidence["extra_keys"] = [label for label in map(_bounded_diagnostic_label, extra) if label] + evidence["missing_keys"] = [label for label in map(_bounded_diagnostic_label, missing) if label] if parse_error is not None: evidence["parse_error"] = {"type": type(parse_error).__name__} for field in ("pos", "lineno", "colno"): @@ -619,11 +624,16 @@ def _chat_canary_turn(client, *, canary: ProviderCanary, chat_kwargs): def assert_normalized_canary_call( message, tools, - required_arguments, + requested_arguments, *, canary=None, usage=None, ): + """Every native call must be schema-valid and carry EXACTLY the requested object. + + An extra declared key is a RED contract fact, not tolerated noise: a route that + forces optional properties (issue #1411) fills keys the prompt never asked for. + """ from jsonschema import validators calls = message.get("tool_calls") if isinstance(message, dict) else None @@ -697,7 +707,16 @@ def assert_normalized_canary_call( canary or "", message, usage, "arguments_unknown_keys", call=call, call_index=call_index, ) - for key, expected in required_arguments.items(): + if set(arguments) != set(requested_arguments): + # Only declared property names reach here, so naming them is safe evidence. + raise _canary_failure( + canary or "", message, usage, "arguments_exact_keys", + call=call, call_index=call_index, key_diff=( + sorted(set(arguments) - set(requested_arguments)), + sorted(set(requested_arguments) - set(arguments)), + ), + ) + for key, expected in requested_arguments.items(): if arguments.get(key) != expected: raise _canary_failure( canary or "", message, usage, f"arguments_{key}", @@ -715,7 +734,6 @@ def run_provider_contract_canary( ): """Exercise the public chat seam without executing the returned tool call.""" requested_arguments = delegate_start_canary_arguments(nonce) - required_arguments = {"prompt": requested_arguments["prompt"]} arguments_json = json.dumps(requested_arguments, ensure_ascii=False, sort_keys=True) final_marker = f"FULL_REGISTRY_CONTINUED_{nonce}" continuation_instruction = ( @@ -755,15 +773,14 @@ def run_provider_contract_canary( "timeout": CANARY_TIMEOUT_SEC, }, ) - # This is a provider schema-admission canary, not a duplicate of the - # runtime selector gate. The provider must preserve the nonce-bearing prompt - # and return declared, schema-valid native calls. Valid assistant text beside - # those calls is tolerated and recorded as bounded warning telemetry; - # subagent_id versus retry_of is enforced by the existing typed runtime tests. + # The provider must return schema-valid native calls whose arguments are + # exactly the requested object: the nonce-bearing prompt and the subagent_id + # selector, no omitted, substituted or provider-filled optional key. Valid + # assistant text beside those calls is tolerated as bounded warning telemetry. calls = assert_normalized_canary_call( message, tools, - required_arguments, + requested_arguments, canary=canary, usage=usage, ) diff --git a/tests/test_arg_feedback.py b/tests/test_arg_feedback.py index dfc2e32d8..2253d98f9 100644 --- a/tests/test_arg_feedback.py +++ b/tests/test_arg_feedback.py @@ -1,6 +1,6 @@ """A filled optional argument that asks for nothing is not a refusal. -Models fill every key of a tool schema. These tests pin, through the REAL registry, +Models may fill every key of a tool schema. These tests pin, through the REAL registry, that such a value takes the omitted path with one disclosure line, and that a genuine argument mistake is refused once, typed, naming the value it received. """ diff --git a/tests/test_llm_supported_parameters.py b/tests/test_llm_supported_parameters.py index f9bcf6616..8e1e47778 100644 --- a/tests/test_llm_supported_parameters.py +++ b/tests/test_llm_supported_parameters.py @@ -128,7 +128,9 @@ class TestSupportedParametersFilter: ) assert len(sent) == 1 assert sent[0]["model"] == model - assert sent[0]["tools"][0]["function"] == tool["function"] + # The OpenRouter wire marks the absent `strict` explicitly false (issue #1411). + assert sent[0]["tools"][0]["function"] == {**tool["function"], "strict": False} + assert "strict" not in tool["function"] assert sent[0]["extra_body"]["provider"] == { "require_parameters": True, "allow_fallbacks": True, } @@ -318,7 +320,7 @@ class TestSupportedParametersFilter: allow_server_web_search=False, skip_capability_fetch=True, ) - assert kwargs["tools"] == [tool] + assert kwargs["tools"] == [{**tool, "function": {**tool["function"], "strict": False}}] def test_chat_path_forwards_main_openrouter_web_search_flag(self, monkeypatch): from types import SimpleNamespace diff --git a/tests/test_openrouter_tool_strictness.py b/tests/test_openrouter_tool_strictness.py new file mode 100644 index 000000000..28a445f5a --- /dev/null +++ b/tests/test_openrouter_tool_strictness.py @@ -0,0 +1,212 @@ +"""OpenRouter function tools carry an explicit ``strict: false`` (issue #1411). + +OpenAI models behind OpenRouter are served by the Responses API, which normalizes a +function tool WITHOUT ``strict`` into strict mode and forces every optional property. +The OpenRouter lane sets ``false`` only where the key is absent, on its own copy of the +catalog, before request-wire identity is bound. Caller values, the server web-search +tool, tool-free requests, every other route and the explicit arguments a model returns +stay exactly as they were. +""" + +from __future__ import annotations + +import copy +import json + +import pytest + +from ouroboros.llm import LLMClient +from ouroboros.request_wire_contract import build_request_wire_profile, function_strictness +from tests.provider_contract_catalog import shipped_builtin_tool_schemas +from tests.test_llm_provider_golden import _observe + +_OPENAI_ON_OPENROUTER = "openai/gpt-5.6-luna" +_SERVER_WEB_TOOL = {"type": "openrouter:web_search", "parameters": {"max_total_results": 3}} + + +def _tool(name, **function_fields): + return {"type": "function", "function": { + "name": name, + "description": f"{name} fixture", + "parameters": { + "type": "object", + "properties": { + "task_id": {"type": "string"}, + "note": {"type": "string"}, + "verbose": {"type": "boolean", "default": False}, + "scope": {"type": "string", "enum": ["own_binding"]}, + }, + "required": ["task_id"], + }, + **function_fields, + }} + + +def _build(client, model, tools, **options): + target = client._resolve_remote_target(model) + return target, client._build_remote_kwargs( + target, [{"role": "user", "content": "call once"}], "medium", 256, "auto", None, + tools, skip_capability_fetch=True, **options, + ) + + +def _functions(payload): + return { + tool["function"]["name"]: tool["function"] + for tool in payload.get("tools") or [] + if isinstance(tool.get("function"), dict) + } + + +def test_openrouter_sets_false_only_where_strict_is_absent(monkeypatch): + monkeypatch.setattr( + LLMClient, "_openrouter_main_web_search_tool", staticmethod(lambda: dict(_SERVER_WEB_TOOL)), + ) + tools = [ + {**_tool("read_task"), "cache_control": {"type": "ephemeral"}}, + _tool("explicit_true", strict=True), + _tool("explicit_false", strict=False), + _tool("explicit_null", strict=None), + ] + before = copy.deepcopy(tools) + _target, payload = _build( + LLMClient(api_key="test"), _OPENAI_ON_OPENROUTER, tools, allow_server_web_search=True, + ) + + assert tools == before, "the caller's catalog must never be mutated" + functions = _functions(payload) + assert functions["read_task"]["strict"] is False + assert functions["explicit_true"]["strict"] is True + assert functions["explicit_false"]["strict"] is False + assert "strict" in functions["explicit_null"] and functions["explicit_null"]["strict"] is None + for name, function in functions.items(): + source = next(tool["function"] for tool in before if tool["function"]["name"] == name) + assert function["parameters"] == source["parameters"], name + assert function is not source + assert payload["tools"][-1] == _SERVER_WEB_TOOL + assert function_strictness({"tools": before}) == "absent+false+true+malformed" + assert function_strictness(payload) == "false+true+malformed" + + +@pytest.mark.parametrize("model", [ + "openai::gpt-5.6-terra", + "deepseek::deepseek-v4-flash", + "zai::glm-5", + "minimax::MiniMax-M3", + "cloudru::zai-org/GLM-4.7", + "openai-compatible::local-model", +]) +def test_other_routes_keep_their_function_tool_wire(model): + tools = [_tool("read_task")] + _target, payload = _build(LLMClient(api_key="test"), model, copy.deepcopy(tools)) + assert "strict" not in _functions(payload)["read_task"] + assert function_strictness(payload) == "absent" + + +def test_tool_free_openrouter_request_is_unchanged(): + _target, payload = _build(LLMClient(api_key="test"), _OPENAI_ON_OPENROUTER, None) + assert "tools" not in payload + assert function_strictness(payload) == "none" + + +def test_full_registry_keeps_every_schema_and_explicit_argument_semantics(): + """Negative controls: only the wire flag changes, never a declared default, enum, + required list or an explicit value the model returns (no neutral-value stripping).""" + registry = shipped_builtin_tool_schemas() + client = LLMClient(api_key="test") + target, payload = _build(client, _OPENAI_ON_OPENROUTER, copy.deepcopy(registry)) + functions = _functions(payload) + assert set(functions) == {tool["function"]["name"] for tool in registry} + for tool in registry: + sent = dict(functions[tool["function"]["name"]]) + assert sent.pop("strict") is False + assert sent == client._sanitize_chat_completion_tools([tool])[0]["function"] + assert functions["schedule_subagent"]["parameters"]["properties"]["may_fan_out"]["default"] is True + assert "none" in functions["switch_model"]["parameters"]["properties"]["effort"]["enum"] + + explicit = { + "schedule_subagent": {"may_fan_out": False, "allowed_origins": []}, + "compact_context": {"keep_unit_ids": []}, + "switch_model": {"effort": "none"}, + "skill_review": {"author_disposition": "accepted", "author_rationale": "reviewed"}, + } + calls = [ + {"id": f"call-{name}", "type": "function", + "function": {"name": name, "arguments": json.dumps(arguments, sort_keys=True)}} + for name, arguments in explicit.items() + ] + message, _usage = client._normalize_remote_response({ + "id": "gen-explicit", + "choices": [{"finish_reason": "tool_calls", "message": { + "role": "assistant", "content": None, "tool_calls": copy.deepcopy(calls), + }}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2}, + }, target, skip_cost_fetch=True) + returned = {call["function"]["name"]: call["function"]["arguments"] for call in message["tool_calls"]} + assert returned == {call["function"]["name"]: call["function"]["arguments"] for call in calls} + + +def _response_step(): + return {"kind": "response", "body": { + "id": "gen-strictness", + "choices": [{"index": 0, "finish_reason": "stop", "message": { + "role": "assistant", "content": "ok", "refusal": None, "annotations": None, + }}], + "usage": {"prompt_tokens": 40, "completion_tokens": 4, "cost": 0.001}, + }} + + +def _spec(method, transport, **kwargs): + return { + "env": {"OPENROUTER_API_KEY": "or-fixture-key"}, + "transport": transport, + "call": {"kind": "method", "name": method, "kwargs": { + "model": _OPENAI_ON_OPENROUTER, + "messages": [{"role": "user", "content": "call once"}], + "tools": [_tool("read_task"), _tool("explicit_true", strict=True)], + **kwargs, + }}, + } + + +def _assert_sealed_false_sends(observed, expected_sends): + assert "raised" not in observed, observed.get("raised") + sends, attempts = observed["sends"], observed["physical_attempts"] + assert len(sends) == len(attempts) == expected_sends + target = LLMClient(api_key="or-fixture-key")._resolve_remote_target(_OPENAI_ON_OPENROUTER) + for send, attempt in zip(sends, attempts): + functions = _functions(send["payload"]) + assert functions["read_task"]["strict"] is False + assert functions["explicit_true"]["strict"] is True + assert functions["read_task"]["parameters"] == _tool("read_task")["function"]["parameters"] + # The bytes handed to the transport are the sealed physical candidate. + assert send["payload_sha256"] == attempt["candidate_raw_sha256"] + wire = observed["returned"]["usage"]["request_wire"] + assert wire["candidate_sha256"] == sends[-1]["payload_sha256"] + profile = build_request_wire_profile( + target, sends[-1]["payload"], api_surface="chat.completions", + ) + assert profile.function_strictness == "false+true" + assert wire["source_profile_fingerprint"] == wire["accepted_profile_fingerprint"] == profile.fingerprint + return wire + + +@pytest.mark.parametrize("method", ["chat", "chat_async"]) +@pytest.mark.parametrize("no_proxy", [False, True]) +def test_physical_send_seals_explicit_false(method, no_proxy): + observed = _observe(_spec(method, [_response_step()], no_proxy=no_proxy)) + wire = _assert_sealed_false_sends(observed, 1) + assert wire["ladder_ordinal"] == 1 and wire["applied_actions"] == [] + + +@pytest.mark.parametrize("method", ["chat", "chat_async"]) +def test_wire_recovery_retry_keeps_explicit_false_sealed(method): + rejection = { + "kind": "error", "status_code": 400, + "message": "temperature: unsupported parameter for this endpoint", + } + observed = _observe(_spec(method, [rejection, _response_step()], temperature=0.7)) + wire = _assert_sealed_false_sends(observed, 2) + first, retried = (send["payload"] for send in observed["sends"]) + assert "temperature" in first and "temperature" not in retried + assert [item["action"]["fields"] for item in wire["applied_actions"]] == [["temperature"]] diff --git a/tests/test_provider_contract_ci.py b/tests/test_provider_contract_ci.py index c4410ce64..abd9f5f61 100644 --- a/tests/test_provider_contract_ci.py +++ b/tests/test_provider_contract_ci.py @@ -702,16 +702,28 @@ def test_reasoning_integrity_canary_rejects_explicit_none_and_clamp(): assert_openai_canary_usage(neutral_repair, model) -def test_normalized_calls_require_unique_schema_valid_delegate_start(): +def _normalized_call_violation(arguments, expected, *, calls=None): + with pytest.raises(AssertionError) as caught: + assert_normalized_canary_call( + {"tool_calls": calls or [_canonical_canary_call("call-violation", arguments)]}, + full_registry_canary_tools(), + expected, + ) + return caught.value.args[0]["provider_contract_violation"] + + +def test_normalized_calls_require_exactly_the_requested_delegate_start(): tools = full_registry_canary_tools() expected = delegate_start_canary_arguments("normalized-call") - required = {"prompt": expected["prompt"]} + assert set(expected) == {"prompt", "subagent_id"} assert assert_normalized_canary_call( {"tool_calls": [_canonical_canary_call("call-ok", expected)]}, tools, - required, + expected, )[0]["id"] == "call-ok" + # Schema-valid neutral values the prompt never asked for are what a route that + # forces optional properties emits (issue #1411): RED, naming declared keys only. provider_defaults = { **expected, "root": "skill_payload", @@ -720,38 +732,52 @@ def test_normalized_calls_require_unique_schema_valid_delegate_start(): "retry_of": "", "max_seconds": 0, } - assert assert_normalized_canary_call( - {"tool_calls": [_canonical_canary_call("call-defaults", provider_defaults)]}, - tools, - required, - )[0]["id"] == "call-defaults" + evidence = _normalized_call_violation(provider_defaults, expected) + assert evidence["violation"] == "arguments_exact_keys" + assert evidence["extra_keys"] == ["bucket", "max_seconds", "retry_of", "root", "skill_name"] + assert evidence["missing_keys"] == [] provider_selector = { "prompt": expected["prompt"], - "retry_of": "provider-contract-canary", + "retry_of": expected["subagent_id"], "root": "skill_payload", "bucket": "external", } - assert assert_normalized_canary_call( - {"tool_calls": [_canonical_canary_call("call-selector", provider_selector)]}, - tools, - required, - )[0]["id"] == "call-selector" + evidence = _normalized_call_violation(provider_selector, expected) + assert evidence["violation"] == "arguments_exact_keys" + assert evidence["extra_keys"] == ["bucket", "retry_of", "root"] + assert evidence["missing_keys"] == ["subagent_id"] + wrong_selector = dict(expected, subagent_id="another-subagent") + assert _normalized_call_violation(wrong_selector, expected)["violation"] == "arguments_subagent_id" wrong = dict(expected, prompt="different nonce-bearing prompt") - with pytest.raises(AssertionError): - assert_normalized_canary_call( - {"tool_calls": [_canonical_canary_call("call-wrong", wrong)]}, - tools, - required, - ) + assert _normalized_call_violation(wrong, expected)["violation"] == "arguments_prompt" duplicate = _canonical_canary_call("call-duplicate", expected) - with pytest.raises(AssertionError): - assert_normalized_canary_call( - {"tool_calls": [duplicate, copy.deepcopy(duplicate)]}, - tools, - required, + assert _normalized_call_violation( + None, expected, calls=[duplicate, copy.deepcopy(duplicate)], + )["violation"] == "duplicate_tool_call_id" + + +def test_canary_run_rejects_provider_filled_optional_keys_as_red(): + canary = next(row for row in provider_canary_matrix() if row.canary_id == "openrouter_gpt") + nonce = "forced-optional" + forced = {**delegate_start_canary_arguments(nonce), "retry_of": "", "max_seconds": 0} + + class ForcedOptionalClient: + def chat(self, **kwargs): + return { + "content": "", + "tool_calls": [_canonical_canary_call("forced-optional", forced)], + }, _fake_usage(canary, 1) + + with pytest.raises(AssertionError) as caught: + run_provider_contract_canary( + ForcedOptionalClient(), canary=canary, tools=full_registry_canary_tools(), nonce=nonce, ) + evidence = caught.value.args[0]["provider_contract_violation"] + assert evidence["violation"] == "arguments_exact_keys" + assert evidence["extra_keys"] == ["max_seconds", "retry_of"] + assert skip_on_provider_environmental_error(canary.canary_id, caught.value) is None def test_production_custom_none_text_is_semantic_success(monkeypatch): diff --git a/tests/test_request_wire_contract.py b/tests/test_request_wire_contract.py index ad0db5a92..735a62d25 100644 --- a/tests/test_request_wire_contract.py +++ b/tests/test_request_wire_contract.py @@ -2,6 +2,8 @@ from __future__ import annotations +import dataclasses +import itertools import json import multiprocessing from datetime import timedelta @@ -123,10 +125,10 @@ def _capture( ) -def _candidate(profile=None, pending=()): +def _candidate(profile=None, pending=(), payload=None): accepted = profile or _profile() target = _target(provider=accepted.provider, model=accepted.model) - source_payload = _payload() + source_payload = payload or _payload() source_payload["model"] = accepted.model source_payload["tool_choice"] = accepted.tool_choice candidate = bind_wire_candidate( @@ -158,10 +160,10 @@ def _custom_candidate(): ) -def _receipt(profile=None, pending=()): +def _receipt(profile=None, pending=(), payload=None): candidate = _custom_candidate() if ( profile is not None and profile.tool_dialect == "openai_chat_custom" - ) else _candidate(profile, pending) + ) else _candidate(profile, pending, payload) custom_receipts = () normalized_response = { "role": "assistant", @@ -240,6 +242,7 @@ def test_profile_isolates_route_surface_model_and_request_dialect_without_option _profile(tool_dialect_override="openai_chat_custom"), _profile(payload=_payload(tool_choice="auto")), _profile(function_strictness_override="plain"), + _profile(function_strictness_override="absent"), ] assert len({base.fingerprint, *(item.fingerprint for item in variants)}) == len(variants) + 1 @@ -253,6 +256,77 @@ def test_profile_isolates_route_surface_model_and_request_dialect_without_option assert named_probe.fingerprint != named_other.fingerprint +def _strict_payload(*values, server=False): + payload = _payload() + payload["tools"] = [ + {"type": "function", "function": {"name": f"probe{index or ''}", "parameters": {"type": "object"}, + **({} if value is _ABSENT else {"strict": value})}} + for index, value in enumerate(values) + ] + ([{"type": "openrouter:web_search"}] if server else []) + return payload + + +_ABSENT = object() + + +@pytest.mark.parametrize(("values", "expected"), [ + ((), "none"), + ((_ABSENT,), "absent"), + ((False,), "false"), + ((True,), "true"), + ((None, 0, 1, "false"), "malformed"), + ((False, _ABSENT), "absent+false"), + ((True, _ABSENT), "absent+true"), + ((True, False), "false+true"), + ((None, True, False, _ABSENT), "absent+false+true+malformed"), +]) +def test_function_strictness_names_each_physical_strict_state(values, expected): + payload = _strict_payload(*values, server=True) + assert wire.function_strictness(payload) == expected + assert _profile(payload=payload).function_strictness == expected + + +def test_absent_and_false_are_distinct_profiles_and_legacy_values_only_decode(): + states = ((_ABSENT,), (False,), (True, _ABSENT), (True, False), (None,)) + assert len({_profile(payload=_strict_payload(*values)).fingerprint for values in states}) == 5 + produced = {wire.function_strictness(_strict_payload(*values)) for size in range(5) + for values in itertools.product((_ABSENT, False, True, None), repeat=size)} + assert produced == wire.FUNCTION_STRICTNESS_VALUES - wire.LEGACY_FUNCTION_STRICTNESS_VALUES + base = _profile(payload=_strict_payload(_ABSENT)) + for legacy in ("plain", "strict", "mixed"): + decoded = dataclasses.replace(base, function_strictness=legacy) + assert decoded.fingerprint == canonical_sha256(dict(base.as_dict(), function_strictness=legacy)) + assert _profile(function_strictness_override=legacy).function_strictness == legacy + with pytest.raises(ValueError, match="function strictness"): + _profile(function_strictness_override="loose") + + +def test_historical_strictness_evidence_stays_stored_but_authorizes_no_current_profile(tmp_path): + action = {"kind": "drop_field", "fields": ["temperature"], "reason_code": "provider_unsupported_field"} + current = {values: _profile(payload=_strict_payload(*values)) + for values in ((_ABSENT,), (False,), (True, _ABSENT), (True, False))} + legacy = [dataclasses.replace(current[(_ABSENT,)], function_strictness="plain"), + dataclasses.replace(current[(True, _ABSENT)], function_strictness="mixed")] + path = tmp_path / "state" / REQUEST_WIRE_STATE_FILE + path.parent.mkdir(parents=True) + now = utc_now().isoformat() + stored = {profile.fingerprint: {"profile": profile.as_dict(), "observed_at": now, + "records": [{"action": action, "observed_at": now}]} + for profile in legacy} + path.write_text(json.dumps({"schema_version": 1, "profiles": stored})) + assert [wire._read_wire_actions_at(tmp_path, profile) for profile in legacy] == [(action,)] * 2 + assert all(wire._read_wire_actions_at(tmp_path, profile) == () for profile in current.values()) + + learned = current[(False,)] + receipt = _receipt(learned, (_pending(learned),), _strict_payload(False)) + assert wire._commit_wire_compatibility_at(tmp_path, receipt).committed + assert {values: wire._read_wire_actions_at(tmp_path, profile) for values, profile in current.items()} == { + (_ABSENT,): (), (False,): (action,), (True, _ABSENT): (), (True, False): ()} + profiles = json.loads(path.read_text())["profiles"] + assert {key: profiles[key] for key in stored} == stored + assert set(profiles) == {*stored, learned.fingerprint} + + def test_profile_isolates_provider_routing_reasoning_options_and_semantic_headers(): left = _payload() left["extra_body"] = {