From 2f8a06515ea30ebd8a4fc58cc39fc796a2ca937c Mon Sep 17 00:00:00 2001 From: Mohammad Hijjawi Date: Fri, 25 Sep 2026 23:59:51 +0100 Subject: [PATCH] Surface output-limit truncation instead of misformat retries Record the Chat Completions finish_reason on LLMResult and add a message_loop_result extension that, when a turn stopped at the output token limit (finish_reason "length"/"max_tokens") without a complete tool request, retries with a warning naming truncation as the cause rather than the generic misformat message. The warning counts toward the existing unusable-response loop limit. Context Doctor now leaves results already handled by an earlier extension untouched, so truncated envelopes are not repaired and dispatched with cut-off arguments. Fixes #1836 --- .../end/_90_stop_unusable_response_loop.py | 1 + .../python/message_loop_result/AGENTS.md | 1 + .../_05_truncated_response.py | 41 +++++++++++++++++++ helpers/litellm_transport.py | 16 +++++++- helpers/litellm_transport.py.dox.md | 1 + helpers/llm_result.py | 11 +++++ helpers/llm_result.py.dox.md | 1 + models.py | 1 + plugins/_context_doctor/AGENTS.md | 1 + .../message_loop_result/_10_context_doctor.py | 2 + .../tests/test_context_doctor.py | 13 ++++++ prompts/AGENTS.md | 1 + prompts/fw.msg_output_limit.md | 1 + prompts/fw.msg_output_limit_response.md | 1 + tests/test_message_loop_result.py | 32 ++++++++++++++- tests/test_model_refusal.py | 30 ++++++++++++++ tests/test_unusable_response_loop.py | 15 +++++++ 17 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 extensions/python/message_loop_result/_05_truncated_response.py create mode 100644 prompts/fw.msg_output_limit.md create mode 100644 prompts/fw.msg_output_limit_response.md diff --git a/extensions/python/_functions/agent/Agent/hist_add_warning/end/_90_stop_unusable_response_loop.py b/extensions/python/_functions/agent/Agent/hist_add_warning/end/_90_stop_unusable_response_loop.py index e28abfcf9..34e52ebf9 100644 --- a/extensions/python/_functions/agent/Agent/hist_add_warning/end/_90_stop_unusable_response_loop.py +++ b/extensions/python/_functions/agent/Agent/hist_add_warning/end/_90_stop_unusable_response_loop.py @@ -25,6 +25,7 @@ class StopUnusableResponseLoop(Extension): self.agent.read_prompt("fw.msg_empty_response.md"), self.agent.read_prompt("fw.msg_reasoning_only.md"), self.agent.read_prompt("fw.msg_thoughts_fallback.md"), + self.agent.read_prompt("fw.msg_output_limit.md"), }: return diff --git a/extensions/python/message_loop_result/AGENTS.md b/extensions/python/message_loop_result/AGENTS.md index 7ad642722..f98d2e984 100644 --- a/extensions/python/message_loop_result/AGENTS.md +++ b/extensions/python/message_loop_result/AGENTS.md @@ -7,6 +7,7 @@ ## Ownership - Extensions receive mutable `result_data` with `llm_result` and may set `skip_default_processing` after fully handling the turn. +- `_05_truncated_response.py` retries turns the provider cut off at the output token limit (`LLMResult.truncated`) that carry no complete tool request: add an agent-facing `fw.msg_output_limit.md` history warning with a separate `fw.msg_output_limit_response.md` user notice naming the finish reason, instead of the generic misformat warning, and count them through the stop-unusable-response-loop extension. - `_20_empty_response.py` retries fully empty turns (no response and no reasoning): count them toward the unusable-response limit without adding a warning to model history, and use `fw.msg_empty_response.md` for agent-prefixed UI warning text only. - `_20_empty_response.py` retries reasoning-only turns: add an agent-facing `fw.msg_reasoning_only.md` history warning with a separate `fw.msg_reasoning_only_response.md` user notice, and count them through the stop-unusable-response-loop extension. - `_30_repeat_response.py` compares `LLMResult.repeat_response_text()` for native calls (public text plus all calls), otherwise full response text. It retries content that exactly matches `loop_data.last_response`, regardless of separate reasoning or provider IDs, using `fw.msg_repeat.md` for history and `fw.msg_repeat_response.md` for the agent-prefixed UI warning text. State advancement is owned by `hist_add_ai_response`; do not call `_remember_llm_result_state` manually. diff --git a/extensions/python/message_loop_result/_05_truncated_response.py b/extensions/python/message_loop_result/_05_truncated_response.py new file mode 100644 index 000000000..f98720e12 --- /dev/null +++ b/extensions/python/message_loop_result/_05_truncated_response.py @@ -0,0 +1,41 @@ +"""Retry model turns cut off by the output token limit with a specific warning.""" + +from __future__ import annotations + +from typing import Any + +from helpers import extract_tools +from helpers.extension import Extension +from helpers.print_style import PrintStyle + + +class TruncatedResponse(Extension): + def execute(self, result_data: dict[str, Any] | None = None, **kwargs: Any) -> None: + if not self.agent or not isinstance(result_data, dict): + return + if result_data.get("skip_default_processing"): + return + + llm_result = result_data.get("llm_result") + if not getattr(llm_result, "truncated", False): + return + if getattr(llm_result, "function_calls", None): + return + response = getattr(llm_result, "response", "") + if isinstance(response, str) and extract_tools.extract_tool_request(response): + return + + # Name truncation as the cause instead of the generic misformat warning. + warning = self.agent.read_prompt("fw.msg_output_limit.md") + warning_message = self.agent.hist_add_warning(message=warning) + notice = self.agent.read_prompt( + "fw.msg_output_limit_response.md", + finish_reason=getattr(llm_result, "finish_reason", ""), + ) + PrintStyle(font_color="orange", padding=True).print(notice) + self.agent.context.log.log( + type="warning", + content=f"{self.agent.agent_name}: {notice}", + id=warning_message.id, + ) + result_data["skip_default_processing"] = True diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.py index 7c38957bd..cd6574c99 100644 --- a/helpers/litellm_transport.py +++ b/helpers/litellm_transport.py @@ -182,6 +182,7 @@ class LiteLLMTransport: stop: Optional[list[str]] = None policy: TransportPolicy = field(init=False) last_result: LLMResult | None = field(init=False, default=None) + last_finish_reason: str = field(init=False, default="") last_request_state: str = field(init=False, default=RESPONSES_STATE_PROVIDER) explicit_prompt_caching: bool = field(init=False, default=False) history_prefix_hash: str = field(init=False, default="") @@ -211,6 +212,7 @@ class LiteLLMTransport: if self.policy.mode is TransportMode.CHAT_COMPLETIONS: raw_response = completion(**self._chat_request(stream=False)) parsed = ChatCompletionsTransport.parse(raw_response) + self.last_finish_reason = _finish_reason(raw_response) self.last_result = self._llm_result_from_chat( parsed, raw_response ) @@ -235,6 +237,7 @@ class LiteLLMTransport: **self._chat_request(stream=False) ) parsed = ChatCompletionsTransport.parse(raw_response) + self.last_finish_reason = _finish_reason(raw_response) self.last_result = self._llm_result_from_chat( parsed, raw_response ) @@ -269,6 +272,7 @@ class LiteLLMTransport: if _has_chunk_delta(parsed): got_any_chunk = True yield parsed + self.last_finish_reason = parser.finish_reason self.last_result = self._stream_result_from_chat_parser(parser) else: request = self._responses_request(stream=True) @@ -310,6 +314,7 @@ class LiteLLMTransport: if _has_chunk_delta(parsed): got_any_chunk = True yield parsed + self.last_finish_reason = parser.finish_reason self.last_result = self._stream_result_from_chat_parser(parser) else: request = self._responses_request(stream=True) @@ -452,6 +457,7 @@ class LiteLLMTransport: output_items=parsed.get("_output_items"), provider_model_key=self.model, capability=self._capability_metadata(), + finish_reason=self.last_finish_reason, ) def _llm_result_from_response( @@ -489,6 +495,7 @@ class LiteLLMTransport: output_items=output_items, provider_model_key=self.model, capability=self._capability_metadata(), + finish_reason=self.last_finish_reason, ) def _capability_metadata(self) -> dict[str, Any]: @@ -680,6 +687,7 @@ class ChatCompletionsStreamParser: self.order: list[str] = [] self.emitted = False self.usage: dict[str, Any] = {} + self.finish_reason = "" def parse(self, chunk: Any) -> ChatChunk: if usage := _reported_usage(chunk): @@ -690,7 +698,9 @@ class ChatCompletionsStreamParser: self._append_tool_calls(_get_value(delta, "tool_calls")) self._append_legacy_function_call(_get_value(delta, "function_call")) - if _get_value(choice, "finish_reason") in {"tool_calls", "function_call"}: + if finish_reason := _finish_reason(chunk): + self.finish_reason = finish_reason + if finish_reason in {"tool_calls", "function_call"}: text = self._emit() if text and not parsed["response_delta"]: parsed["response_delta"] = text @@ -2222,6 +2232,10 @@ def _first_choice(chunk: Any) -> Any: return choices[0] if choices else {} +def _finish_reason(chunk: Any) -> str: + return str(_get_value(_first_choice(chunk), "finish_reason") or "") + + def _get_value(obj: Any, key: str) -> Any: if isinstance(obj, dict): return obj.get(key) diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.md index 9707f56b0..2bf844204 100644 --- a/helpers/litellm_transport.py.dox.md +++ b/helpers/litellm_transport.py.dox.md @@ -37,6 +37,7 @@ - Fall back to Chat Completions when LiteLLM's Responses mock streaming path tries to JSON-decode a real SSE stream before any output. - Preserve Chat Completions tool calls from both non-streaming responses and streaming deltas as canonical `LLMResult` function-call items. Non-streaming calls stay canonical when accompanied by commentary; retain that commentary as a separate output message. - Chat Completions `content_filter` and `refusal` finish reasons raise LiteLLM `ContentPolicyViolationError` on streaming and non-streaming paths; never treat blocked output as an empty success or dispatch partial tool calls. Verify with `tests/test_model_refusal.py`. +- Record the Chat Completions `finish_reason` from streaming and non-streaming responses as `last_finish_reason` and on the resulting `LLMResult`, so callers can tell output-limit truncation apart from misformatted output. Verify with `tests/test_model_refusal.py`. - Responses messages explicitly marked `phase: commentary` produce an incremental `response_preview` with public text in `thoughts`, followed by the native tool preview. Previews reuse parsed call state; multiple calls form one parallel envelope. Only native agent-turn callbacks consume the preview: legacy callbacks, response/reasoning deltas, completed output items and call IDs remain unchanged. Unmarked/final-answer text and reasoning are not reclassified as commentary. - Preserve provider usage and LiteLLM response cost for both transports only when the response or stream actually supplies them; do not synthesize unavailable provider accounting. - Streaming Chat Completions requests include `stream_options.include_usage` so terminal usage events reach `LLMResult.usage` for OpenAI-compatible endpoints and Messages API providers alike, unless the model configuration previously rejected the option or the request already carries it. A pre-output rejection retries once without the transport-injected `include_usage` and remembers the rejection per model configuration in a process-global cache that naturally clears when the framework process restarts; tests reset it through `clear_transport_capability_cache`. User-provided `stream_options` keys are always passed through untouched, even when the provider rejects them. diff --git a/helpers/llm_result.py b/helpers/llm_result.py index 85efb12a2..c3cbf16c5 100644 --- a/helpers/llm_result.py +++ b/helpers/llm_result.py @@ -9,6 +9,7 @@ RESPONSE_METADATA_KEY = "responses" LOCAL_FUNCTION_TOOL_TYPES = {"function_call"} TEXT_OUTPUT_TYPES = {"message"} REASONING_OUTPUT_TYPES = {"reasoning"} +TRUNCATED_FINISH_REASONS = {"length", "max_tokens"} @dataclass @@ -64,6 +65,7 @@ class LLMResult: raw: dict[str, Any] = field(default_factory=dict) capability: dict[str, Any] = field(default_factory=dict) history_extras: str = "" + finish_reason: str = "" @classmethod def from_dict(cls, data: dict[str, Any] | None) -> "LLMResult": @@ -86,6 +88,7 @@ class LLMResult: raw=object_to_dict(data.get("raw") or {}), capability=object_to_dict(data.get("capability") or {}), history_extras=extras if isinstance(extras, str) else "", + finish_reason=str(data.get("finish_reason") or ""), ) @classmethod @@ -133,6 +136,7 @@ class LLMResult: output_items: list[dict[str, Any]] | None = None, provider_model_key: str = "", capability: dict[str, Any] | None = None, + finish_reason: str = "", ) -> "LLMResult": items = [ResponseItem.from_any(item) for item in output_items or []] if response and not items: @@ -167,6 +171,7 @@ class LLMResult: state="off", usage=object_to_dict(usage or {}), capability=dict(capability or {}), + finish_reason=finish_reason, ) if not result.response and result.function_calls: result.response = result.function_calls_text() @@ -177,6 +182,11 @@ class LLMResult: """Sentinel for non-LLM AI turns; no response_id, mode/state off.""" return cls(mode="", state="off") + @property + def truncated(self) -> bool: + """True when the provider stopped because the output token limit was reached.""" + return self.finish_reason in TRUNCATED_FINISH_REASONS + @property def function_calls(self) -> list[ResponseFunctionCall]: calls: list[ResponseFunctionCall] = [] @@ -233,6 +243,7 @@ class LLMResult: "raw": self.raw, "capability": self.capability, "history_extras": self.history_extras, + "finish_reason": self.finish_reason, } def metadata(self) -> dict[str, Any]: diff --git a/helpers/llm_result.py.dox.md b/helpers/llm_result.py.dox.md index 85e30b66f..1306ca53b 100644 --- a/helpers/llm_result.py.dox.md +++ b/helpers/llm_result.py.dox.md @@ -22,6 +22,7 @@ - `from_dict(...)` preserves explicit modes, including the empty non-LLM sentinel; only missing or null modes use the legacy `responses` default. - `from_chat(...)` must produce an equivalent chat-completions result with `mode="chat_completions"` and `state="off"`, preserving optional function-call output items and provider usage when the chat transport supplies them. - Function-call output items must preserve `call_id` and optional acknowledged safety checks. +- `finish_reason` carries the provider stop reason for the turn and is not persisted in history metadata; `truncated` is true for output-token-limit reasons (`length`, `max_tokens`). - `repeat_response_text()` combines response text and canonical calls for native repeat detection, excluding provider IDs, usage and separate reasoning. Non-native replies retain their full response text. This comparison value is transient and does not replace canonical call history or provider metadata. - Argument parsing must tolerate JSON strings, dictionaries, and malformed values without throwing. diff --git a/models.py b/models.py index 53a9d1973..3d2f3f5c1 100644 --- a/models.py +++ b/models.py @@ -776,6 +776,7 @@ class LiteLLMChatWrapper(SimpleChatModel): input_items=ResponsesTransport.input_from_messages(msgs_conv), provider_model_key=self.model_name, capability=transport._capability_metadata(), + finish_reason=transport.last_finish_reason, ) if result.output()["response_delta"] and not llm_result.function_calls: llm_result.response = result.output()["response_delta"] diff --git a/plugins/_context_doctor/AGENTS.md b/plugins/_context_doctor/AGENTS.md index 1c5d41471..6d9dda7c0 100644 --- a/plugins/_context_doctor/AGENTS.md +++ b/plugins/_context_doctor/AGENTS.md @@ -17,6 +17,7 @@ - Leave canonical `LLMResult.function_calls` and their accompanying text untouched in either transport. The agent dispatches those calls through the normal tool-policy gate; text repair must not suppress them or reinterpret a textual follow-up as another call. - Repaired and fallback JSON is always minified. +- Leave results already handled by an earlier `message_loop_result` extension (`skip_default_processing`) untouched, such as output-limit truncation retries. - Usable Responses output text stays untouched for the core response-tool dispatcher. Recognizable tool intent still enters repair; Chat Completions non-tool output becomes `{"thoughts":[raw]}` and XML-like output becomes `{}` when suppression is enabled. - Blank-line-separated thoughts expand into separate entries after repair when the split strategy is enabled (default on). This rewrites model-authored thoughts for every repaired turn by design, not only the raw-text fallback branch. - `_select_value` returns only repaired tool calls or model-authored partial responses; `transform_response` wraps otherwise-raw output as thoughts afterward. `looks_like_tool_call` rejects that synthesized wrapper, so it reaches the fallback branch. Repaired output is usable only with a non-empty thoughts list, headline/tool name string, or tool-args dict. diff --git a/plugins/_context_doctor/extensions/python/message_loop_result/_10_context_doctor.py b/plugins/_context_doctor/extensions/python/message_loop_result/_10_context_doctor.py index 5b7467ff9..868001ae4 100644 --- a/plugins/_context_doctor/extensions/python/message_loop_result/_10_context_doctor.py +++ b/plugins/_context_doctor/extensions/python/message_loop_result/_10_context_doctor.py @@ -21,6 +21,8 @@ class ContextDoctor(Extension): def execute(self, result_data: dict[str, Any] | None = None, **kwargs: Any) -> None: if not self.agent or not isinstance(result_data, dict): return + if result_data.get("skip_default_processing"): + return # Extract LLM response llm_result = result_data.get("llm_result") diff --git a/plugins/_context_doctor/tests/test_context_doctor.py b/plugins/_context_doctor/tests/test_context_doctor.py index d01ae0b08..cb26491b1 100644 --- a/plugins/_context_doctor/tests/test_context_doctor.py +++ b/plugins/_context_doctor/tests/test_context_doctor.py @@ -214,6 +214,19 @@ async def test_extension_leaves_native_calls_and_accompanying_text_intact(monkey assert agent._execute_tool_request.call_args.kwargs["tool_name"] == "code_execution_tool" +def test_leaves_results_handled_by_earlier_extensions_untouched(): + response = '{"thoughts":["planning"],"tool_name":"response","tool_args":{"text":"cut' + result = LLMResult.from_chat(response=response, finish_reason="length") + agent = SimpleNamespace(hist_add_ai_response=AsyncMock(), hist_add_warning=AsyncMock()) + data = {"llm_result": result, "skip_default_processing": True} + + ContextDoctor(agent).execute(data) + + assert result.response == response + agent.hist_add_ai_response.assert_not_called() + agent.hist_add_warning.assert_not_called() + + @pytest.mark.asyncio @pytest.mark.parametrize("text", ["Done.", "First paragraph.\n\nSecond paragraph.", "

Done.

"]) async def test_responses_text_reaches_core_response_dispatch(monkeypatch, text): diff --git a/prompts/AGENTS.md b/prompts/AGENTS.md index fe976a894..48f0a2c3d 100644 --- a/prompts/AGENTS.md +++ b/prompts/AGENTS.md @@ -33,6 +33,7 @@ - `Agent.read_prompt` and `Agent.parse_prompt` strip trailing newlines from returned content, so prompt template files may safely keep a final newline. Do not rely on trailing newlines for formatting; render-time joins in `output_text()` already separate turns. - `fw.initial_user_message.md` owns the placeholder user turn (`Hello!`) that `extensions/python/agent_init/_10_initial_message.py` pairs before the AI greeting from `fw.initial_message.md`; the user-first ordering guarantees `output_langchain` does not pop the greeting as a leading `AIMessage` on project or non-project paths. Do not remove either prompt without replacing the turn-order guarantee. - `fw.msg_reasoning_only.md` and `fw.msg_reasoning_only_response.md` own the reasoning-only turn warning pair used by `extensions/python/message_loop_result/_20_empty_response.py`. +- `fw.msg_output_limit.md` and `fw.msg_output_limit_response.md` own the output-limit truncation warning pair used by `extensions/python/message_loop_result/_05_truncated_response.py`. ## Work Guidance diff --git a/prompts/fw.msg_output_limit.md b/prompts/fw.msg_output_limit.md new file mode 100644 index 000000000..15a69b1ad --- /dev/null +++ b/prompts/fw.msg_output_limit.md @@ -0,0 +1 @@ +Your previous response was cut off because it reached the output token limit, so no complete tool request was received. Respond again with a single valid tool-call JSON object that fits within the limit: keep thoughts and tool arguments brief, and split large content across multiple steps. diff --git a/prompts/fw.msg_output_limit_response.md b/prompts/fw.msg_output_limit_response.md new file mode 100644 index 000000000..8a8546895 --- /dev/null +++ b/prompts/fw.msg_output_limit_response.md @@ -0,0 +1 @@ +Response was truncated at the model output token limit (finish_reason={{finish_reason}}). Retrying; if this repeats, raise the model's output token limit. diff --git a/tests/test_message_loop_result.py b/tests/test_message_loop_result.py index 2757097a9..a28cc2aac 100644 --- a/tests/test_message_loop_result.py +++ b/tests/test_message_loop_result.py @@ -4,6 +4,7 @@ from types import SimpleNamespace import pytest from extensions.python.message_loop_result import _20_empty_response as empty_response +from extensions.python.message_loop_result._05_truncated_response import TruncatedResponse from extensions.python.message_loop_result._20_empty_response import EmptyResponse from extensions.python.message_loop_result._30_repeat_response import RepeatResponse from extensions.python._functions.agent.Agent.hist_add_warning.end import ( @@ -39,6 +40,8 @@ class FakeAgent: "fw.msg_repeat_response.md": "Repeated response detected. Retrying.", "fw.msg_reasoning_only.md": "reasoning only", "fw.msg_reasoning_only_response.md": "Reasoning-only response detected. Retrying.", + "fw.msg_output_limit.md": "output limit", + "fw.msg_output_limit_response.md": f"truncated ({kwargs.get('finish_reason')})", }[name] def hist_add_ai_response(self, response, **kwargs): @@ -53,10 +56,13 @@ class FakeAgent: return SimpleNamespace(id="warning") -def _run(agent): +def _run(agent, **llm_result): result_data = { - "llm_result": SimpleNamespace(response=agent.response, reasoning=agent.reasoning) + "llm_result": SimpleNamespace( + response=agent.response, reasoning=agent.reasoning, **llm_result + ) } + TruncatedResponse(agent).execute(result_data) EmptyResponse(agent).execute(result_data) RepeatResponse(agent).execute(result_data) return result_data @@ -224,3 +230,25 @@ def test_chat_repeat_includes_public_planning_fields(field, change): agent = FakeAgent(json.dumps(current), last_response=json.dumps(previous)) assert bool(_run(agent).get("skip_default_processing")) is (change == "same") + + +def test_truncated_result_warns_about_output_limit_instead_of_misformat(): + agent = FakeAgent('{"thoughts":["planning"],"tool_name":"response","tool_args":{"text":"cut') + + result = _run(agent, finish_reason="length", truncated=True, function_calls=[]) + + assert result["skip_default_processing"] is True + assert agent.history == [] + assert agent.warnings == ["output limit"] + assert agent.logs == [ + {"type": "warning", "content": "A0: truncated (length)", "id": "warning"} + ] + + +def test_truncated_result_with_complete_tool_request_is_processed_normally(): + agent = FakeAgent('{"tool_name":"response","tool_args":{"text":"done"}}') + + result = _run(agent, finish_reason="length", truncated=True, function_calls=[]) + + assert "skip_default_processing" not in result + assert agent.warnings == [] diff --git a/tests/test_model_refusal.py b/tests/test_model_refusal.py index d69aacabc..7f3864c34 100644 --- a/tests/test_model_refusal.py +++ b/tests/test_model_refusal.py @@ -51,3 +51,33 @@ async def test_refusal_stops_model_call_without_retry_and_closes_stream(monkeypa ) assert calls == 1 assert closed == stream + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_length_finish_reason_is_recorded_as_truncation(monkeypatch, stream): + partial = '{"thoughts":["planning"],"tool_name":"response","tool_args":{"text":"cut' + raw = {"model": "test", "choices": [{"finish_reason": "length", "message": {"content": partial}}]} + chunks_raw = [ + {"model": "test", "choices": [{"finish_reason": None, "delta": {"content": partial}}]}, + {"model": "test", "choices": [{"finish_reason": "length", "delta": {}}]}, + ] + + async def chunks(): + for chunk in chunks_raw: + yield chunk + + async def completion(**kwargs): + return chunks() if kwargs["stream"] else raw + + async def callback(*args): + return None + + monkeypatch.setattr(litellm_transport, "acompletion", completion) + wrapper = models.LiteLLMChatWrapper(model="test", provider="openai") + result = await wrapper.unified_turn.__wrapped__( + wrapper, messages=[], response_callback=callback if stream else None, + ) + assert result.response == partial + assert result.finish_reason == "length" + assert result.truncated is True diff --git a/tests/test_unusable_response_loop.py b/tests/test_unusable_response_loop.py index 51c46d26f..23af8e5f5 100644 --- a/tests/test_unusable_response_loop.py +++ b/tests/test_unusable_response_loop.py @@ -24,6 +24,7 @@ def _agent(): "fw.msg_empty_response.md": "empty response", "fw.msg_reasoning_only.md": "reasoning only", "fw.msg_thoughts_fallback.md": "thoughts fallback", + "fw.msg_output_limit.md": "output limit", } def read_prompt(name, **kwargs): @@ -130,3 +131,17 @@ def test_general_settings_expose_the_default_failure_limit(): assert "after 3 consecutive" in read_prompt_file( "fw.msg_unusable_response_limit.md", ["prompts"], limit=3 ) + + +def test_output_limit_warning_counts_toward_the_limit(monkeypatch): + monkeypatch.setattr( + response_loop, + "get_settings", + lambda: {"max_consecutive_unusable_responses": 1}, + ) + agent = _agent() + extension = response_loop.StopUnusableResponseLoop(agent=agent) + + data = _run(extension, agent, "output limit") + + assert isinstance(data["exception"], HandledException)