From 431cea3eea16012aa60dc2cfccedbaeadd62b729 Mon Sep 17 00:00:00 2001 From: Wojciech Wais <12673503+wojciech-wais@users.noreply.github.com> Date: Wed, 15 Apr 2026 10:32:47 +0200 Subject: [PATCH] [Bugfix] Fix tool_calls Iterable consumed when debug logging is enabled (#34844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Wojciech Wais Signed-off-by: mgoin Signed-off-by: Xinyu Chen Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Signed-off-by: Rishi Puri Signed-off-by: Jaebok Lee Signed-off-by: DarkLight1337 Signed-off-by: yuwei Signed-off-by: Artem Perevedentsev Signed-off-by: Ibrahim Arshad <38925737+ibrahim1023@users.noreply.github.com> Signed-off-by: Li Signed-off-by: chaunceyjiang Signed-off-by: Kunshang Ji Signed-off-by: Kunshang Ji Signed-off-by: R Signed-off-by: Lucas Wilkinson Signed-off-by: lkm2835 Signed-off-by: Ronen Schaffer Signed-off-by: vnadathur Signed-off-by: WorldExplored Signed-off-by: Srreyansh Sethi <107075589+WorldExplored@users.noreply.github.com> Signed-off-by: Isotr0py Signed-off-by: Elham Harirpoush Signed-off-by: Yan Ma Signed-off-by: Nick Hill Signed-off-by: jackcfwang Signed-off-by: Chendi Xue Signed-off-by: Injae Ryou Signed-off-by: Richard Zou Signed-off-by: milesial Signed-off-by: Elvir Crncevic Signed-off-by: whx-sjtu <2952154980@qq.com> Signed-off-by: Lalithnarayan C Signed-off-by: PatchouliTaisa Signed-off-by: jatseng-ai Signed-off-by: jatseng-ai Signed-off-by: Matthias Gehre Signed-off-by: xaguilar-amd Signed-off-by: rdondeti Signed-off-by: Ravitez Dondeti Signed-off-by: NickLucche Signed-off-by: Peter Nguyen Signed-off-by: wang.yuqi Signed-off-by: zhuhaoran Signed-off-by: Jee Jee Li Signed-off-by: tjtanaa Signed-off-by: Jesus Federico Signed-off-by: manu Signed-off-by: ZhanqiuHu Signed-off-by: Yifan Zong Signed-off-by: Rahul-Tuli Signed-off-by: Fynn Schmitt-Ulms Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Michael Goin Signed-off-by: Benjamin Chislett Signed-off-by: Tianyu Guo Signed-off-by: leeyongjun Signed-off-by: Ziying Tao Signed-off-by: jiang1.li Signed-off-by: Vibhav Agarwal Signed-off-by: ShubyM Signed-off-by: wzhao18 Signed-off-by: Itay Etelis Signed-off-by: EdalatiAli Signed-off-by: Andreas Karatzas Signed-off-by: r266-tech Signed-off-by: Roger Wang Signed-off-by: Martin Hickey Signed-off-by: Mark McLoughlin Signed-off-by: Animesh Jain Signed-off-by: Yongye Zhu Signed-off-by: zhxchen17 Signed-off-by: EricccYang Signed-off-by: Kaicheng Yang <53411596+EricccYang@users.noreply.github.com> Signed-off-by: baoloongmao Signed-off-by: sihao.li Signed-off-by: sfeng33 <4florafeng@gmail.com> Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Signed-off-by: Zhu, Zufang Signed-off-by: Tihomir Elek Signed-off-by: yiliu30 Signed-off-by: yewentao256 Signed-off-by: Santino Ramos Signed-off-by: haosdent Signed-off-by: JartX Signed-off-by: George-ao Signed-off-by: Yuyi Ao Signed-off-by: Tyler Michael Smith Signed-off-by: Mukesh Baphna Signed-off-by: Pedram Razavi Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Michael Goin Co-authored-by: Xinyu Chen Co-authored-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: Rishi Puri Co-authored-by: zzaebok <44357534+zzaebok@users.noreply.github.com> Co-authored-by: Cyrus Leung Co-authored-by: Yuwei An Co-authored-by: yuwei Co-authored-by: Artem Perevedentsev Co-authored-by: Ibrahim Arshad <38925737+ibrahim1023@users.noreply.github.com> Co-authored-by: Chuan (Richard) Li Co-authored-by: Chauncey Co-authored-by: Kunshang Ji Co-authored-by: Ganesh R Co-authored-by: Lucas Wilkinson Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Kyungmin Lee <30465912+lkm2835@users.noreply.github.com> Co-authored-by: Ronen Schaffer Co-authored-by: Srreyansh Sethi <107075589+WorldExplored@users.noreply.github.com> Co-authored-by: vnadathur Co-authored-by: vnadathur <236933696+vnadathur@users.noreply.github.com> Co-authored-by: Isotr0py Co-authored-by: Elham Co-authored-by: Yan Ma Co-authored-by: Nick Hill Co-authored-by: Chaofan Wang Co-authored-by: Chendi.Xue Co-authored-by: Injae Ryou Co-authored-by: Richard Zou Co-authored-by: milesial Co-authored-by: Elvir Crnčević Co-authored-by: Claude Sonnet 4 Co-authored-by: Hexiang Wang <56632993+whx-sjtu@users.noreply.github.com> Co-authored-by: Lalithnarayan C Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Luka Govedič Co-authored-by: PatchyTIS <58251192+PatchouliTIS@users.noreply.github.com> Co-authored-by: PatchouliTaisa Co-authored-by: jatseng-ai Co-authored-by: Matthias Gehre Co-authored-by: xaguilar-amd Co-authored-by: Ravitez Dondeti Co-authored-by: Nicolò Lucchesi Co-authored-by: Peter Nguyen Co-authored-by: wang.yuqi Co-authored-by: zhrrr <43847754+izhuhaoran@users.noreply.github.com> Co-authored-by: Jee Jee Li Co-authored-by: TJian Co-authored-by: Jesus Federico <14651+jefp@users.noreply.github.com> Co-authored-by: Manu Co-authored-by: zhanqiuhu <49648934+ZhanqiuHu@users.noreply.github.com> Co-authored-by: yzong-rh Co-authored-by: Fynn Schmitt-Ulms Co-authored-by: Rahul-Tuli Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Benjamin Chislett Co-authored-by: Tianyu Guo Co-authored-by: Lee Yongjun <35302114+elwhyjay@users.noreply.github.com> Co-authored-by: z1ying <55220715+z1ying@users.noreply.github.com> Co-authored-by: Li, Jiang Co-authored-by: Vibhav Agarwal Co-authored-by: vibhav-agarwal Co-authored-by: ShubyM Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Itay Etelis Co-authored-by: EdalatiAli Co-authored-by: Andreas Karatzas Co-authored-by: r266-tech Co-authored-by: Roger Wang Co-authored-by: Martin Hickey Co-authored-by: Or Ozeri Co-authored-by: Mark McLoughlin Co-authored-by: Le Yang <562593859@qq.com> Co-authored-by: Animesh Jain Co-authored-by: Yongye Zhu Co-authored-by: Zhengxu Chen Co-authored-by: Kaicheng Yang <53411596+EricccYang@users.noreply.github.com> Co-authored-by: maobaolong Co-authored-by: sihao_li <165983188+1643661061leo@users.noreply.github.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: zofia <110436990+zufangzhu@users.noreply.github.com> Co-authored-by: Tihomir Elek Co-authored-by: Yi Liu Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Santino Ramos <51103228+santiramos27@users.noreply.github.com> Co-authored-by: haosdent Co-authored-by: JartX Co-authored-by: Yuyi Ao Co-authored-by: Tyler Michael Smith Co-authored-by: mukesh-hai Co-authored-by: Pedram Razavi --- .../openai/test_tool_calls_serialization.py | 150 ++++++++++++++++++ vllm/entrypoints/chat_utils.py | 4 +- .../openai/chat_completion/protocol.py | 41 +++++ 3 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 tests/entrypoints/openai/test_tool_calls_serialization.py diff --git a/tests/entrypoints/openai/test_tool_calls_serialization.py b/tests/entrypoints/openai/test_tool_calls_serialization.py new file mode 100644 index 0000000000..cedc2575b8 --- /dev/null +++ b/tests/entrypoints/openai/test_tool_calls_serialization.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for tool_calls Iterable → list materialisation. + +Regression tests for https://github.com/vllm-project/vllm/issues/34792. + +Setting VLLM_LOGGING_LEVEL=debug caused tool calling to break for Mistral +models because: + 1. The OpenAI Python SDK types tool_calls as Iterable[...] in + ChatCompletionAssistantMessageParam. + 2. Pydantic v2, when validating from Python objects (not from raw JSON), + wraps Iterable fields in a one-shot lazy iterator. + 3. Debug logging called model_dump_json() which consumed that iterator. + 4. The Mistral tokenizer then saw empty tool_calls and raised + "ValueError: Unexpected tool call id ...". +""" + +import pytest + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + + +def _make_tool_call(tc_id: str, name: str, args: str) -> dict: + return { + "id": tc_id, + "type": "function", + "function": {"name": name, "arguments": args}, + } + + +def _make_request(messages: list) -> ChatCompletionRequest: + return ChatCompletionRequest( + model="test-model", + messages=messages, + ) + + +def test_tool_calls_list_preserved_after_model_dump(): + """tool_calls in assistant messages must be readable after model_dump_json. + + When the request is built from Python dicts (as in the Anthropic → OpenAI + conversion path), Pydantic v2 previously wrapped the Iterable tool_calls + in a one-shot iterator. model_dump_json() consumed it, leaving subsequent + readers (e.g. the Mistral tokenizer) with an empty sequence. + """ + tool_call = _make_tool_call("call_abc123", "get_weather", '{"city": "Paris"}') + messages = [ + {"role": "user", "content": "What is the weather in Paris?"}, + {"role": "assistant", "content": None, "tool_calls": [tool_call]}, + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": '{"temperature": 20}', + }, + ] + + req = _make_request(messages) + + # Simulate debug logging: serialize the model (this was the trigger) + _ = req.model_dump_json() + + # The assistant message must still have accessible tool_calls afterwards + assistant_msg = req.messages[1] + assert isinstance(assistant_msg, dict) + tool_calls = assistant_msg.get("tool_calls") + assert tool_calls is not None, "tool_calls must not be None after model_dump_json" + assert isinstance(tool_calls, list), "tool_calls must be a list" + assert len(tool_calls) > 0, "tool_calls must not be empty after model_dump_json" + + +def test_tool_calls_from_generator_are_materialised(): + """tool_calls passed as a generator must be converted to list on validation.""" + tool_call = _make_tool_call("call_gen1", "search", '{"query": "vllm"}') + + def tool_calls_gen(): + yield tool_call + + messages = [ + {"role": "user", "content": "Search for vllm"}, + { + "role": "assistant", + "content": None, + "tool_calls": tool_calls_gen(), # one-shot generator + }, + ] + + req = _make_request(messages) + assistant_msg = req.messages[1] + assert isinstance(assistant_msg, dict) + + # Iterate twice — must not raise or return empty on second pass + tool_calls_first = list(assistant_msg.get("tool_calls", [])) + tool_calls_second = list(assistant_msg.get("tool_calls", [])) + + assert len(tool_calls_first) == 1, "First read must return the tool call" + assert len(tool_calls_second) == 1, "Second read must also return the tool call" + + +def test_tool_calls_list_passthrough(): + """tool_calls already provided as a list must remain a list.""" + tool_call = _make_tool_call("call_list1", "calculate", '{"expr": "2+2"}') + messages = [ + {"role": "user", "content": "Calculate 2+2"}, + {"role": "assistant", "content": None, "tool_calls": [tool_call]}, + ] + + req = _make_request(messages) + assistant_msg = req.messages[1] + assert isinstance(assistant_msg, dict) + assert isinstance(assistant_msg.get("tool_calls"), list) + + +def test_messages_without_tool_calls_unaffected(): + """Messages without tool_calls must be handled correctly.""" + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": "Hi there!"}, + ] + + req = _make_request(messages) + # None of the messages should have tool_calls injected + for msg in req.messages: + assert isinstance(msg, dict) + assert msg.get("tool_calls") is None or msg.get("tool_calls") == [] + + +@pytest.mark.parametrize("num_tool_calls", [1, 3]) +def test_multiple_tool_calls_materialised(num_tool_calls: int): + """Multiple tool calls in a single message are all preserved.""" + tool_calls = [ + _make_tool_call(f"call_{i}", f"func_{i}", f'{{"arg": {i}}}') + for i in range(num_tool_calls) + ] + messages = [ + {"role": "user", "content": "Do things"}, + {"role": "assistant", "content": None, "tool_calls": iter(tool_calls)}, + ] + + req = _make_request(messages) + assistant_msg = req.messages[1] + assert isinstance(assistant_msg, dict) + + result_tool_calls = assistant_msg.get("tool_calls") + assert isinstance(result_tool_calls, list) + assert len(result_tool_calls) == num_tool_calls + + # Verify after model_dump_json too + _ = req.model_dump_json() + assert len(assistant_msg.get("tool_calls", [])) == num_tool_calls diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index c1324d2f03..3710473560 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -290,7 +290,7 @@ class CustomChatCompletionMessageParam(TypedDict, total=False): tool_call_id: str | None """Tool call that this message is responding to.""" - tool_calls: Iterable[ChatCompletionMessageToolCallParam] | None + tool_calls: list[ChatCompletionMessageToolCallParam] | None """The tool calls generated by the model, such as function calls.""" reasoning: str | None @@ -321,7 +321,7 @@ class ConversationMessage(TypedDict, total=False): name: str | None """The name of the function to call""" - tool_calls: Iterable[ChatCompletionMessageToolCallParam] | None + tool_calls: list[ChatCompletionMessageToolCallParam] | None """The tool calls generated by the model, such as function calls.""" reasoning: str | None diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 533959df60..2bc1b6e087 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -357,6 +357,47 @@ class ChatCompletionRequest(OpenAIBaseModel): # --8<-- [end:chat-completion-extra-params] + @model_validator(mode="before") + @classmethod + def _materialize_tool_calls_before(cls, data: Any) -> Any: + """Eagerly convert tool_calls generators/iterators to lists. + + Must run before Pydantic field validation so that one-shot + generators are not consumed during union type matching of + ChatCompletionAssistantMessageParam (which types tool_calls + as Iterable[...]). + """ + if not isinstance(data, dict): + return data + messages = data.get("messages") + if not isinstance(messages, list): + return data + for msg in messages: + if not isinstance(msg, dict): + continue + tool_calls = msg.get("tool_calls") + if tool_calls is not None and not isinstance(tool_calls, list): + msg["tool_calls"] = list(tool_calls) + return data + + @model_validator(mode="after") + def _materialize_tool_calls_after(self) -> "ChatCompletionRequest": + """Convert Pydantic ValidatorIterator wrappers back to lists. + + Even after the "before" validator converts iterables to lists, + Pydantic re-wraps them in a ValidatorIterator when validating + against ChatCompletionAssistantMessageParam's Iterable[...] type. + This "after" pass materialises those wrappers so downstream code + (tokenizers, model_dump_json) always sees plain lists. + """ + for msg in self.messages: + if not isinstance(msg, dict): + continue + tool_calls = msg.get("tool_calls") + if tool_calls is not None and not isinstance(tool_calls, list): + msg["tool_calls"] = list(tool_calls) + return self + def build_chat_params( self, default_template: str | None,