diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 89c188ec8a..c06e2b5915 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -1359,6 +1359,7 @@ def _make_simple_context_with_output( def _make_serving_instance( *, reasoning_parser: str = "", + tool_parser: str | None = None, enable_per_request_metrics: bool = False, ) -> OpenAIServingResponses: engine_client = MagicMock() @@ -1379,6 +1380,8 @@ def _make_serving_instance( chat_template=None, chat_template_content_format="auto", reasoning_parser=reasoning_parser, + enable_auto_tools=tool_parser is not None, + tool_parser=tool_parser, enable_per_request_metrics=enable_per_request_metrics, ) @@ -1977,3 +1980,56 @@ class TestAutoToolStreaming: assert len(function_done) == 1 assert function_done[0].item.name == "get_weather" assert function_done[0].item.arguments == tool_args + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)]) +async def test_streaming_parallel_tool_calls( + monkeypatch, parallel_tool_calls, num_calls +): + """parallel_tool_calls=False keeps only the first call even when the grammar + does not limit the model; None keeps every call.""" + monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) + serving = _make_serving_instance(tool_parser="hermes") + request = ResponsesRequest( + input="hi", + tools=[{"type": "function", "name": "get_weather", "parameters": {}}], + parallel_tool_calls=parallel_tool_calls, + stream=True, + ) + context = SimpleContext( + response_parser=serving._make_response_parser(request, MagicMock(), {}) + ) + args = [f'{{"city": "{city}"}}' for city in ("Paris", "Berlin", "Tokyo")] + + async def result_generator(): + for i, arg in enumerate(args): + text = f'\n{{"name": "get_weather", "arguments": {arg}}}' + context.append_output(_make_request_output(f"{text}\n\n", [i])) + yield context + + events = [ + event + async for event in serving.responses_stream_generator( + request=request, + sampling_params=SamplingParams(max_tokens=64), + result_generator=result_generator(), + context=context, + model_name="test-model", + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="req"), + ) + ] + + assert [ + event.output_index + for event in events + if event.type == "response.output_item.added" + ] == list(range(num_calls)) + assert all(getattr(event, "output_index", 0) < num_calls for event in events) + assert [ + event.item.arguments + for event in events + if event.type == "response.output_item.done" + ] == args[:num_calls] + assert [item.arguments for item in events[-1].response.output] == args[:num_calls] diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index fa233dcce1..726a955a55 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -37,6 +37,9 @@ from vllm.entrypoints.openai.responses.utils import ( build_response_output_items, construct_tool_dicts, ) +from vllm.entrypoints.serve.utils.tool_calls_utils import ( + maybe_filter_parallel_tool_calls, +) from vllm.outputs import RequestOutput from vllm.parser.abstract_parser import Parser from vllm.tokenizers import TokenizerLike @@ -369,7 +372,9 @@ class ParsableContext(ConversationContext): build_response_output_items( reasoning=reasoning, content=content, - tool_calls=tool_calls, + tool_calls=maybe_filter_parallel_tool_calls( + tool_calls or [], self.request + ), tools=self.request.tools, ) ) diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index f18c493259..4facd2fb3b 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -73,6 +73,9 @@ from vllm.entrypoints.openai.responses.utils import ( from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.utils.api_utils import get_max_tokens from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.entrypoints.serve.utils.tool_calls_utils import ( + maybe_filter_parallel_tool_calls, +) from vllm.exceptions import GenerationError, VLLMValidationError from vllm.inputs import EngineInput from vllm.logger import init_logger @@ -1028,7 +1031,7 @@ class OpenAIServingResponses(GenerateBaseServing): return build_response_output_items( reasoning=reasoning, content=content, - tool_calls=tool_calls, + tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request), logprobs=logprobs, tools=request.tools, ) @@ -1231,6 +1234,7 @@ class OpenAIServingResponses(GenerateBaseServing): if not delta_message: continue + delta_message = maybe_filter_parallel_tool_calls(delta_message, request) for dm in split_delta(delta_message): target_state, tool_call = processor.resolve_target_state(dm) diff --git a/vllm/entrypoints/serve/utils/tool_calls_utils.py b/vllm/entrypoints/serve/utils/tool_calls_utils.py index ae2fa24f7f..1252218127 100644 --- a/vllm/entrypoints/serve/utils/tool_calls_utils.py +++ b/vllm/entrypoints/serve/utils/tool_calls_utils.py @@ -2,35 +2,42 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import TypeVar +from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, ) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest # Used internally -_ChatCompletionResponseChoiceT = TypeVar( - "_ChatCompletionResponseChoiceT", +_ToolCallsContainerT = TypeVar( + "_ToolCallsContainerT", ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, + DeltaMessage, + list[FunctionCall], ) def maybe_filter_parallel_tool_calls( - choice: _ChatCompletionResponseChoiceT, request: ChatCompletionRequest -) -> _ChatCompletionResponseChoiceT: + item: _ToolCallsContainerT, + request: ChatCompletionRequest | ResponsesRequest, +) -> _ToolCallsContainerT: """Filter to first tool call only when parallel_tool_calls is explicitly False.""" if request.parallel_tool_calls is not False: - return choice + return item - if isinstance(choice, ChatCompletionResponseChoice) and choice.message.tool_calls: - choice.message.tool_calls = choice.message.tool_calls[:1] - elif ( - isinstance(choice, ChatCompletionResponseStreamChoice) - and choice.delta.tool_calls - ): - choice.delta.tool_calls = [ - tool_call for tool_call in choice.delta.tool_calls if tool_call.index == 0 + if isinstance(item, list): + return item[:1] + if isinstance(item, ChatCompletionResponseChoice): + if item.message.tool_calls: + item.message.tool_calls = item.message.tool_calls[:1] + return item + + delta = item.delta if isinstance(item, ChatCompletionResponseStreamChoice) else item + if delta.tool_calls: + delta.tool_calls = [ + tool_call for tool_call in delta.tool_calls if tool_call.index == 0 ] - - return choice + return item