[Bugfix][Frontend] Honor parallel_tool_calls=false in the Responses API (#59298)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-09-29 22:36:11 +00:00
committed by GitHub
parent a7a9ac2745
commit 797505747f
4 changed files with 89 additions and 17 deletions
@@ -1359,6 +1359,7 @@ def _make_simple_context_with_output(
def _make_serving_instance(
*,
reasoning_parser: str = "",
tool_parser: str | None = None,
enable_per_request_metrics: bool = False,
) -> OpenAIServingResponses:
engine_client = MagicMock()
@@ -1379,6 +1380,8 @@ def _make_serving_instance(
chat_template=None,
chat_template_content_format="auto",
reasoning_parser=reasoning_parser,
enable_auto_tools=tool_parser is not None,
tool_parser=tool_parser,
enable_per_request_metrics=enable_per_request_metrics,
)
@@ -1977,3 +1980,56 @@ class TestAutoToolStreaming:
assert len(function_done) == 1
assert function_done[0].item.name == "get_weather"
assert function_done[0].item.arguments == tool_args
@pytest.mark.asyncio
@pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)])
async def test_streaming_parallel_tool_calls(
monkeypatch, parallel_tool_calls, num_calls
):
"""parallel_tool_calls=False keeps only the first call even when the grammar
does not limit the model; None keeps every call."""
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
serving = _make_serving_instance(tool_parser="hermes")
request = ResponsesRequest(
input="hi",
tools=[{"type": "function", "name": "get_weather", "parameters": {}}],
parallel_tool_calls=parallel_tool_calls,
stream=True,
)
context = SimpleContext(
response_parser=serving._make_response_parser(request, MagicMock(), {})
)
args = [f'{{"city": "{city}"}}' for city in ("Paris", "Berlin", "Tokyo")]
async def result_generator():
for i, arg in enumerate(args):
text = f'<tool_call>\n{{"name": "get_weather", "arguments": {arg}}}'
context.append_output(_make_request_output(f"{text}\n</tool_call>\n", [i]))
yield context
events = [
event
async for event in serving.responses_stream_generator(
request=request,
sampling_params=SamplingParams(max_tokens=64),
result_generator=result_generator(),
context=context,
model_name="test-model",
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="req"),
)
]
assert [
event.output_index
for event in events
if event.type == "response.output_item.added"
] == list(range(num_calls))
assert all(getattr(event, "output_index", 0) < num_calls for event in events)
assert [
event.item.arguments
for event in events
if event.type == "response.output_item.done"
] == args[:num_calls]
assert [item.arguments for item in events[-1].response.output] == args[:num_calls]
+6 -1
View File
@@ -37,6 +37,9 @@ from vllm.entrypoints.openai.responses.utils import (
build_response_output_items,
construct_tool_dicts,
)
from vllm.entrypoints.serve.utils.tool_calls_utils import (
maybe_filter_parallel_tool_calls,
)
from vllm.outputs import RequestOutput
from vllm.parser.abstract_parser import Parser
from vllm.tokenizers import TokenizerLike
@@ -369,7 +372,9 @@ class ParsableContext(ConversationContext):
build_response_output_items(
reasoning=reasoning,
content=content,
tool_calls=tool_calls,
tool_calls=maybe_filter_parallel_tool_calls(
tool_calls or [], self.request
),
tools=self.request.tools,
)
)
+5 -1
View File
@@ -73,6 +73,9 @@ from vllm.entrypoints.openai.responses.utils import (
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.entrypoints.serve.utils.tool_calls_utils import (
maybe_filter_parallel_tool_calls,
)
from vllm.exceptions import GenerationError, VLLMValidationError
from vllm.inputs import EngineInput
from vllm.logger import init_logger
@@ -1028,7 +1031,7 @@ class OpenAIServingResponses(GenerateBaseServing):
return build_response_output_items(
reasoning=reasoning,
content=content,
tool_calls=tool_calls,
tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request),
logprobs=logprobs,
tools=request.tools,
)
@@ -1231,6 +1234,7 @@ class OpenAIServingResponses(GenerateBaseServing):
if not delta_message:
continue
delta_message = maybe_filter_parallel_tool_calls(delta_message, request)
for dm in split_delta(delta_message):
target_state, tool_call = processor.resolve_target_state(dm)
@@ -2,35 +2,42 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TypeVar
from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponseChoice,
ChatCompletionResponseStreamChoice,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
# Used internally
_ChatCompletionResponseChoiceT = TypeVar(
"_ChatCompletionResponseChoiceT",
_ToolCallsContainerT = TypeVar(
"_ToolCallsContainerT",
ChatCompletionResponseChoice,
ChatCompletionResponseStreamChoice,
DeltaMessage,
list[FunctionCall],
)
def maybe_filter_parallel_tool_calls(
choice: _ChatCompletionResponseChoiceT, request: ChatCompletionRequest
) -> _ChatCompletionResponseChoiceT:
item: _ToolCallsContainerT,
request: ChatCompletionRequest | ResponsesRequest,
) -> _ToolCallsContainerT:
"""Filter to first tool call only when parallel_tool_calls is explicitly False."""
if request.parallel_tool_calls is not False:
return choice
return item
if isinstance(choice, ChatCompletionResponseChoice) and choice.message.tool_calls:
choice.message.tool_calls = choice.message.tool_calls[:1]
elif (
isinstance(choice, ChatCompletionResponseStreamChoice)
and choice.delta.tool_calls
):
choice.delta.tool_calls = [
tool_call for tool_call in choice.delta.tool_calls if tool_call.index == 0
if isinstance(item, list):
return item[:1]
if isinstance(item, ChatCompletionResponseChoice):
if item.message.tool_calls:
item.message.tool_calls = item.message.tool_calls[:1]
return item
delta = item.delta if isinstance(item, ChatCompletionResponseStreamChoice) else item
if delta.tool_calls:
delta.tool_calls = [
tool_call for tool_call in delta.tool_calls if tool_call.index == 0
]
return choice
return item