[Bugfix][Frontend] Honor parallel_tool_calls=false in the Responses API (#59298)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -1359,6 +1359,7 @@ def _make_simple_context_with_output(
|
||||
def _make_serving_instance(
|
||||
*,
|
||||
reasoning_parser: str = "",
|
||||
tool_parser: str | None = None,
|
||||
enable_per_request_metrics: bool = False,
|
||||
) -> OpenAIServingResponses:
|
||||
engine_client = MagicMock()
|
||||
@@ -1379,6 +1380,8 @@ def _make_serving_instance(
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
reasoning_parser=reasoning_parser,
|
||||
enable_auto_tools=tool_parser is not None,
|
||||
tool_parser=tool_parser,
|
||||
enable_per_request_metrics=enable_per_request_metrics,
|
||||
)
|
||||
|
||||
@@ -1977,3 +1980,56 @@ class TestAutoToolStreaming:
|
||||
assert len(function_done) == 1
|
||||
assert function_done[0].item.name == "get_weather"
|
||||
assert function_done[0].item.arguments == tool_args
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)])
|
||||
async def test_streaming_parallel_tool_calls(
|
||||
monkeypatch, parallel_tool_calls, num_calls
|
||||
):
|
||||
"""parallel_tool_calls=False keeps only the first call even when the grammar
|
||||
does not limit the model; None keeps every call."""
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
serving = _make_serving_instance(tool_parser="hermes")
|
||||
request = ResponsesRequest(
|
||||
input="hi",
|
||||
tools=[{"type": "function", "name": "get_weather", "parameters": {}}],
|
||||
parallel_tool_calls=parallel_tool_calls,
|
||||
stream=True,
|
||||
)
|
||||
context = SimpleContext(
|
||||
response_parser=serving._make_response_parser(request, MagicMock(), {})
|
||||
)
|
||||
args = [f'{{"city": "{city}"}}' for city in ("Paris", "Berlin", "Tokyo")]
|
||||
|
||||
async def result_generator():
|
||||
for i, arg in enumerate(args):
|
||||
text = f'<tool_call>\n{{"name": "get_weather", "arguments": {arg}}}'
|
||||
context.append_output(_make_request_output(f"{text}\n</tool_call>\n", [i]))
|
||||
yield context
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in serving.responses_stream_generator(
|
||||
request=request,
|
||||
sampling_params=SamplingParams(max_tokens=64),
|
||||
result_generator=result_generator(),
|
||||
context=context,
|
||||
model_name="test-model",
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="req"),
|
||||
)
|
||||
]
|
||||
|
||||
assert [
|
||||
event.output_index
|
||||
for event in events
|
||||
if event.type == "response.output_item.added"
|
||||
] == list(range(num_calls))
|
||||
assert all(getattr(event, "output_index", 0) < num_calls for event in events)
|
||||
assert [
|
||||
event.item.arguments
|
||||
for event in events
|
||||
if event.type == "response.output_item.done"
|
||||
] == args[:num_calls]
|
||||
assert [item.arguments for item in events[-1].response.output] == args[:num_calls]
|
||||
|
||||
@@ -37,6 +37,9 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
build_response_output_items,
|
||||
construct_tool_dicts,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||
maybe_filter_parallel_tool_calls,
|
||||
)
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -369,7 +372,9 @@ class ParsableContext(ConversationContext):
|
||||
build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
tool_calls=maybe_filter_parallel_tool_calls(
|
||||
tool_calls or [], self.request
|
||||
),
|
||||
tools=self.request.tools,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -73,6 +73,9 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||
maybe_filter_parallel_tool_calls,
|
||||
)
|
||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
@@ -1028,7 +1031,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
return build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request),
|
||||
logprobs=logprobs,
|
||||
tools=request.tools,
|
||||
)
|
||||
@@ -1231,6 +1234,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
|
||||
if not delta_message:
|
||||
continue
|
||||
delta_message = maybe_filter_parallel_tool_calls(delta_message, request)
|
||||
|
||||
for dm in split_delta(delta_message):
|
||||
target_state, tool_call = processor.resolve_target_state(dm)
|
||||
|
||||
@@ -2,35 +2,42 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TypeVar
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponseChoice,
|
||||
ChatCompletionResponseStreamChoice,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
# Used internally
|
||||
_ChatCompletionResponseChoiceT = TypeVar(
|
||||
"_ChatCompletionResponseChoiceT",
|
||||
_ToolCallsContainerT = TypeVar(
|
||||
"_ToolCallsContainerT",
|
||||
ChatCompletionResponseChoice,
|
||||
ChatCompletionResponseStreamChoice,
|
||||
DeltaMessage,
|
||||
list[FunctionCall],
|
||||
)
|
||||
|
||||
|
||||
def maybe_filter_parallel_tool_calls(
|
||||
choice: _ChatCompletionResponseChoiceT, request: ChatCompletionRequest
|
||||
) -> _ChatCompletionResponseChoiceT:
|
||||
item: _ToolCallsContainerT,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> _ToolCallsContainerT:
|
||||
"""Filter to first tool call only when parallel_tool_calls is explicitly False."""
|
||||
if request.parallel_tool_calls is not False:
|
||||
return choice
|
||||
return item
|
||||
|
||||
if isinstance(choice, ChatCompletionResponseChoice) and choice.message.tool_calls:
|
||||
choice.message.tool_calls = choice.message.tool_calls[:1]
|
||||
elif (
|
||||
isinstance(choice, ChatCompletionResponseStreamChoice)
|
||||
and choice.delta.tool_calls
|
||||
):
|
||||
choice.delta.tool_calls = [
|
||||
tool_call for tool_call in choice.delta.tool_calls if tool_call.index == 0
|
||||
if isinstance(item, list):
|
||||
return item[:1]
|
||||
if isinstance(item, ChatCompletionResponseChoice):
|
||||
if item.message.tool_calls:
|
||||
item.message.tool_calls = item.message.tool_calls[:1]
|
||||
return item
|
||||
|
||||
delta = item.delta if isinstance(item, ChatCompletionResponseStreamChoice) else item
|
||||
if delta.tool_calls:
|
||||
delta.tool_calls = [
|
||||
tool_call for tool_call in delta.tool_calls if tool_call.index == 0
|
||||
]
|
||||
|
||||
return choice
|
||||
return item
|
||||
|
||||
Reference in New Issue
Block a user