[Bugfix] Fix crash when tool_choice=required exceeds max_tokens (#36841)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-03-12 03:28:45 -07:00
committed by GitHub
parent f0d3658c0f
commit 5a71cdd76e
3 changed files with 36 additions and 9 deletions
@@ -514,3 +514,27 @@ async def test_inconsistent_tool_choice_and_tools(
], ],
tool_choice={}, tool_choice={},
) )
@pytest.mark.asyncio
async def test_max_tokens_with_tool_choice_required(client: openai.AsyncOpenAI):
""" """
models = await client.models.list()
model_name: str = models.data[0].id
# This combination previously crashed the engine
chat_completion = await client.chat.completions.create(
messages=messages,
temperature=0,
max_completion_tokens=1,
model=model_name,
tools=tools,
tool_choice="required",
)
# When `tool_choice="required"` and the tokens of `tools` exceed `max_tokens`,
# both `tool_calls` and `content` should be empty.
# This behavior should be consistent with OpenAI.
choice = chat_completion.choices[0]
assert choice.finish_reason == "length"
assert len(choice.message.tool_calls) == 0
assert choice.message.content == ""
@@ -1507,7 +1507,7 @@ class OpenAIServingChat(OpenAIServing):
elif request.tool_choice and request.tool_choice == "required": elif request.tool_choice and request.tool_choice == "required":
tool_call_class_items = [] tool_call_class_items = []
assert tool_calls is not None and len(tool_calls) > 0 tool_calls = tool_calls or []
for idx, tool_call in enumerate(tool_calls): for idx, tool_call in enumerate(tool_calls):
# Use native ID if available, # Use native ID if available,
# otherwise generate ID with correct id_type # otherwise generate ID with correct id_type
+11 -8
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio import asyncio
import contextlib
import json import json
import time import time
from collections.abc import AsyncGenerator, Callable, Mapping, Sequence from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
@@ -13,7 +14,7 @@ from fastapi import Request
from openai.types.responses import ( from openai.types.responses import (
ToolChoiceFunction, ToolChoiceFunction,
) )
from pydantic import ConfigDict, TypeAdapter from pydantic import ConfigDict, TypeAdapter, ValidationError
from starlette.datastructures import Headers from starlette.datastructures import Headers
import vllm.envs as envs import vllm.envs as envs
@@ -1125,17 +1126,19 @@ class OpenAIServing:
) )
content = None # Clear content since tool is called. content = None # Clear content since tool is called.
elif request.tool_choice == "required": elif request.tool_choice == "required":
assert content is not None tool_calls = []
tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(content) with contextlib.suppress(ValidationError):
function_calls.extend( content = content or ""
[ tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
content
)
for tool_call in tool_calls:
function_calls.append(
FunctionCall( FunctionCall(
name=tool_call.name, name=tool_call.name,
arguments=json.dumps(tool_call.parameters, ensure_ascii=False), arguments=json.dumps(tool_call.parameters, ensure_ascii=False),
) )
for tool_call in tool_calls )
]
)
content = None # Clear content since tool is called. content = None # Clear content since tool is called.
elif ( elif (
tool_parser_cls tool_parser_cls