[BugFix] Support custom tool parsers when tool_choice is required and named function (#39870)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Jared Wen
2026-04-17 16:38:10 +00:00
committed by GitHub
co-authored by sfeng33
parent 747256bb5d
commit ceade1952c
5 changed files with 100 additions and 12 deletions
@@ -557,6 +557,20 @@ class OpenAIServingChat(OpenAIServing):
and self._should_stream_with_auto_tool_parsing(request)
)
# Determine whether required/named tool_choice should fall back to
# the auto tool_parser path instead of the standard JSON-based parsing.
# This happens when the parser declares supports_required_and_named=False
# (e.g. GLM models that output XML instead of JSON).
tool_choice_uses_parser = (
self.tool_parser is not None
and not self.tool_parser.supports_required_and_named
and request.tools
and (
request.tool_choice == "required"
or isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
)
)
all_previous_token_ids: list[list[int]] | None
function_name_returned = [False] * num_choices
if self.tool_call_id_type == "kimi_k2":
@@ -569,7 +583,12 @@ class OpenAIServingChat(OpenAIServing):
# Only one of these will be used, thus previous_texts and
# all_previous_token_ids will not be used twice in the same iteration.
if is_mistral_grammar_path or tool_choice_auto or reasoning_parser:
if (
is_mistral_grammar_path
or tool_choice_auto
or tool_choice_uses_parser
or reasoning_parser
):
# These are only required in "auto" tool choice case
all_previous_token_ids = [[] for _ in range(num_choices)]
reasoning_end_arr = [False] * num_choices
@@ -764,7 +783,12 @@ class OpenAIServingChat(OpenAIServing):
delta_message: DeltaMessage | None
# just update previous_texts and previous_token_ids
if is_mistral_grammar_path or tool_choice_auto or reasoning_parser:
if (
is_mistral_grammar_path
or tool_choice_auto
or tool_choice_uses_parser
or reasoning_parser
):
assert previous_texts is not None
assert all_previous_token_ids is not None
previous_text = previous_texts[i]
@@ -813,7 +837,9 @@ class OpenAIServingChat(OpenAIServing):
if result.tools_called:
tools_streamed[i] = True
# handle streaming deltas for tools with named tool_choice
elif tool_choice_function_name:
# Skip when tool_choice_uses_parser so it falls through
# to the auto tool_parser branches below.
elif tool_choice_function_name and not tool_choice_uses_parser:
# When encountering think end id in prompt_token_ids
# i.e {"enable_thinking": False},
# check BEFORE calling the parser to avoid a spurious
@@ -851,7 +877,6 @@ class OpenAIServingChat(OpenAIServing):
):
reasoning_end_arr[i] = True
if delta_message and delta_message.content:
# This need to be added to next `delta_text`
current_text = delta_message.content
delta_message.content = None
else:
@@ -896,7 +921,12 @@ class OpenAIServingChat(OpenAIServing):
)
tools_streamed[i] = True
elif request.tool_choice == "required":
# Skip when tool_choice_uses_parser so it falls through
# to the auto tool_parser branches below.
elif (
request.tool_choice == "required"
and not tool_choice_uses_parser
):
assert previous_texts is not None
previous_text = previous_texts[i]
current_text = previous_text + delta_text
@@ -966,7 +996,10 @@ class OpenAIServingChat(OpenAIServing):
# update the previous values for the next iteration
if (
is_mistral_grammar_path or tool_choice_auto or reasoning_parser
is_mistral_grammar_path
or tool_choice_auto
or tool_choice_uses_parser
or reasoning_parser
) and not self.use_harmony:
assert previous_texts is not None
assert all_previous_token_ids is not None
+26 -5
View File
@@ -627,7 +627,7 @@ class OpenAIServing:
and isinstance(request.tool_choice, ToolChoiceFunction)
):
assert content is not None
# Forced Function Call
# Forced Function Call (Responses API)
function_calls.append(
FunctionCall(name=request.tool_choice.name, arguments=content)
)
@@ -636,14 +636,20 @@ class OpenAIServing:
not use_mistral_tool_parser
and request.tool_choice
and isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
):
# Named function with standard JSON-based parsing
assert content is not None
# Forced Function Call
function_calls.append(
FunctionCall(name=request.tool_choice.function.name, arguments=content)
)
content = None # Clear content since tool is called.
elif not use_mistral_tool_parser and request.tool_choice == "required":
elif (
not use_mistral_tool_parser
and request.tool_choice == "required"
and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
):
# "required" with standard JSON-based parsing
tool_calls = []
with contextlib.suppress(ValidationError):
content = content or ""
@@ -662,15 +668,30 @@ class OpenAIServing:
use_mistral_tool_parser
or (
enable_auto_tools
and (request.tool_choice == "auto" or request.tool_choice is None)
and (
request.tool_choice == "auto"
or request.tool_choice is None
or (
not tool_parser_cls.supports_required_and_named
and request.tools
and (
request.tool_choice == "required"
or isinstance(
request.tool_choice,
ChatCompletionNamedToolChoiceParam,
)
)
)
)
)
):
# Automatic Tool Call Parsing (also used as fallback for
# required/named when supports_required_and_named=False)
if tokenizer is None:
raise ValueError(
"Tokenizer not available when `skip_tokenizer_init=True`"
)
# Automatic Tool Call Parsing
try:
tool_parser = tool_parser_cls(tokenizer, request.tools)
except RuntimeError as e: