[BugFix] Support custom tool parsers when tool_choice is required and named function (#39870)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -557,6 +557,20 @@ class OpenAIServingChat(OpenAIServing):
|
||||
and self._should_stream_with_auto_tool_parsing(request)
|
||||
)
|
||||
|
||||
# Determine whether required/named tool_choice should fall back to
|
||||
# the auto tool_parser path instead of the standard JSON-based parsing.
|
||||
# This happens when the parser declares supports_required_and_named=False
|
||||
# (e.g. GLM models that output XML instead of JSON).
|
||||
tool_choice_uses_parser = (
|
||||
self.tool_parser is not None
|
||||
and not self.tool_parser.supports_required_and_named
|
||||
and request.tools
|
||||
and (
|
||||
request.tool_choice == "required"
|
||||
or isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
|
||||
)
|
||||
)
|
||||
|
||||
all_previous_token_ids: list[list[int]] | None
|
||||
function_name_returned = [False] * num_choices
|
||||
if self.tool_call_id_type == "kimi_k2":
|
||||
@@ -569,7 +583,12 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
# Only one of these will be used, thus previous_texts and
|
||||
# all_previous_token_ids will not be used twice in the same iteration.
|
||||
if is_mistral_grammar_path or tool_choice_auto or reasoning_parser:
|
||||
if (
|
||||
is_mistral_grammar_path
|
||||
or tool_choice_auto
|
||||
or tool_choice_uses_parser
|
||||
or reasoning_parser
|
||||
):
|
||||
# These are only required in "auto" tool choice case
|
||||
all_previous_token_ids = [[] for _ in range(num_choices)]
|
||||
reasoning_end_arr = [False] * num_choices
|
||||
@@ -764,7 +783,12 @@ class OpenAIServingChat(OpenAIServing):
|
||||
delta_message: DeltaMessage | None
|
||||
|
||||
# just update previous_texts and previous_token_ids
|
||||
if is_mistral_grammar_path or tool_choice_auto or reasoning_parser:
|
||||
if (
|
||||
is_mistral_grammar_path
|
||||
or tool_choice_auto
|
||||
or tool_choice_uses_parser
|
||||
or reasoning_parser
|
||||
):
|
||||
assert previous_texts is not None
|
||||
assert all_previous_token_ids is not None
|
||||
previous_text = previous_texts[i]
|
||||
@@ -813,7 +837,9 @@ class OpenAIServingChat(OpenAIServing):
|
||||
if result.tools_called:
|
||||
tools_streamed[i] = True
|
||||
# handle streaming deltas for tools with named tool_choice
|
||||
elif tool_choice_function_name:
|
||||
# Skip when tool_choice_uses_parser so it falls through
|
||||
# to the auto tool_parser branches below.
|
||||
elif tool_choice_function_name and not tool_choice_uses_parser:
|
||||
# When encountering think end id in prompt_token_ids
|
||||
# i.e {"enable_thinking": False},
|
||||
# check BEFORE calling the parser to avoid a spurious
|
||||
@@ -851,7 +877,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
):
|
||||
reasoning_end_arr[i] = True
|
||||
if delta_message and delta_message.content:
|
||||
# This need to be added to next `delta_text`
|
||||
current_text = delta_message.content
|
||||
delta_message.content = None
|
||||
else:
|
||||
@@ -896,7 +921,12 @@ class OpenAIServingChat(OpenAIServing):
|
||||
)
|
||||
tools_streamed[i] = True
|
||||
|
||||
elif request.tool_choice == "required":
|
||||
# Skip when tool_choice_uses_parser so it falls through
|
||||
# to the auto tool_parser branches below.
|
||||
elif (
|
||||
request.tool_choice == "required"
|
||||
and not tool_choice_uses_parser
|
||||
):
|
||||
assert previous_texts is not None
|
||||
previous_text = previous_texts[i]
|
||||
current_text = previous_text + delta_text
|
||||
@@ -966,7 +996,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
# update the previous values for the next iteration
|
||||
if (
|
||||
is_mistral_grammar_path or tool_choice_auto or reasoning_parser
|
||||
is_mistral_grammar_path
|
||||
or tool_choice_auto
|
||||
or tool_choice_uses_parser
|
||||
or reasoning_parser
|
||||
) and not self.use_harmony:
|
||||
assert previous_texts is not None
|
||||
assert all_previous_token_ids is not None
|
||||
|
||||
@@ -627,7 +627,7 @@ class OpenAIServing:
|
||||
and isinstance(request.tool_choice, ToolChoiceFunction)
|
||||
):
|
||||
assert content is not None
|
||||
# Forced Function Call
|
||||
# Forced Function Call (Responses API)
|
||||
function_calls.append(
|
||||
FunctionCall(name=request.tool_choice.name, arguments=content)
|
||||
)
|
||||
@@ -636,14 +636,20 @@ class OpenAIServing:
|
||||
not use_mistral_tool_parser
|
||||
and request.tool_choice
|
||||
and isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
|
||||
and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
|
||||
):
|
||||
# Named function with standard JSON-based parsing
|
||||
assert content is not None
|
||||
# Forced Function Call
|
||||
function_calls.append(
|
||||
FunctionCall(name=request.tool_choice.function.name, arguments=content)
|
||||
)
|
||||
content = None # Clear content since tool is called.
|
||||
elif not use_mistral_tool_parser and request.tool_choice == "required":
|
||||
elif (
|
||||
not use_mistral_tool_parser
|
||||
and request.tool_choice == "required"
|
||||
and (tool_parser_cls is None or tool_parser_cls.supports_required_and_named)
|
||||
):
|
||||
# "required" with standard JSON-based parsing
|
||||
tool_calls = []
|
||||
with contextlib.suppress(ValidationError):
|
||||
content = content or ""
|
||||
@@ -662,15 +668,30 @@ class OpenAIServing:
|
||||
use_mistral_tool_parser
|
||||
or (
|
||||
enable_auto_tools
|
||||
and (request.tool_choice == "auto" or request.tool_choice is None)
|
||||
and (
|
||||
request.tool_choice == "auto"
|
||||
or request.tool_choice is None
|
||||
or (
|
||||
not tool_parser_cls.supports_required_and_named
|
||||
and request.tools
|
||||
and (
|
||||
request.tool_choice == "required"
|
||||
or isinstance(
|
||||
request.tool_choice,
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
):
|
||||
# Automatic Tool Call Parsing (also used as fallback for
|
||||
# required/named when supports_required_and_named=False)
|
||||
if tokenizer is None:
|
||||
raise ValueError(
|
||||
"Tokenizer not available when `skip_tokenizer_init=True`"
|
||||
)
|
||||
|
||||
# Automatic Tool Call Parsing
|
||||
try:
|
||||
tool_parser = tool_parser_cls(tokenizer, request.tools)
|
||||
except RuntimeError as e:
|
||||
|
||||
Reference in New Issue
Block a user