[Refactor] Remove dead states from chat completion serving (#45081)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -376,7 +376,6 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
conversation,
|
conversation,
|
||||||
tokenizer,
|
tokenizer,
|
||||||
request_metadata,
|
request_metadata,
|
||||||
reasoning_parser,
|
|
||||||
chat_template_kwargs=chat_template_kwargs,
|
chat_template_kwargs=chat_template_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -405,7 +404,6 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
conversation: list[ConversationMessage],
|
conversation: list[ConversationMessage],
|
||||||
tokenizer: TokenizerLike,
|
tokenizer: TokenizerLike,
|
||||||
request_metadata: RequestResponseMetadata,
|
request_metadata: RequestResponseMetadata,
|
||||||
reasoning_parser: ReasoningParser | None = None,
|
|
||||||
chat_template_kwargs: dict[str, Any] | None = None,
|
chat_template_kwargs: dict[str, Any] | None = None,
|
||||||
) -> AsyncGenerator[str, None]:
|
) -> AsyncGenerator[str, None]:
|
||||||
created_time = int(time.time())
|
created_time = int(time.time())
|
||||||
@@ -430,33 +428,13 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
else:
|
else:
|
||||||
tool_choice_function_name = None
|
tool_choice_function_name = None
|
||||||
|
|
||||||
# Determine whether tools are in use with "auto" tool choice
|
|
||||||
tool_choice_auto = (
|
|
||||||
not tool_choice_function_name
|
|
||||||
and self._should_stream_with_auto_tool_parsing(request)
|
|
||||||
)
|
|
||||||
|
|
||||||
all_previous_token_ids: list[list[int]] | None
|
|
||||||
if self.tool_call_id_type == "kimi_k2":
|
if self.tool_call_id_type == "kimi_k2":
|
||||||
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
|
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
|
||||||
else:
|
else:
|
||||||
history_tool_call_cnt = 0
|
history_tool_call_cnt = 0
|
||||||
|
|
||||||
# Always track previous_texts for comprehensive output logging
|
|
||||||
previous_texts = [""] * num_choices
|
previous_texts = [""] * num_choices
|
||||||
|
|
||||||
# Only one of these will be used, thus previous_texts and
|
|
||||||
# all_previous_token_ids will not be used twice in the same iteration.
|
|
||||||
if (
|
|
||||||
tool_choice_auto
|
|
||||||
or tool_choice_function_name
|
|
||||||
or request.tool_choice == "required"
|
|
||||||
or reasoning_parser
|
|
||||||
):
|
|
||||||
all_previous_token_ids = [[] for _ in range(num_choices)]
|
|
||||||
else:
|
|
||||||
all_previous_token_ids = None
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if self.parser_cls is not None:
|
if self.parser_cls is not None:
|
||||||
if tokenizer is None:
|
if tokenizer is None:
|
||||||
@@ -639,26 +617,6 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
|
|
||||||
delta_message: DeltaMessage | None
|
delta_message: DeltaMessage | None
|
||||||
|
|
||||||
# just update previous_texts and previous_token_ids
|
|
||||||
if (
|
|
||||||
tool_choice_auto
|
|
||||||
or tool_choice_function_name
|
|
||||||
or request.tool_choice == "required"
|
|
||||||
or reasoning_parser
|
|
||||||
):
|
|
||||||
assert previous_texts is not None
|
|
||||||
assert all_previous_token_ids is not None
|
|
||||||
previous_text = previous_texts[i]
|
|
||||||
previous_token_ids = all_previous_token_ids[i]
|
|
||||||
current_text = previous_text + delta_text
|
|
||||||
# avoid the None + list error.
|
|
||||||
if previous_token_ids:
|
|
||||||
current_token_ids = previous_token_ids + as_list(
|
|
||||||
output.token_ids
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
current_token_ids = as_list(output.token_ids)
|
|
||||||
|
|
||||||
if self.use_harmony:
|
if self.use_harmony:
|
||||||
delta_message, tools_streamed_flag = (
|
delta_message, tools_streamed_flag = (
|
||||||
extract_harmony_streaming_delta(
|
extract_harmony_streaming_delta(
|
||||||
@@ -683,21 +641,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
else:
|
else:
|
||||||
delta_message = DeltaMessage(content=delta_text)
|
delta_message = DeltaMessage(content=delta_text)
|
||||||
|
|
||||||
# update the previous values for the next iteration
|
previous_texts[i] += delta_text
|
||||||
if (
|
|
||||||
tool_choice_auto
|
|
||||||
or tool_choice_function_name
|
|
||||||
or request.tool_choice == "required"
|
|
||||||
or reasoning_parser
|
|
||||||
) and not self.use_harmony:
|
|
||||||
assert previous_texts is not None
|
|
||||||
assert all_previous_token_ids is not None
|
|
||||||
previous_texts[i] = current_text
|
|
||||||
all_previous_token_ids[i] = current_token_ids
|
|
||||||
else:
|
|
||||||
# Update for comprehensive logging even in simple case
|
|
||||||
assert previous_texts is not None
|
|
||||||
previous_texts[i] += delta_text
|
|
||||||
|
|
||||||
# set the previous values for the next iteration
|
# set the previous values for the next iteration
|
||||||
previous_num_tokens[i] += len(output.token_ids)
|
previous_num_tokens[i] += len(output.token_ids)
|
||||||
@@ -1326,19 +1270,3 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
)
|
)
|
||||||
|
|
||||||
return ChatCompletionLogProbs(content=logprobs_content)
|
return ChatCompletionLogProbs(content=logprobs_content)
|
||||||
|
|
||||||
def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
|
|
||||||
"""
|
|
||||||
Utility function to check if streamed tokens should go through the tool
|
|
||||||
call parser that was configured.
|
|
||||||
|
|
||||||
We only want to do this IF user-provided tools are set, a tool parser
|
|
||||||
is configured, "auto" tool choice is enabled, and the request's tool
|
|
||||||
choice field indicates that "auto" tool choice should be used.
|
|
||||||
"""
|
|
||||||
return (
|
|
||||||
request.tools
|
|
||||||
and self.tool_parser
|
|
||||||
and self.enable_auto_tools
|
|
||||||
and request.tool_choice in ["auto", None]
|
|
||||||
)
|
|
||||||
|
|||||||
Reference in New Issue
Block a user