[Refactor] Chat Completions Streaming Harmony Refactor and Bugfixes (#45104)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -33,10 +33,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionStreamResponse,
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
|
||||
TokenState,
|
||||
extract_harmony_streaming_delta,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
ErrorResponse,
|
||||
@@ -52,9 +48,6 @@ from vllm.entrypoints.openai.engine.serving import (
|
||||
clamp_prompt_logprobs,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
get_streamable_parser_for_assistant,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.entrypoints.serve.utils.tool_calls_utils import (
|
||||
@@ -155,7 +148,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
if mc.generation_config not in ("auto", "vllm")
|
||||
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
||||
)
|
||||
self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
|
||||
self.tool_call_id_type = get_tool_call_id_type(self.model_config)
|
||||
|
||||
# NOTE(woosuk): While OpenAI's chat completion API supports browsing
|
||||
@@ -408,11 +400,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
finish_reason_sent = [False] * num_choices
|
||||
num_prompt_tokens = 0
|
||||
num_cached_tokens = None
|
||||
if self.use_harmony:
|
||||
harmony_parsers = [
|
||||
get_streamable_parser_for_assistant() for _ in range(num_choices)
|
||||
]
|
||||
harmony_tools_streamed = [False] * num_choices
|
||||
tools_streamed = [False] * num_choices
|
||||
|
||||
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
|
||||
@@ -443,6 +430,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
]
|
||||
for p in parsers:
|
||||
if p is not None:
|
||||
# NOTE: HarmonyParser ignores _stream_state (uses its own FSM).
|
||||
p._stream_state.tool_call_id_type = self.tool_call_id_type
|
||||
p._stream_state.history_tool_call_cnt = history_tool_call_cnt
|
||||
else:
|
||||
@@ -572,32 +560,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
if self.use_harmony:
|
||||
harmony_parser = harmony_parsers[i]
|
||||
prev_recipient = harmony_parser.current_recipient
|
||||
|
||||
# Track accumulated content per token with their state
|
||||
token_states: list[TokenState] = []
|
||||
for token_id in output.token_ids:
|
||||
harmony_parser.process(token_id)
|
||||
token_delta = harmony_parser.last_content_delta or ""
|
||||
token_states.append(
|
||||
TokenState(
|
||||
harmony_parser.current_channel,
|
||||
harmony_parser.current_recipient,
|
||||
token_delta,
|
||||
)
|
||||
)
|
||||
delta_text = "".join(delta for _, _, delta in token_states)
|
||||
cur_channel = harmony_parser.current_channel
|
||||
|
||||
# handle the case where several tokens where generated at once
|
||||
# including the final token, leading to a delta in the text
|
||||
# but the current channel to be empty (start state)
|
||||
if not cur_channel and delta_text:
|
||||
cur_channel = "final"
|
||||
else:
|
||||
delta_text = output.text
|
||||
delta_text = output.text
|
||||
|
||||
if (
|
||||
not delta_text
|
||||
@@ -609,17 +572,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
delta_message: DeltaMessage | None
|
||||
|
||||
if self.use_harmony:
|
||||
delta_message, tools_streamed_flag = (
|
||||
extract_harmony_streaming_delta(
|
||||
harmony_parser=harmony_parser,
|
||||
token_states=token_states,
|
||||
prev_recipient=prev_recipient,
|
||||
include_reasoning=request.include_reasoning,
|
||||
)
|
||||
)
|
||||
harmony_tools_streamed[i] |= tools_streamed_flag
|
||||
elif parser is not None:
|
||||
if parser is not None:
|
||||
delta_message = parser.parse_delta(
|
||||
delta_text=delta_text,
|
||||
delta_token_ids=as_list(output.token_ids),
|
||||
@@ -627,8 +580,20 @@ class OpenAIServingChat(OpenAIServing):
|
||||
prompt_token_ids=res.prompt_token_ids,
|
||||
finished=output.finish_reason is not None,
|
||||
)
|
||||
if delta_message and delta_message.tool_calls:
|
||||
tools_streamed[i] = True
|
||||
if delta_message is not None:
|
||||
if delta_message.tool_calls:
|
||||
tools_streamed[i] = True
|
||||
|
||||
if (
|
||||
delta_message.reasoning
|
||||
and not request.include_reasoning
|
||||
):
|
||||
delta_message.reasoning = None
|
||||
if not (
|
||||
delta_message.content or delta_message.tool_calls
|
||||
):
|
||||
delta_message = None
|
||||
|
||||
# handle streaming just a content delta (no parsers)
|
||||
else:
|
||||
delta_message = DeltaMessage(content=delta_text)
|
||||
@@ -706,9 +671,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
# finish_reason is:
|
||||
# "tool_calls" for "auto" or "required" tool calls,
|
||||
# and "stop" for named tool calls.
|
||||
if (tools_streamed[i] and not tool_choice_function_name) or (
|
||||
self.use_harmony and harmony_tools_streamed[i]
|
||||
):
|
||||
if tools_streamed[i] and not tool_choice_function_name:
|
||||
finish_reason_ = "tool_calls"
|
||||
else:
|
||||
finish_reason_ = (
|
||||
|
||||
Reference in New Issue
Block a user