[Refactor] Chat Completions Streaming Harmony Refactor and Bugfixes (#45104)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-06-12 01:09:47 +00:00
committed by GitHub
parent 6f573f486b
commit e0871ad225
5 changed files with 394 additions and 733 deletions
@@ -33,10 +33,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionStreamResponse,
ChatMessage,
)
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
TokenState,
extract_harmony_streaming_delta,
)
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
ErrorResponse,
@@ -52,9 +48,6 @@ from vllm.entrypoints.openai.engine.serving import (
clamp_prompt_logprobs,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.parser.harmony_utils import (
get_streamable_parser_for_assistant,
)
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.entrypoints.serve.utils.tool_calls_utils import (
@@ -155,7 +148,6 @@ class OpenAIServingChat(OpenAIServing):
if mc.generation_config not in ("auto", "vllm")
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
)
self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
self.tool_call_id_type = get_tool_call_id_type(self.model_config)
# NOTE(woosuk): While OpenAI's chat completion API supports browsing
@@ -408,11 +400,6 @@ class OpenAIServingChat(OpenAIServing):
finish_reason_sent = [False] * num_choices
num_prompt_tokens = 0
num_cached_tokens = None
if self.use_harmony:
harmony_parsers = [
get_streamable_parser_for_assistant() for _ in range(num_choices)
]
harmony_tools_streamed = [False] * num_choices
tools_streamed = [False] * num_choices
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
@@ -443,6 +430,7 @@ class OpenAIServingChat(OpenAIServing):
]
for p in parsers:
if p is not None:
# NOTE: HarmonyParser ignores _stream_state (uses its own FSM).
p._stream_state.tool_call_id_type = self.tool_call_id_type
p._stream_state.history_tool_call_cnt = history_tool_call_cnt
else:
@@ -572,32 +560,7 @@ class OpenAIServingChat(OpenAIServing):
else:
logprobs = None
if self.use_harmony:
harmony_parser = harmony_parsers[i]
prev_recipient = harmony_parser.current_recipient
# Track accumulated content per token with their state
token_states: list[TokenState] = []
for token_id in output.token_ids:
harmony_parser.process(token_id)
token_delta = harmony_parser.last_content_delta or ""
token_states.append(
TokenState(
harmony_parser.current_channel,
harmony_parser.current_recipient,
token_delta,
)
)
delta_text = "".join(delta for _, _, delta in token_states)
cur_channel = harmony_parser.current_channel
# handle the case where several tokens where generated at once
# including the final token, leading to a delta in the text
# but the current channel to be empty (start state)
if not cur_channel and delta_text:
cur_channel = "final"
else:
delta_text = output.text
delta_text = output.text
if (
not delta_text
@@ -609,17 +572,7 @@ class OpenAIServingChat(OpenAIServing):
delta_message: DeltaMessage | None
if self.use_harmony:
delta_message, tools_streamed_flag = (
extract_harmony_streaming_delta(
harmony_parser=harmony_parser,
token_states=token_states,
prev_recipient=prev_recipient,
include_reasoning=request.include_reasoning,
)
)
harmony_tools_streamed[i] |= tools_streamed_flag
elif parser is not None:
if parser is not None:
delta_message = parser.parse_delta(
delta_text=delta_text,
delta_token_ids=as_list(output.token_ids),
@@ -627,8 +580,20 @@ class OpenAIServingChat(OpenAIServing):
prompt_token_ids=res.prompt_token_ids,
finished=output.finish_reason is not None,
)
if delta_message and delta_message.tool_calls:
tools_streamed[i] = True
if delta_message is not None:
if delta_message.tool_calls:
tools_streamed[i] = True
if (
delta_message.reasoning
and not request.include_reasoning
):
delta_message.reasoning = None
if not (
delta_message.content or delta_message.tool_calls
):
delta_message = None
# handle streaming just a content delta (no parsers)
else:
delta_message = DeltaMessage(content=delta_text)
@@ -706,9 +671,7 @@ class OpenAIServingChat(OpenAIServing):
# finish_reason is:
# "tool_calls" for "auto" or "required" tool calls,
# and "stop" for named tool calls.
if (tools_streamed[i] and not tool_choice_function_name) or (
self.use_harmony and harmony_tools_streamed[i]
):
if tools_streamed[i] and not tool_choice_function_name:
finish_reason_ = "tool_calls"
else:
finish_reason_ = (