[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-06-11 21:18:30 +00:00
committed by GitHub
parent 5a6c7b7ab5
commit f712fd0d7d
16 changed files with 822 additions and 816 deletions
@@ -54,7 +54,6 @@ from vllm.entrypoints.openai.engine.serving import (
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.parser.harmony_utils import (
get_streamable_parser_for_assistant,
parse_chat_output,
)
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -135,6 +134,7 @@ class OpenAIServingChat(OpenAIServing):
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
model_name=self.model_config.model,
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
)
if (
is_mistral_tool_parser(self.tool_parser)
@@ -359,14 +359,6 @@ class OpenAIServingChat(OpenAIServing):
assert len(generators) == 1
(result_generator,) = generators
parser: Parser | None = None
if self.parser_cls is not None:
parser = self.parser_cls(
tokenizer,
request.tools,
chat_template_kwargs=chat_template_kwargs,
)
if request.stream:
return self.chat_completion_stream_generator(
request,
@@ -387,7 +379,7 @@ class OpenAIServingChat(OpenAIServing):
conversation,
tokenizer,
request_metadata,
parser,
chat_template_kwargs=chat_template_kwargs,
)
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
@@ -840,7 +832,7 @@ class OpenAIServingChat(OpenAIServing):
conversation: list[ConversationMessage],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
parser: Parser | None = None,
chat_template_kwargs: dict[str, Any] | None = None,
) -> ErrorResponse | ChatCompletionResponse:
created_time = int(time.time())
final_res: RequestOutput | None = None
@@ -871,7 +863,6 @@ class OpenAIServingChat(OpenAIServing):
self._raise_if_error(output.finish_reason, request_id)
token_ids = output.token_ids
out_logprobs = output.logprobs
tool_call_info = None
if request.logprobs and request.top_logprobs is not None:
assert out_logprobs is not None, "Did not output logprobs"
@@ -885,75 +876,20 @@ class OpenAIServingChat(OpenAIServing):
else:
logprobs = None
if self.use_harmony:
reasoning, content, _ = parse_chat_output(token_ids)
if not request.include_reasoning:
reasoning = None
if self.tool_parser is not None:
if tokenizer is None:
raise ValueError(
"Tokenizer not available when `skip_tokenizer_init=True`"
)
tool_parser = self.tool_parser(tokenizer, request.tools)
# NOTE: We use token_ids for openai tool parser
tool_call_info = tool_parser.extract_tool_calls(
"",
request=request,
token_ids=token_ids, # type: ignore
)
content = tool_call_info.content
message = ChatMessage(
role=role,
reasoning=reasoning,
content=content,
tool_calls=tool_call_info.tool_calls,
)
else:
message = ChatMessage(
role=role,
reasoning=reasoning,
content=content,
)
# Encode routed_experts for transport. JSON can't carry raw
# bytes, so we write the ndarray as a ``.npy`` byte stream
# and base64-encode it. ``pybase64`` is ~3x faster than the
# stdlib ``base64`` on large payloads thanks to SIMD.
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
"ascii"
)
choice_data = ChatCompletionResponseChoice(
index=output.index,
message=message,
logprobs=logprobs,
finish_reason=(
"tool_calls"
if (tool_call_info is not None and tool_call_info.tools_called)
else output.finish_reason
if output.finish_reason
else "stop"
),
stop_reason=output.stop_reason,
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=routed_experts_b64,
parser: Parser | None = None
if self.parser_cls is not None:
parser = self.parser_cls(
tokenizer,
request.tools,
chat_template_kwargs=chat_template_kwargs,
)
choices.append(choice_data)
continue
if parser is not None:
reasoning, content, tool_calls = parser.parse(
output.text,
request,
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=token_ids,
)
if not request.include_reasoning:
reasoning = None