[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -54,7 +54,6 @@ from vllm.entrypoints.openai.engine.serving import (
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
get_streamable_parser_for_assistant,
|
||||
parse_chat_output,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
@@ -135,6 +134,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
model_name=self.model_config.model,
|
||||
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||
)
|
||||
if (
|
||||
is_mistral_tool_parser(self.tool_parser)
|
||||
@@ -359,14 +359,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
assert len(generators) == 1
|
||||
(result_generator,) = generators
|
||||
|
||||
parser: Parser | None = None
|
||||
if self.parser_cls is not None:
|
||||
parser = self.parser_cls(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
if request.stream:
|
||||
return self.chat_completion_stream_generator(
|
||||
request,
|
||||
@@ -387,7 +379,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
conversation,
|
||||
tokenizer,
|
||||
request_metadata,
|
||||
parser,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
|
||||
@@ -840,7 +832,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
conversation: list[ConversationMessage],
|
||||
tokenizer: TokenizerLike,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
parser: Parser | None = None,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
) -> ErrorResponse | ChatCompletionResponse:
|
||||
created_time = int(time.time())
|
||||
final_res: RequestOutput | None = None
|
||||
@@ -871,7 +863,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
token_ids = output.token_ids
|
||||
out_logprobs = output.logprobs
|
||||
tool_call_info = None
|
||||
|
||||
if request.logprobs and request.top_logprobs is not None:
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
@@ -885,75 +876,20 @@ class OpenAIServingChat(OpenAIServing):
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
if self.use_harmony:
|
||||
reasoning, content, _ = parse_chat_output(token_ids)
|
||||
if not request.include_reasoning:
|
||||
reasoning = None
|
||||
|
||||
if self.tool_parser is not None:
|
||||
if tokenizer is None:
|
||||
raise ValueError(
|
||||
"Tokenizer not available when `skip_tokenizer_init=True`"
|
||||
)
|
||||
|
||||
tool_parser = self.tool_parser(tokenizer, request.tools)
|
||||
# NOTE: We use token_ids for openai tool parser
|
||||
tool_call_info = tool_parser.extract_tool_calls(
|
||||
"",
|
||||
request=request,
|
||||
token_ids=token_ids, # type: ignore
|
||||
)
|
||||
content = tool_call_info.content
|
||||
message = ChatMessage(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_call_info.tool_calls,
|
||||
)
|
||||
else:
|
||||
message = ChatMessage(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
)
|
||||
|
||||
# Encode routed_experts for transport. JSON can't carry raw
|
||||
# bytes, so we write the ndarray as a ``.npy`` byte stream
|
||||
# and base64-encode it. ``pybase64`` is ~3x faster than the
|
||||
# stdlib ``base64`` on large payloads thanks to SIMD.
|
||||
routed_experts_b64 = None
|
||||
if output.routed_experts is not None:
|
||||
buf = io.BytesIO()
|
||||
np.save(buf, output.routed_experts)
|
||||
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
|
||||
"ascii"
|
||||
)
|
||||
|
||||
choice_data = ChatCompletionResponseChoice(
|
||||
index=output.index,
|
||||
message=message,
|
||||
logprobs=logprobs,
|
||||
finish_reason=(
|
||||
"tool_calls"
|
||||
if (tool_call_info is not None and tool_call_info.tools_called)
|
||||
else output.finish_reason
|
||||
if output.finish_reason
|
||||
else "stop"
|
||||
),
|
||||
stop_reason=output.stop_reason,
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
),
|
||||
routed_experts=routed_experts_b64,
|
||||
parser: Parser | None = None
|
||||
if self.parser_cls is not None:
|
||||
parser = self.parser_cls(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
choices.append(choice_data)
|
||||
continue
|
||||
|
||||
if parser is not None:
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
output.text,
|
||||
request,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
model_output_token_ids=token_ids,
|
||||
)
|
||||
if not request.include_reasoning:
|
||||
reasoning = None
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import datetime
|
||||
from collections.abc import Iterable, Sequence
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from openai.types.responses.tool import Tool
|
||||
@@ -456,65 +456,3 @@ def render_for_completion(messages: list[Message]) -> list[int]:
|
||||
|
||||
def get_streamable_parser_for_assistant() -> StreamableParser:
|
||||
return StreamableParser(get_encoding(), role=Role.ASSISTANT)
|
||||
|
||||
|
||||
def parse_output_into_messages(token_ids: Iterable[int]) -> StreamableParser:
|
||||
parser = get_streamable_parser_for_assistant()
|
||||
for token_id in token_ids:
|
||||
parser.process(token_id)
|
||||
return parser
|
||||
|
||||
|
||||
def parse_chat_output(
|
||||
token_ids: Sequence[int],
|
||||
) -> tuple[str | None, str | None, bool]:
|
||||
"""
|
||||
Parse the output of a Harmony chat completion into reasoning and final content.
|
||||
Note that when the `openai` tool parser is used, serving_chat only uses this
|
||||
for the reasoning content and gets the final content from the tool call parser.
|
||||
|
||||
When the `openai` tool parser is not enabled, or when `GptOssReasoningParser` is
|
||||
in use,this needs to return the final content without any tool calls parsed.
|
||||
|
||||
Empty reasoning or final content is returned as None instead of an empty string.
|
||||
"""
|
||||
parser = parse_output_into_messages(token_ids)
|
||||
output_msgs = parser.messages
|
||||
is_tool_call = False # TODO: update this when tool call is supported
|
||||
|
||||
# Get completed messages from the parser
|
||||
# - analysis channel: hidden reasoning
|
||||
# - commentary channel without recipient (preambles): visible to user
|
||||
# - final channel: visible to user
|
||||
# - commentary with recipient (tool calls): handled separately by tool parser
|
||||
reasoning_texts = [
|
||||
msg.content[0].text for msg in output_msgs if msg.channel == "analysis"
|
||||
]
|
||||
final_texts = [
|
||||
msg.content[0].text
|
||||
for msg in output_msgs
|
||||
if msg.channel == "final" or (msg.channel == "commentary" and not msg.recipient)
|
||||
]
|
||||
|
||||
# Extract partial messages from the parser
|
||||
if parser.current_channel == "analysis" and parser.current_content:
|
||||
reasoning_texts.append(parser.current_content)
|
||||
elif parser.current_channel == "final" and parser.current_content:
|
||||
final_texts.append(parser.current_content)
|
||||
elif (
|
||||
parser.current_channel == "commentary"
|
||||
and not parser.current_recipient
|
||||
and parser.current_content
|
||||
):
|
||||
# Preambles (commentary without recipient) are visible to user
|
||||
final_texts.append(parser.current_content)
|
||||
|
||||
# Flatten multiple messages into a single string
|
||||
reasoning: str | None = "\n".join(reasoning_texts)
|
||||
final_content: str | None = "\n".join(final_texts)
|
||||
|
||||
# Return None instead of empty string since existing callers check for None
|
||||
reasoning = reasoning or None
|
||||
final_content = final_content or None
|
||||
|
||||
return reasoning, final_content, is_tool_call
|
||||
|
||||
@@ -191,6 +191,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
model_name=self.model_config.model,
|
||||
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||
)
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
self.enable_force_include_usage = enable_force_include_usage
|
||||
|
||||
Reference in New Issue
Block a user