[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-06-11 21:18:30 +00:00
committed by GitHub
parent 5a6c7b7ab5
commit f712fd0d7d
16 changed files with 822 additions and 816 deletions
+2
View File
@@ -5,10 +5,12 @@ from vllm.parser.abstract_parser import (
DelegatingParser,
Parser,
)
from vllm.parser.harmony import HarmonyParser
from vllm.parser.parser_manager import ParserManager
__all__ = [
"Parser",
"DelegatingParser",
"HarmonyParser",
"ParserManager",
]
+3
View File
@@ -282,6 +282,7 @@ class Parser:
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
"""Parse a complete model output, extracting reasoning and tool calls.
@@ -289,6 +290,7 @@ class Parser:
model_output: The complete model-generated string.
request: The request object used to generate the output.
enable_auto_tools: Whether to enable automatic tool call parsing.
model_output_token_ids: The generated raw output token IDs.
Returns:
A tuple of (reasoning, content, tool_calls).
@@ -642,6 +644,7 @@ class DelegatingParser(Parser):
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
reasoning, content = self.extract_reasoning(model_output, request)
tool_calls, content = self._extract_tool_calls(
+240
View File
@@ -0,0 +1,240 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import json
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum, auto
from typing import TYPE_CHECKING, NamedTuple
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
FunctionCall,
)
from vllm.entrypoints.openai.parser.harmony_utils import (
extract_function_from_recipient,
get_streamable_parser_for_assistant,
is_function_recipient,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.abstract_parser import DelegatingParser
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
if TYPE_CHECKING:
from openai_harmony import Message, Role
from openai_harmony import StreamState as HarmonyStreamState
class _SegmentType(Enum):
TOOL = auto()
REASONING = auto()
CONTENT = auto()
IGNORE = auto()
@staticmethod
def from_channel_and_recipient(
channel: str | None, recipient: str | None
) -> _SegmentType:
if recipient and is_function_recipient(recipient):
return _SegmentType.TOOL
if channel == "analysis":
return _SegmentType.REASONING
if channel == "final" or (channel == "commentary" and recipient is None):
return _SegmentType.CONTENT
return _SegmentType.IGNORE
class Segment(NamedTuple):
channel: str | None
recipient: str | None
delta: str
is_boundary: bool = False
completed_message: Message | None = None
@dataclass
class ChunkResult:
segments: list[Segment]
reasoning_token_count: int
class HarmonyParser(DelegatingParser):
def __init__(self, tokenizer, tools=None, *args, **kwargs):
super().__init__(tokenizer, tools, *args, **kwargs)
if self._reasoning_parser and not isinstance(
self._reasoning_parser, GptOssReasoningParser
):
raise ValueError(
"Harmony requires GptOssReasoningParser, "
f"got {self._reasoning_parser.__class__.__name__}."
)
if self._tool_parser and not isinstance(self._tool_parser, GptOssToolParser):
raise ValueError(
"Harmony requires GptOssToolParser, "
f"got {self._tool_parser.__class__.__name__}."
)
self._harmony_parser = get_streamable_parser_for_assistant()
@property
def messages(self) -> list[Message]:
return self._harmony_parser.messages
@property
def state(self) -> HarmonyStreamState:
return self._harmony_parser.state
@property
def current_role(self) -> Role | None:
return self._harmony_parser.current_role
@property
def current_channel(self) -> str | None:
return self._harmony_parser.current_channel
@property
def current_recipient(self) -> str | None:
return self._harmony_parser.current_recipient
@property
def current_content(self) -> str:
return self._harmony_parser.current_content
@property
def current_content_type(self) -> str | None:
return self._harmony_parser.current_content_type
def parse(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
"""Parse Harmony output from token IDs.
Tool calls are always extracted regardless of ``enable_auto_tools``.
Callers must decide whether to surface them.
"""
result = self.process_chunk(model_output_token_ids)
reasoning_parts: list[str] = []
content_parts: list[str] = []
tool_calls: list[FunctionCall] = []
def _append_parsed_message(
channel: str | None,
recipient: str | None,
text: str,
content_type: str | None = None,
) -> None:
segment_type = _SegmentType.from_channel_and_recipient(channel, recipient)
match segment_type:
case _SegmentType.REASONING if self.reasoning_parser and text:
reasoning_parts.append(text)
case _SegmentType.CONTENT if text:
content_parts.append(text)
case _SegmentType.TOOL if self.tool_parser:
assert recipient is not None
if content_type is not None and "json" not in content_type:
arguments = text
else:
try:
arguments = json.dumps(json.loads(text))
except json.JSONDecodeError:
arguments = text
tool_calls.append(
FunctionCall(
name=extract_function_from_recipient(recipient),
arguments=arguments,
)
)
for segment in result.segments:
msg = segment.completed_message
if msg is None:
continue
if msg.author.role != "assistant" or not msg.content:
continue
_append_parsed_message(
channel=msg.channel,
recipient=msg.recipient,
text=msg.content[0].text,
content_type=msg.content_type,
)
if (
self.current_channel is not None
or self.current_recipient is not None
or self.current_content
):
_append_parsed_message(
channel=self.current_channel,
recipient=self.current_recipient,
text=self.current_content,
content_type=self.current_content_type,
)
reasoning = "\n".join(reasoning_parts) or None
content = "\n".join(content_parts) or None
return reasoning, content, tool_calls or None
def parse_delta(
self,
delta_text: str,
delta_token_ids: list[int],
request: ChatCompletionRequest | ResponsesRequest,
prompt_token_ids: list[int] | None = None,
*,
finished: bool,
) -> DeltaMessage | None:
raise NotImplementedError(
"HarmonyParser streaming parsing is deferred. "
"Use the existing harmony streaming path."
)
def process_chunk(self, token_ids: Sequence[int]) -> ChunkResult:
if not token_ids:
return ChunkResult(segments=[], reasoning_token_count=0)
from openai_harmony import StreamState
segments: list[Segment] = []
reasoning_token_count = 0
for token_id in token_ids:
self._harmony_parser.process(token_id)
channel = self.current_channel
recipient = self.current_recipient
delta = self._harmony_parser.last_content_delta or ""
completed_message = None
is_boundary = self.state == StreamState.EXPECT_START
if is_boundary and self.messages:
completed_message = self.messages[-1]
if channel == "analysis" or (
channel == "commentary" and recipient is not None
):
reasoning_token_count += 1
segments.append(
Segment(
channel=channel,
recipient=recipient,
delta=delta,
is_boundary=is_boundary,
completed_message=completed_message,
)
)
# TODO: Optionally merge and suppress empty Segments
return ChunkResult(
segments=segments,
reasoning_token_count=reasoning_token_count,
)
+6 -1
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
from collections.abc import Sequence
from typing import TYPE_CHECKING
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall
@@ -43,10 +44,14 @@ class MistralParser(DelegatingParser):
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
enable_auto_tools: bool = False,
model_output_token_ids: Sequence[int] = (),
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
self._maybe_force_auto_tool_parsing(request)
reasoning, content, tool_calls = super().parse(
model_output, request, enable_auto_tools
model_output,
request,
enable_auto_tools,
model_output_token_ids,
)
if tool_calls:
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
+10
View File
@@ -79,6 +79,7 @@ class ParserManager:
reasoning_parser_name: str | None = None,
enable_auto_tools: bool = False,
model_name: str | None = None,
is_harmony: bool = False,
) -> type[Parser] | None:
"""
Get a Parser that handles both reasoning and tool parsing.
@@ -91,6 +92,8 @@ class ParserManager:
reasoning_parser_name: The name of the reasoning parser.
enable_auto_tools: Whether auto tool choice is enabled.
model_name: The model name for parser-specific warnings.
is_harmony: Whether the selected model uses the Harmony format.
If True, HarmonyParser is always returned.
Returns:
A Parser class, or None if neither parser is specified.
@@ -108,6 +111,13 @@ class ParserManager:
from vllm.utils.mistral import is_mistral_tool_parser
if is_harmony:
from vllm.parser.harmony import HarmonyParser
HarmonyParser.reasoning_parser_cls = reasoning_parser_cls
HarmonyParser.tool_parser_cls = tool_parser_cls
return HarmonyParser
if is_mistral_tool_parser(tool_parser_cls):
from vllm.parser.mistral import MistralParser