[Frontend][Gpt-oss] Use process_eos() to flush Harmony Parser outputs. (#46437)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-06-26 17:18:47 -04:00
committed by GitHub
parent c40d307731
commit 77f8796d16
2 changed files with 91 additions and 61 deletions
+30 -1
View File
@@ -131,6 +131,31 @@ def tool_call_entries(delta_message) -> list[tuple[int, str | None, str | None]]
]
class TestFlush:
def test_flush(self, harmony_parser):
harmony_parser.process_chunk(
encode_output("<|channel|>analysis<|message|>Think")
)
flushed = harmony_parser.flush()
assert flushed is not None
assert flushed.channel == "analysis"
assert flushed.recipient is None
assert flushed.delta == ""
assert flushed.completed_message is not None
assert get_text(flushed.completed_message) == "Think"
assert harmony_parser._parser is None
def test_flush_resets_after_eos_error(self, harmony_parser):
harmony_parser.process_chunk(encode_output("<|channel|>analysis"))
flushed = harmony_parser.flush()
assert flushed is None
assert harmony_parser._parser is None
class TestParse:
# Rendered conversation outputs.
@@ -339,6 +364,7 @@ class TestParse:
assert reasoning is None
assert content == "I'm in the middle of answering"
assert tool_calls is None
assert harmony_parser._parser is None
def test_interrupted_reasoning_first_message(self, harmony_parser, chat_request):
reasoning, content, tool_calls = harmony_parser.parse(
@@ -352,6 +378,7 @@ class TestParse:
assert reasoning == "I'm in the middle of thinking"
assert content is None
assert tool_calls is None
assert harmony_parser._parser is None
def test_truncated_output(self, harmony_parser, chat_request):
reasoning, content, tool_calls = harmony_parser.parse(
@@ -367,6 +394,7 @@ class TestParse:
assert reasoning == "I'm thinking."
assert content == "I'm in the middle of answering"
assert tool_calls is None
assert harmony_parser._parser is None
@pytest.mark.parametrize(
("harmony_str", "expected_content"),
@@ -435,7 +463,7 @@ class TestParseDelta:
"<|end|><|start|>assistant<|channel|>final<|message|>Answer"
),
request=chat_request,
finished=False,
finished=True,
)
assert first_delta is not None
@@ -444,6 +472,7 @@ class TestParseDelta:
assert second_delta is not None
assert second_delta.content == "Answer"
assert second_delta.reasoning is None
assert parser._parser is None
def test_multi_token(self, gpt_oss_tokenizer, chat_request):
parser = HarmonyParser(gpt_oss_tokenizer)
+61 -60
View File
@@ -3,12 +3,15 @@
from __future__ import annotations
import contextlib
import json
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum, auto
from typing import TYPE_CHECKING, NamedTuple
from openai_harmony import HarmonyError
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import (
@@ -28,8 +31,7 @@ from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
if TYPE_CHECKING:
from openai_harmony import Message, Role
from openai_harmony import StreamState as HarmonyStreamState
from openai_harmony import Message, StreamableParser
class _SegmentType(Enum):
@@ -82,33 +84,46 @@ class HarmonyParser(DelegatingParser):
f"got {self.tool_parser.__class__.__name__}."
)
self._harmony_parser = get_streamable_parser_for_assistant()
self._parser: StreamableParser | None = None
self._next_tool_call_index = 0
self._num_processed_messages = 0
@property
def state(self) -> HarmonyStreamState:
return self._harmony_parser.state
def _harmony_parser(self) -> StreamableParser:
"""Lazily initializes the Harmony parser."""
if self._parser is None:
self._parser = get_streamable_parser_for_assistant()
return self._parser
@property
def current_role(self) -> Role | None:
return self._harmony_parser.current_role
def _poll_completed_message(self) -> Message | None:
messages = self._harmony_parser.messages
if len(messages) <= self._num_processed_messages:
return None
msg = messages[self._num_processed_messages]
self._num_processed_messages += 1
return msg
@property
def current_channel(self) -> str | None:
return self._harmony_parser.current_channel
def flush(self) -> Segment | None:
msg = None
with contextlib.suppress(HarmonyError):
self._harmony_parser.process_eos()
# TODO: Consider reraising
@property
def current_recipient(self) -> str | None:
return self._harmony_parser.current_recipient
msg = self._poll_completed_message()
@property
def current_content(self) -> str:
return self._harmony_parser.current_content
# Reset to the initial assistant-parser state for the next turn.
self._parser = None
self._num_processed_messages = 0
@property
def current_content_type(self) -> str | None:
return self._harmony_parser.current_content_type
if msg is None:
return None
return Segment(
channel=msg.channel,
recipient=msg.recipient,
delta="",
completed_message=msg,
)
def parse(
self,
@@ -123,24 +138,32 @@ class HarmonyParser(DelegatingParser):
Callers must decide whether to surface them.
"""
result = self.process_chunk(model_output_token_ids)
flushed_segment = self.flush()
if flushed_segment is not None:
result.segments.append(flushed_segment)
reasoning_parts: list[str] = []
content_parts: list[str] = []
tool_calls: list[FunctionCall] = []
def _append_parsed_message(
channel: str | None,
recipient: str | None,
text: str,
content_type: str | None = None,
) -> None:
segment_type = _SegmentType.from_channel_and_recipient(channel, recipient)
for segment in result.segments:
msg = segment.completed_message
if msg is None:
continue
if msg.author.role != "assistant" or not msg.content:
continue
text = msg.content[0].text
segment_type = _SegmentType.from_channel_and_recipient(
msg.channel, msg.recipient
)
match segment_type:
case _SegmentType.REASONING if self.reasoning_parser and text:
reasoning_parts.append(text)
case _SegmentType.CONTENT if text:
content_parts.append(text)
case _SegmentType.TOOL if self.tool_parser:
recipient = msg.recipient
content_type = msg.content_type
assert recipient is not None
if content_type is not None and "json" not in content_type:
arguments = text
@@ -156,31 +179,6 @@ class HarmonyParser(DelegatingParser):
)
)
for segment in result.segments:
msg = segment.completed_message
if msg is None:
continue
if msg.author.role != "assistant" or not msg.content:
continue
_append_parsed_message(
channel=msg.channel,
recipient=msg.recipient,
text=msg.content[0].text,
content_type=msg.content_type,
)
if (
self.current_channel is not None
or self.current_recipient is not None
or self.current_content
):
_append_parsed_message(
channel=self.current_channel,
recipient=self.current_recipient,
text=self.current_content,
content_type=self.current_content_type,
)
reasoning = "\n".join(reasoning_parts) or None
content = "\n".join(content_parts) or None
return reasoning, content, tool_calls or None
@@ -194,8 +192,12 @@ class HarmonyParser(DelegatingParser):
*,
finished: bool,
) -> DeltaMessage | None:
prev_recipient = self.current_recipient
prev_recipient = self._harmony_parser.current_recipient
result = self.process_chunk(delta_token_ids)
if finished:
flushed_segment = self.flush()
if flushed_segment is not None:
result.segments.append(flushed_segment)
combined_content = ""
combined_reasoning = ""
tool_messages: list[DeltaToolCall] = []
@@ -248,6 +250,9 @@ class HarmonyParser(DelegatingParser):
)
)
if finished:
self._next_tool_call_index = 0
if not combined_content and not combined_reasoning and not tool_messages:
return None
@@ -268,14 +273,10 @@ class HarmonyParser(DelegatingParser):
reasoning_token_count = 0
for token_id in token_ids:
self._harmony_parser.process(token_id)
channel = self.current_channel
recipient = self.current_recipient
channel = self._harmony_parser.current_channel
recipient = self._harmony_parser.current_recipient
delta = self._harmony_parser.last_content_delta or ""
completed_message = None
_messages = self._harmony_parser.messages
if len(_messages) > self._num_processed_messages:
completed_message = _messages[self._num_processed_messages]
self._num_processed_messages += 1
completed_message = self._poll_completed_message()
if channel == "analysis" or (
channel == "commentary" and recipient is not None