[Frontend][Gpt-oss] Use process_eos() to flush Harmony Parser outputs. (#46437)
Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
@@ -131,6 +131,31 @@ def tool_call_entries(delta_message) -> list[tuple[int, str | None, str | None]]
|
||||
]
|
||||
|
||||
|
||||
class TestFlush:
|
||||
def test_flush(self, harmony_parser):
|
||||
harmony_parser.process_chunk(
|
||||
encode_output("<|channel|>analysis<|message|>Think")
|
||||
)
|
||||
|
||||
flushed = harmony_parser.flush()
|
||||
|
||||
assert flushed is not None
|
||||
assert flushed.channel == "analysis"
|
||||
assert flushed.recipient is None
|
||||
assert flushed.delta == ""
|
||||
assert flushed.completed_message is not None
|
||||
assert get_text(flushed.completed_message) == "Think"
|
||||
assert harmony_parser._parser is None
|
||||
|
||||
def test_flush_resets_after_eos_error(self, harmony_parser):
|
||||
harmony_parser.process_chunk(encode_output("<|channel|>analysis"))
|
||||
|
||||
flushed = harmony_parser.flush()
|
||||
|
||||
assert flushed is None
|
||||
assert harmony_parser._parser is None
|
||||
|
||||
|
||||
class TestParse:
|
||||
# Rendered conversation outputs.
|
||||
|
||||
@@ -339,6 +364,7 @@ class TestParse:
|
||||
assert reasoning is None
|
||||
assert content == "I'm in the middle of answering"
|
||||
assert tool_calls is None
|
||||
assert harmony_parser._parser is None
|
||||
|
||||
def test_interrupted_reasoning_first_message(self, harmony_parser, chat_request):
|
||||
reasoning, content, tool_calls = harmony_parser.parse(
|
||||
@@ -352,6 +378,7 @@ class TestParse:
|
||||
assert reasoning == "I'm in the middle of thinking"
|
||||
assert content is None
|
||||
assert tool_calls is None
|
||||
assert harmony_parser._parser is None
|
||||
|
||||
def test_truncated_output(self, harmony_parser, chat_request):
|
||||
reasoning, content, tool_calls = harmony_parser.parse(
|
||||
@@ -367,6 +394,7 @@ class TestParse:
|
||||
assert reasoning == "I'm thinking."
|
||||
assert content == "I'm in the middle of answering"
|
||||
assert tool_calls is None
|
||||
assert harmony_parser._parser is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("harmony_str", "expected_content"),
|
||||
@@ -435,7 +463,7 @@ class TestParseDelta:
|
||||
"<|end|><|start|>assistant<|channel|>final<|message|>Answer"
|
||||
),
|
||||
request=chat_request,
|
||||
finished=False,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
assert first_delta is not None
|
||||
@@ -444,6 +472,7 @@ class TestParseDelta:
|
||||
assert second_delta is not None
|
||||
assert second_delta.content == "Answer"
|
||||
assert second_delta.reasoning is None
|
||||
assert parser._parser is None
|
||||
|
||||
def test_multi_token(self, gpt_oss_tokenizer, chat_request):
|
||||
parser = HarmonyParser(gpt_oss_tokenizer)
|
||||
|
||||
+61
-60
@@ -3,12 +3,15 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
from openai_harmony import HarmonyError
|
||||
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
@@ -28,8 +31,7 @@ from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
|
||||
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai_harmony import Message, Role
|
||||
from openai_harmony import StreamState as HarmonyStreamState
|
||||
from openai_harmony import Message, StreamableParser
|
||||
|
||||
|
||||
class _SegmentType(Enum):
|
||||
@@ -82,33 +84,46 @@ class HarmonyParser(DelegatingParser):
|
||||
f"got {self.tool_parser.__class__.__name__}."
|
||||
)
|
||||
|
||||
self._harmony_parser = get_streamable_parser_for_assistant()
|
||||
self._parser: StreamableParser | None = None
|
||||
self._next_tool_call_index = 0
|
||||
self._num_processed_messages = 0
|
||||
|
||||
@property
|
||||
def state(self) -> HarmonyStreamState:
|
||||
return self._harmony_parser.state
|
||||
def _harmony_parser(self) -> StreamableParser:
|
||||
"""Lazily initializes the Harmony parser."""
|
||||
if self._parser is None:
|
||||
self._parser = get_streamable_parser_for_assistant()
|
||||
return self._parser
|
||||
|
||||
@property
|
||||
def current_role(self) -> Role | None:
|
||||
return self._harmony_parser.current_role
|
||||
def _poll_completed_message(self) -> Message | None:
|
||||
messages = self._harmony_parser.messages
|
||||
if len(messages) <= self._num_processed_messages:
|
||||
return None
|
||||
msg = messages[self._num_processed_messages]
|
||||
self._num_processed_messages += 1
|
||||
return msg
|
||||
|
||||
@property
|
||||
def current_channel(self) -> str | None:
|
||||
return self._harmony_parser.current_channel
|
||||
def flush(self) -> Segment | None:
|
||||
msg = None
|
||||
with contextlib.suppress(HarmonyError):
|
||||
self._harmony_parser.process_eos()
|
||||
# TODO: Consider reraising
|
||||
|
||||
@property
|
||||
def current_recipient(self) -> str | None:
|
||||
return self._harmony_parser.current_recipient
|
||||
msg = self._poll_completed_message()
|
||||
|
||||
@property
|
||||
def current_content(self) -> str:
|
||||
return self._harmony_parser.current_content
|
||||
# Reset to the initial assistant-parser state for the next turn.
|
||||
self._parser = None
|
||||
self._num_processed_messages = 0
|
||||
|
||||
@property
|
||||
def current_content_type(self) -> str | None:
|
||||
return self._harmony_parser.current_content_type
|
||||
if msg is None:
|
||||
return None
|
||||
|
||||
return Segment(
|
||||
channel=msg.channel,
|
||||
recipient=msg.recipient,
|
||||
delta="",
|
||||
completed_message=msg,
|
||||
)
|
||||
|
||||
def parse(
|
||||
self,
|
||||
@@ -123,24 +138,32 @@ class HarmonyParser(DelegatingParser):
|
||||
Callers must decide whether to surface them.
|
||||
"""
|
||||
result = self.process_chunk(model_output_token_ids)
|
||||
flushed_segment = self.flush()
|
||||
if flushed_segment is not None:
|
||||
result.segments.append(flushed_segment)
|
||||
|
||||
reasoning_parts: list[str] = []
|
||||
content_parts: list[str] = []
|
||||
tool_calls: list[FunctionCall] = []
|
||||
|
||||
def _append_parsed_message(
|
||||
channel: str | None,
|
||||
recipient: str | None,
|
||||
text: str,
|
||||
content_type: str | None = None,
|
||||
) -> None:
|
||||
segment_type = _SegmentType.from_channel_and_recipient(channel, recipient)
|
||||
for segment in result.segments:
|
||||
msg = segment.completed_message
|
||||
if msg is None:
|
||||
continue
|
||||
if msg.author.role != "assistant" or not msg.content:
|
||||
continue
|
||||
text = msg.content[0].text
|
||||
segment_type = _SegmentType.from_channel_and_recipient(
|
||||
msg.channel, msg.recipient
|
||||
)
|
||||
match segment_type:
|
||||
case _SegmentType.REASONING if self.reasoning_parser and text:
|
||||
reasoning_parts.append(text)
|
||||
case _SegmentType.CONTENT if text:
|
||||
content_parts.append(text)
|
||||
case _SegmentType.TOOL if self.tool_parser:
|
||||
recipient = msg.recipient
|
||||
content_type = msg.content_type
|
||||
assert recipient is not None
|
||||
if content_type is not None and "json" not in content_type:
|
||||
arguments = text
|
||||
@@ -156,31 +179,6 @@ class HarmonyParser(DelegatingParser):
|
||||
)
|
||||
)
|
||||
|
||||
for segment in result.segments:
|
||||
msg = segment.completed_message
|
||||
if msg is None:
|
||||
continue
|
||||
if msg.author.role != "assistant" or not msg.content:
|
||||
continue
|
||||
_append_parsed_message(
|
||||
channel=msg.channel,
|
||||
recipient=msg.recipient,
|
||||
text=msg.content[0].text,
|
||||
content_type=msg.content_type,
|
||||
)
|
||||
|
||||
if (
|
||||
self.current_channel is not None
|
||||
or self.current_recipient is not None
|
||||
or self.current_content
|
||||
):
|
||||
_append_parsed_message(
|
||||
channel=self.current_channel,
|
||||
recipient=self.current_recipient,
|
||||
text=self.current_content,
|
||||
content_type=self.current_content_type,
|
||||
)
|
||||
|
||||
reasoning = "\n".join(reasoning_parts) or None
|
||||
content = "\n".join(content_parts) or None
|
||||
return reasoning, content, tool_calls or None
|
||||
@@ -194,8 +192,12 @@ class HarmonyParser(DelegatingParser):
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
prev_recipient = self.current_recipient
|
||||
prev_recipient = self._harmony_parser.current_recipient
|
||||
result = self.process_chunk(delta_token_ids)
|
||||
if finished:
|
||||
flushed_segment = self.flush()
|
||||
if flushed_segment is not None:
|
||||
result.segments.append(flushed_segment)
|
||||
combined_content = ""
|
||||
combined_reasoning = ""
|
||||
tool_messages: list[DeltaToolCall] = []
|
||||
@@ -248,6 +250,9 @@ class HarmonyParser(DelegatingParser):
|
||||
)
|
||||
)
|
||||
|
||||
if finished:
|
||||
self._next_tool_call_index = 0
|
||||
|
||||
if not combined_content and not combined_reasoning and not tool_messages:
|
||||
return None
|
||||
|
||||
@@ -268,14 +273,10 @@ class HarmonyParser(DelegatingParser):
|
||||
reasoning_token_count = 0
|
||||
for token_id in token_ids:
|
||||
self._harmony_parser.process(token_id)
|
||||
channel = self.current_channel
|
||||
recipient = self.current_recipient
|
||||
channel = self._harmony_parser.current_channel
|
||||
recipient = self._harmony_parser.current_recipient
|
||||
delta = self._harmony_parser.last_content_delta or ""
|
||||
completed_message = None
|
||||
_messages = self._harmony_parser.messages
|
||||
if len(_messages) > self._num_processed_messages:
|
||||
completed_message = _messages[self._num_processed_messages]
|
||||
self._num_processed_messages += 1
|
||||
completed_message = self._poll_completed_message()
|
||||
|
||||
if channel == "analysis" or (
|
||||
channel == "commentary" and recipient is not None
|
||||
|
||||
Reference in New Issue
Block a user