diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py index e6646eb763..f9ca0b7b32 100644 --- a/tests/parser/test_harmony.py +++ b/tests/parser/test_harmony.py @@ -131,6 +131,31 @@ def tool_call_entries(delta_message) -> list[tuple[int, str | None, str | None]] ] +class TestFlush: + def test_flush(self, harmony_parser): + harmony_parser.process_chunk( + encode_output("<|channel|>analysis<|message|>Think") + ) + + flushed = harmony_parser.flush() + + assert flushed is not None + assert flushed.channel == "analysis" + assert flushed.recipient is None + assert flushed.delta == "" + assert flushed.completed_message is not None + assert get_text(flushed.completed_message) == "Think" + assert harmony_parser._parser is None + + def test_flush_resets_after_eos_error(self, harmony_parser): + harmony_parser.process_chunk(encode_output("<|channel|>analysis")) + + flushed = harmony_parser.flush() + + assert flushed is None + assert harmony_parser._parser is None + + class TestParse: # Rendered conversation outputs. @@ -339,6 +364,7 @@ class TestParse: assert reasoning is None assert content == "I'm in the middle of answering" assert tool_calls is None + assert harmony_parser._parser is None def test_interrupted_reasoning_first_message(self, harmony_parser, chat_request): reasoning, content, tool_calls = harmony_parser.parse( @@ -352,6 +378,7 @@ class TestParse: assert reasoning == "I'm in the middle of thinking" assert content is None assert tool_calls is None + assert harmony_parser._parser is None def test_truncated_output(self, harmony_parser, chat_request): reasoning, content, tool_calls = harmony_parser.parse( @@ -367,6 +394,7 @@ class TestParse: assert reasoning == "I'm thinking." assert content == "I'm in the middle of answering" assert tool_calls is None + assert harmony_parser._parser is None @pytest.mark.parametrize( ("harmony_str", "expected_content"), @@ -435,7 +463,7 @@ class TestParseDelta: "<|end|><|start|>assistant<|channel|>final<|message|>Answer" ), request=chat_request, - finished=False, + finished=True, ) assert first_delta is not None @@ -444,6 +472,7 @@ class TestParseDelta: assert second_delta is not None assert second_delta.content == "Answer" assert second_delta.reasoning is None + assert parser._parser is None def test_multi_token(self, gpt_oss_tokenizer, chat_request): parser = HarmonyParser(gpt_oss_tokenizer) diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index ff022a00eb..4919e3da7e 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -3,12 +3,15 @@ from __future__ import annotations +import contextlib import json from collections.abc import Sequence from dataclasses import dataclass from enum import Enum, auto from typing import TYPE_CHECKING, NamedTuple +from openai_harmony import HarmonyError + from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.engine.protocol import ( @@ -28,8 +31,7 @@ from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser if TYPE_CHECKING: - from openai_harmony import Message, Role - from openai_harmony import StreamState as HarmonyStreamState + from openai_harmony import Message, StreamableParser class _SegmentType(Enum): @@ -82,33 +84,46 @@ class HarmonyParser(DelegatingParser): f"got {self.tool_parser.__class__.__name__}." ) - self._harmony_parser = get_streamable_parser_for_assistant() + self._parser: StreamableParser | None = None self._next_tool_call_index = 0 self._num_processed_messages = 0 @property - def state(self) -> HarmonyStreamState: - return self._harmony_parser.state + def _harmony_parser(self) -> StreamableParser: + """Lazily initializes the Harmony parser.""" + if self._parser is None: + self._parser = get_streamable_parser_for_assistant() + return self._parser - @property - def current_role(self) -> Role | None: - return self._harmony_parser.current_role + def _poll_completed_message(self) -> Message | None: + messages = self._harmony_parser.messages + if len(messages) <= self._num_processed_messages: + return None + msg = messages[self._num_processed_messages] + self._num_processed_messages += 1 + return msg - @property - def current_channel(self) -> str | None: - return self._harmony_parser.current_channel + def flush(self) -> Segment | None: + msg = None + with contextlib.suppress(HarmonyError): + self._harmony_parser.process_eos() + # TODO: Consider reraising - @property - def current_recipient(self) -> str | None: - return self._harmony_parser.current_recipient + msg = self._poll_completed_message() - @property - def current_content(self) -> str: - return self._harmony_parser.current_content + # Reset to the initial assistant-parser state for the next turn. + self._parser = None + self._num_processed_messages = 0 - @property - def current_content_type(self) -> str | None: - return self._harmony_parser.current_content_type + if msg is None: + return None + + return Segment( + channel=msg.channel, + recipient=msg.recipient, + delta="", + completed_message=msg, + ) def parse( self, @@ -123,24 +138,32 @@ class HarmonyParser(DelegatingParser): Callers must decide whether to surface them. """ result = self.process_chunk(model_output_token_ids) + flushed_segment = self.flush() + if flushed_segment is not None: + result.segments.append(flushed_segment) reasoning_parts: list[str] = [] content_parts: list[str] = [] tool_calls: list[FunctionCall] = [] - def _append_parsed_message( - channel: str | None, - recipient: str | None, - text: str, - content_type: str | None = None, - ) -> None: - segment_type = _SegmentType.from_channel_and_recipient(channel, recipient) + for segment in result.segments: + msg = segment.completed_message + if msg is None: + continue + if msg.author.role != "assistant" or not msg.content: + continue + text = msg.content[0].text + segment_type = _SegmentType.from_channel_and_recipient( + msg.channel, msg.recipient + ) match segment_type: case _SegmentType.REASONING if self.reasoning_parser and text: reasoning_parts.append(text) case _SegmentType.CONTENT if text: content_parts.append(text) case _SegmentType.TOOL if self.tool_parser: + recipient = msg.recipient + content_type = msg.content_type assert recipient is not None if content_type is not None and "json" not in content_type: arguments = text @@ -156,31 +179,6 @@ class HarmonyParser(DelegatingParser): ) ) - for segment in result.segments: - msg = segment.completed_message - if msg is None: - continue - if msg.author.role != "assistant" or not msg.content: - continue - _append_parsed_message( - channel=msg.channel, - recipient=msg.recipient, - text=msg.content[0].text, - content_type=msg.content_type, - ) - - if ( - self.current_channel is not None - or self.current_recipient is not None - or self.current_content - ): - _append_parsed_message( - channel=self.current_channel, - recipient=self.current_recipient, - text=self.current_content, - content_type=self.current_content_type, - ) - reasoning = "\n".join(reasoning_parts) or None content = "\n".join(content_parts) or None return reasoning, content, tool_calls or None @@ -194,8 +192,12 @@ class HarmonyParser(DelegatingParser): *, finished: bool, ) -> DeltaMessage | None: - prev_recipient = self.current_recipient + prev_recipient = self._harmony_parser.current_recipient result = self.process_chunk(delta_token_ids) + if finished: + flushed_segment = self.flush() + if flushed_segment is not None: + result.segments.append(flushed_segment) combined_content = "" combined_reasoning = "" tool_messages: list[DeltaToolCall] = [] @@ -248,6 +250,9 @@ class HarmonyParser(DelegatingParser): ) ) + if finished: + self._next_tool_call_index = 0 + if not combined_content and not combined_reasoning and not tool_messages: return None @@ -268,14 +273,10 @@ class HarmonyParser(DelegatingParser): reasoning_token_count = 0 for token_id in token_ids: self._harmony_parser.process(token_id) - channel = self.current_channel - recipient = self.current_recipient + channel = self._harmony_parser.current_channel + recipient = self._harmony_parser.current_recipient delta = self._harmony_parser.last_content_delta or "" - completed_message = None - _messages = self._harmony_parser.messages - if len(_messages) > self._num_processed_messages: - completed_message = _messages[self._num_processed_messages] - self._num_processed_messages += 1 + completed_message = self._poll_completed_message() if channel == "analysis" or ( channel == "commentary" and recipient is not None