[Bugfix][Frontend][gpt-oss] Return raw output when Harmony parser ends non-terminal (#47062)
Signed-off-by: Achyuthan Sivasankar <achyuthan.sivasankar@gmail.com>
This commit is contained in:
@@ -7,6 +7,7 @@ from collections.abc import Sequence
|
||||
import pytest
|
||||
from openai_harmony import (
|
||||
Conversation,
|
||||
HarmonyError,
|
||||
Message,
|
||||
RenderConversationConfig,
|
||||
Role,
|
||||
@@ -81,9 +82,8 @@ def get_model_output_tokens(
|
||||
Role.ASSISTANT,
|
||||
config=config,
|
||||
)
|
||||
full_ids = enc.render_conversation_for_completion(
|
||||
full_ids = enc.render_conversation(
|
||||
Conversation.from_messages([*prompt_messages, *response_messages]),
|
||||
Role.ASSISTANT,
|
||||
config=config,
|
||||
)
|
||||
assert full_ids[: len(prompt_ids)] == prompt_ids
|
||||
@@ -147,12 +147,12 @@ class TestFlush:
|
||||
assert get_text(flushed.completed_message) == "Think"
|
||||
assert harmony_parser._parser is None
|
||||
|
||||
def test_flush_resets_after_eos_error(self, harmony_parser):
|
||||
def test_flush_raises_and_resets_on_non_terminal_eos(self, harmony_parser):
|
||||
harmony_parser.process_chunk(encode_output("<|channel|>analysis"))
|
||||
|
||||
flushed = harmony_parser.flush()
|
||||
with pytest.raises(HarmonyError):
|
||||
harmony_parser.flush()
|
||||
|
||||
assert flushed is None
|
||||
assert harmony_parser._parser is None
|
||||
|
||||
|
||||
@@ -396,6 +396,23 @@ class TestParse:
|
||||
assert tool_calls is None
|
||||
assert harmony_parser._parser is None
|
||||
|
||||
def test_malformed_final_recovers_raw_content(self, harmony_parser, chat_request):
|
||||
raw_output = (
|
||||
"<|channel|>analysis<|message|>thinking<|end|>"
|
||||
'<|start|>assistant<|channel|>final {"answer": "hi"}<|return|>'
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = harmony_parser.parse(
|
||||
raw_output,
|
||||
chat_request,
|
||||
model_output_token_ids=encode_output(raw_output),
|
||||
)
|
||||
|
||||
assert content == raw_output
|
||||
assert reasoning is None
|
||||
assert tool_calls is None
|
||||
assert harmony_parser._parser is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("harmony_str", "expected_content"),
|
||||
[
|
||||
@@ -489,6 +506,26 @@ class TestParseDelta:
|
||||
assert delta.reasoning is None
|
||||
assert not delta.tool_calls
|
||||
|
||||
def test_malformed_final_recovers_raw_content(
|
||||
self, gpt_oss_tokenizer, chat_request
|
||||
):
|
||||
parser = HarmonyParser(gpt_oss_tokenizer)
|
||||
|
||||
delta = parser.parse_delta(
|
||||
delta_text='final {"answer": "hi"}',
|
||||
delta_token_ids=encode_output(
|
||||
'<|channel|>final {"answer": "hi"}<|return|>'
|
||||
),
|
||||
request=chat_request,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
assert delta is not None
|
||||
assert delta.content == 'final {"answer": "hi"}'
|
||||
assert delta.reasoning is None
|
||||
assert not delta.tool_calls
|
||||
assert parser._parser is None
|
||||
|
||||
@pytest.mark.parametrize("tool_channel", ["commentary", "analysis"])
|
||||
def test_tool_call_split_across_deltas(
|
||||
self, gpt_oss_tokenizer, chat_request, tool_channel
|
||||
|
||||
+27
-12
@@ -3,7 +3,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
@@ -26,6 +25,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
is_function_recipient,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.reasoning.gptoss_reasoning_parser import GptOssReasoningParser
|
||||
from vllm.tool_parsers.gptoss_tool_parser import GptOssToolParser
|
||||
@@ -34,6 +34,9 @@ if TYPE_CHECKING:
|
||||
from openai_harmony import Message, StreamableParser
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class _SegmentType(Enum):
|
||||
TOOL = auto()
|
||||
REASONING = auto()
|
||||
@@ -105,16 +108,21 @@ class HarmonyParser(DelegatingParser):
|
||||
return msg
|
||||
|
||||
def flush(self) -> Segment | None:
|
||||
msg = None
|
||||
with contextlib.suppress(HarmonyError):
|
||||
try:
|
||||
self._harmony_parser.process_eos()
|
||||
# TODO: Consider reraising
|
||||
|
||||
msg = self._poll_completed_message()
|
||||
|
||||
# Reset to the initial assistant-parser state for the next turn.
|
||||
self._parser = None
|
||||
self._num_processed_messages = 0
|
||||
msg = self._poll_completed_message()
|
||||
except HarmonyError:
|
||||
logger.warning(
|
||||
"Harmony parser ended in a non-terminal state; returning the "
|
||||
"raw unparsed output. This usually indicates a malformed "
|
||||
"assistant turn, e.g. a 'final' channel missing the "
|
||||
"<|message|> delimiter."
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
# Reset to the initial assistant-parser state for the next turn.
|
||||
self._parser = None
|
||||
self._num_processed_messages = 0
|
||||
|
||||
if msg is None:
|
||||
return None
|
||||
@@ -139,7 +147,10 @@ class HarmonyParser(DelegatingParser):
|
||||
Callers must decide whether to surface them.
|
||||
"""
|
||||
result = self.process_chunk(model_output_token_ids)
|
||||
flushed_segment = self.flush()
|
||||
try:
|
||||
flushed_segment = self.flush()
|
||||
except HarmonyError:
|
||||
return None, model_output, None
|
||||
if flushed_segment is not None:
|
||||
result.segments.append(flushed_segment)
|
||||
|
||||
@@ -198,7 +209,11 @@ class HarmonyParser(DelegatingParser):
|
||||
)
|
||||
result = self.process_chunk(delta_token_ids)
|
||||
if finished:
|
||||
flushed_segment = self.flush()
|
||||
try:
|
||||
flushed_segment = self.flush()
|
||||
except HarmonyError:
|
||||
self._next_tool_call_index = 0
|
||||
return DeltaMessage(content=delta_text)
|
||||
if flushed_segment is not None:
|
||||
result.segments.append(flushed_segment)
|
||||
combined_content = ""
|
||||
|
||||
Reference in New Issue
Block a user