[Refactor][GPT-OSS] Harmony Responses API Refactor to use HarmonyParser (#47185)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-06-30 19:23:20 -04:00
committed by GitHub
parent 92c7fac640
commit b1190d03cc
9 changed files with 438 additions and 868 deletions
+116 -89
View File
@@ -1,18 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock, patch
from unittest.mock import patch
import pytest
from openai_harmony import Author, Message, Role, StreamState, TextContent
from openai_harmony import Author, Message, Role, TextContent
from vllm.entrypoints.openai.responses.context import (
HarmonyContext,
SimpleContext,
StreamingHarmonyContext,
TurnMetrics,
)
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser.harmony import ChunkResult, HarmonyParser, Segment
def create_mock_request_output(
@@ -68,25 +68,59 @@ async def generate_mock_outputs(
)
@pytest.fixture
def mock_parser():
"""Set up a mock parser for tests."""
with patch(
"vllm.entrypoints.openai.responses.context.get_streamable_parser_for_assistant"
) as mock_parser_factory:
# Create a mock parser object
parser = MagicMock()
parser.messages = []
parser.current_channel = None
parser.state = StreamState.EXPECT_START
mock_parser_factory.return_value = parser
yield parser
class FakeHarmonyParser(HarmonyParser):
def __init__(self):
# Skip HarmonyParser initialization and script outputs directly.
self.reasoning_parser = None
self.tool_parser = None
self._chunk_results: list[ChunkResult] = []
self._flush_results: list[Segment | None] = []
self.processed_chunks: list[list[int]] = []
def enqueue_chunk_result(
self,
segments: list[Segment] | None = None,
reasoning_token_count: int = 0,
) -> None:
self._chunk_results.append(
ChunkResult(
segments=[] if segments is None else segments,
reasoning_token_count=reasoning_token_count,
)
)
def enqueue_flush_result(self, segment: Segment | None) -> None:
self._flush_results.append(segment)
def process_chunk(self, token_ids) -> ChunkResult:
self.processed_chunks.append(list(token_ids))
if self._chunk_results:
return self._chunk_results.pop(0)
return ChunkResult(segments=[], reasoning_token_count=0)
def flush(self) -> Segment | None:
if self._flush_results:
return self._flush_results.pop(0)
return None
def make_harmony_context(
messages=None, available_tools=None, function_tool_names=None
) -> tuple[HarmonyContext, FakeHarmonyParser]:
fake_parser = FakeHarmonyParser()
context = HarmonyContext(
messages=[] if messages is None else messages,
available_tools=[] if available_tools is None else available_tools,
function_tool_names=function_tool_names,
response_parser=fake_parser,
)
return context, fake_parser
def test_single_turn_token_counting():
"""Test token counting behavior for a single turn."""
# Create a context
context = HarmonyContext(messages=[], available_tools=[])
context, _ = make_harmony_context()
# Create a mock RequestOutput with specific token counts
mock_output = create_mock_request_output(
@@ -118,7 +152,7 @@ def test_single_turn_token_counting():
async def test_multi_turn_token_counting():
"""Test token counting behavior across multiple turns with tool output."""
# Create a context
context = HarmonyContext(messages=[], available_tools=["browser"])
context, _ = make_harmony_context(available_tools=["browser"])
# Simulate a conversation with 3 turns
# Turn 1: prefill 5, decode 3, tool 7
@@ -177,7 +211,7 @@ async def test_multi_turn_token_counting():
def test_empty_output_tokens():
"""Test behavior when RequestOutput has empty output tokens."""
context = HarmonyContext(messages=[], available_tools=[])
context, _ = make_harmony_context()
# Create a RequestOutput with empty output tokens
mock_output = create_mock_request_output(
@@ -197,7 +231,7 @@ def test_empty_output_tokens():
def test_missing_prompt_token_ids():
"""Test behavior when RequestOutput has None prompt_token_ids."""
context = HarmonyContext(messages=[], available_tools=[])
context, _ = make_harmony_context()
mock_output = create_mock_request_output(
prompt_token_ids=None, # No prompt token IDs
@@ -216,12 +250,10 @@ def test_missing_prompt_token_ids():
assert context.num_tool_output_tokens == 0
def test_reasoning_tokens_counting(mock_parser):
def test_reasoning_tokens_counting():
"""Test that reasoning tokens are counted correctly."""
context = HarmonyContext(messages=[], available_tools=[])
# Mock parser to simulate reasoning channel
mock_parser.current_channel = "analysis" # Reasoning channel
context, parser = make_harmony_context()
parser.enqueue_chunk_result(reasoning_token_count=4)
mock_output = create_mock_request_output(
prompt_token_ids=[1, 2, 3],
@@ -236,13 +268,11 @@ def test_reasoning_tokens_counting(mock_parser):
assert context.num_output_tokens == 4
def test_preamble_tokens_not_counted_as_reasoning(mock_parser):
def test_preamble_tokens_not_counted_as_reasoning():
"""Preambles (commentary with no recipient) are visible user text,
not hidden reasoning. They must NOT inflate num_reasoning_tokens."""
context = HarmonyContext(messages=[], available_tools=[])
mock_parser.current_channel = "commentary"
mock_parser.current_recipient = None # preamble
context, parser = make_harmony_context()
parser.enqueue_chunk_result(reasoning_token_count=0)
mock_output = create_mock_request_output(
prompt_token_ids=[1, 2, 3],
@@ -255,13 +285,11 @@ def test_preamble_tokens_not_counted_as_reasoning(mock_parser):
assert context.num_output_tokens == 3
def test_commentary_with_recipient_counted_as_reasoning(mock_parser):
def test_commentary_with_recipient_counted_as_reasoning():
"""Commentary directed at a tool (recipient != None) is hidden from
the user, so it should still count as reasoning tokens."""
context = HarmonyContext(messages=[], available_tools=[])
mock_parser.current_channel = "commentary"
mock_parser.current_recipient = "python"
context, parser = make_harmony_context()
parser.enqueue_chunk_result(reasoning_token_count=3)
mock_output = create_mock_request_output(
prompt_token_ids=[1, 2, 3],
@@ -276,7 +304,7 @@ def test_commentary_with_recipient_counted_as_reasoning(mock_parser):
def test_zero_tokens_edge_case():
"""Test behavior with all zero token counts."""
context = HarmonyContext(messages=[], available_tools=[])
context, _ = make_harmony_context()
# Create a request with empty lists (not None) for both prompt and
# output tokens
@@ -299,10 +327,7 @@ def test_zero_tokens_edge_case():
@pytest.mark.asyncio
async def test_single_turn_no_tool_output():
"""Test that first turn never generates tool output tokens."""
context = HarmonyContext(
messages=[],
available_tools=["browser"], # Tools available
)
context, _ = make_harmony_context(available_tools=["browser"])
# Even with large prompt in first turn, no tool tokens should be counted
mock_output = create_mock_request_output(
@@ -324,7 +349,7 @@ async def test_negative_tool_tokens_edge_case():
tokens. We should log an error and clamp the value to 0."""
# Use patch to check if logger.error was called
with patch("vllm.entrypoints.openai.responses.context.logger.error") as mock_log:
context = HarmonyContext(messages=[], available_tools=["browser"])
context, _ = make_harmony_context(available_tools=["browser"])
# First turn
mock_output1 = create_mock_request_output(
@@ -360,15 +385,15 @@ async def test_negative_tool_tokens_edge_case():
@pytest.mark.asyncio
async def test_streaming_multi_turn_token_counting(mock_parser):
async def test_streaming_multi_turn_token_counting():
"""Test token counting for streaming multi-turn conversations.
This test focuses on how StreamingHarmonyContext counts tokens in a
This test focuses on how HarmonyContext counts tokens in a
multi-turn conversation with streaming (token-by-token) outputs and
message boundaries.
"""
# Create a streaming context
context = StreamingHarmonyContext(messages=[], available_tools=["browser"])
context, parser = make_harmony_context(available_tools=["browser"])
num_prompt_tokens = [3, 8, 13]
num_output_tokens = [3, 3, 2]
@@ -413,10 +438,8 @@ async def test_streaming_multi_turn_token_counting(mock_parser):
assert context.num_tool_output_tokens == 0 # No tool output in first turn
assert context.first_tok_of_message is True # Ready for next message
# Second turn: reasoning tokens in analysis channel
mock_parser.current_channel = "analysis" # Set to reasoning channel
# First token of second turn
parser.enqueue_chunk_result(reasoning_token_count=1)
context.append_output(
create_mock_request_output(
prompt_token_ids=[
@@ -436,6 +459,7 @@ async def test_streaming_multi_turn_token_counting(mock_parser):
)
# More tokens in reasoning channel
parser.enqueue_chunk_result(reasoning_token_count=1)
context.append_output(
create_mock_request_output(
output_token_ids=[202],
@@ -443,6 +467,7 @@ async def test_streaming_multi_turn_token_counting(mock_parser):
)
)
parser.enqueue_chunk_result(reasoning_token_count=1)
context.append_output(
create_mock_request_output(
output_token_ids=[203],
@@ -460,9 +485,6 @@ async def test_streaming_multi_turn_token_counting(mock_parser):
expected_tool_tokens = 8 - 3 - 3 # = 2
assert context.num_tool_output_tokens == expected_tool_tokens
# Third turn: regular output channel
mock_parser.current_channel = "final" # Switch back to regular channel
# Third turn (with more cached tokens)
context.append_output(
create_mock_request_output(
@@ -520,13 +542,8 @@ async def test_streaming_multi_turn_token_counting(mock_parser):
@pytest.mark.asyncio
async def test_streaming_message_synchronization(mock_parser):
"""Test message synchronization logic from lines 413-417 in context.py.
This test verifies that when parser.messages contains more messages than
the context's _messages (minus initial messages), the context properly
extends its message list with the new parser messages.
"""
async def test_streaming_message_synchronization():
"""Completed messages from append-local and flush segments sync into context."""
# Create a streaming context with some initial messages
initial_messages = [
@@ -536,23 +553,30 @@ async def test_streaming_message_synchronization(mock_parser):
recipient=Role.ASSISTANT,
)
]
context = StreamingHarmonyContext(messages=initial_messages, available_tools=[])
context, parser = make_harmony_context(messages=initial_messages)
# Verify initial state
assert len(context._messages) == 1
assert context.num_init_messages == 1
# Mock parser to have more messages than context
# Simulate parser having processed 3 new messages
mock_parser.messages = [
Message(
author=Author(role=Role.ASSISTANT, name="assistant"),
content=[TextContent(text="Response 1")],
recipient=Role.USER,
),
]
response_text = "First response"
message = Message(
author=Author(role=Role.ASSISTANT, name="assistant"),
content=[TextContent(text=response_text)],
recipient=Role.USER,
)
parser.enqueue_chunk_result(
segments=[
Segment(
channel="commentary",
recipient=None,
delta="",
completed_message=message,
)
]
)
# This should trigger the message synchronization logic
# This should sync the completed message from the latest append
context.append_output(
create_mock_request_output(
prompt_token_ids=[1, 2, 3], output_token_ids=[101], finished=False
@@ -563,36 +587,39 @@ async def test_streaming_message_synchronization(mock_parser):
assert len(context._messages) == 2
# Verify the new messages were added correctly
assert context._messages[1].content[0].text == "Response 1"
assert context._messages[1].content[0].text == response_text
# Test the specific condition from line 413-414:
# len(self._messages) - self.num_init_messages < len(self.parser.messages)
messages_minus_init = len(context._messages) - context.num_init_messages
parser_messages_count = len(mock_parser.messages)
assert messages_minus_init == 1
# After synchronization, they should be equal (no longer less than)
assert messages_minus_init == parser_messages_count
response_text = "Second response"
message = Message(
author=Author(role=Role.ASSISTANT, name="assistant"),
content=[TextContent(text=response_text)],
recipient=Role.USER,
)
flush_segment = Segment(
channel="commentary",
recipient=None,
delta="",
completed_message=message,
)
parser.enqueue_flush_result(flush_segment)
# Test edge case: add one more parser message
mock_parser.messages.append(
Message(
author=Author(role=Role.ASSISTANT, name="assistant"),
content=[TextContent(text="Response 4")],
recipient=Role.USER,
# Create another output to trigger synchronization via flush()
context.append_output(
create_mock_request_output(
prompt_token_ids=[1, 2, 3], output_token_ids=[102], finished=True
)
)
# Create another output to trigger synchronization again
mock_output2 = create_mock_request_output(
prompt_token_ids=[1, 2, 3], output_token_ids=[102], finished=True
)
context.append_output(mock_output2)
# Verify the fourth message was added, num_init_messages is still 1
# Verify the flushed response was added, num_init_messages is still 1
assert len(context._messages) == 3
assert context.num_init_messages == 1
assert context._messages[2].content[0].text == "Response 4"
assert context._messages[2].content[0].text == response_text
assert context.last_append_flush_status is True
assert len(context.last_append_segments) == 1
assert context.last_append_segments[0].completed_message is message
def test_turn_metrics_copy_and_reset():