[Bugfix] GPT-OSS Autodrop reasoning in Response API and cleanup (#45048)

Signed-off-by: Yifan Zong <yzong@redhat.com>
Co-authored-by: Ben Browning <bbrownin@redhat.com>
This commit is contained in:
yzong-rh
2026-06-23 09:36:33 -04:00
committed by GitHub
co-authored by Ben Browning
parent 9f5117820f
commit f59db63732
5 changed files with 137 additions and 46 deletions
@@ -1429,15 +1429,23 @@ class TestServingChatWithHarmony:
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
req_2
)
expected_input_messages_2 = [
{"role": "system"},
{"role": "user"},
]
if include_reasoning:
expected_input_messages_2.append(
{
"role": "assistant",
"channel": "analysis",
}
)
expected_input_messages_2.append(
{"role": "assistant", "channel": "final", "content": final_str}
)
verify_harmony_messages(
input_messages_2,
[
{"role": "system"},
{"role": "user"},
# The analysis message should be dropped on subsequent inputs because
# of the subsequent assistant message to the final channel.
{"role": "assistant", "channel": "final", "content": final_str},
],
expected_input_messages_2,
)
@pytest.mark.asyncio
@@ -1635,7 +1643,6 @@ class TestServingChatWithHarmony:
{
"role": "assistant",
"channel": "analysis",
"content": reasoning_str,
},
{
"role": "assistant",
@@ -1684,6 +1691,11 @@ class TestServingChatWithHarmony:
{"role": "system"},
{"role": "developer"},
{"role": "user"},
{
"role": "assistant",
"channel": "analysis",
"content": reasoning_str,
},
{
"role": "assistant",
"channel": "commentary",
@@ -1749,6 +1761,10 @@ class TestServingChatWithHarmony:
{"role": "system"},
{"role": "developer"},
{"role": "user"},
{
"role": "assistant",
"channel": "analysis",
},
{"role": "assistant"},
{"role": "tool"},
{
@@ -1798,8 +1814,10 @@ class TestServingChatWithHarmony:
[
{"role": "system"},
{"role": "user", "content": messages[0]["content"]},
# The reasoning that would have resulted in an analysis message is
# dropped because of a later assistant message to the final channel.
{
"role": "assistant",
"channel": "analysis",
},
{
"role": "assistant",
"channel": "final",
@@ -23,11 +23,15 @@ from openai.types.responses import ResponseFunctionToolCall
from tests.entrypoints.openai.utils import verify_harmony_messages
from vllm.entrypoints.openai.parser.harmony_utils import (
get_encoding,
get_system_message,
parse_chat_input_to_harmony_message,
render_for_completion,
)
from vllm.entrypoints.openai.responses.harmony import response_input_to_harmony
from vllm.entrypoints.openai.responses.harmony import (
response_input_to_harmony,
response_previous_input_to_harmony,
)
# Use a fixed date so the system message is deterministic across both paths.
_DATE = "2025-01-01"
@@ -395,6 +399,9 @@ class TestResponseInputToHarmonyRenderParity:
reasoning trace. Reasoning traces in between commentary-channel tool
calls must survive as analysis-channel messages in both paths.
"""
first_reasoning = "I need current weather first."
second_reasoning = "Now I need the weekly forecast."
prev_call_1 = ResponseFunctionToolCall(
id="fc_1",
call_id="call_1",
@@ -420,7 +427,7 @@ class TestResponseInputToHarmonyRenderParity:
chat_msgs += parse_chat_input_to_harmony_message(
{
"role": "assistant",
"reasoning": "I need current weather first.",
"reasoning": first_reasoning,
"tool_calls": [
{
"id": "call_1",
@@ -440,7 +447,7 @@ class TestResponseInputToHarmonyRenderParity:
chat_msgs += parse_chat_input_to_harmony_message(
{
"role": "assistant",
"reasoning": "Now I need the weekly forecast.",
"reasoning": second_reasoning,
"tool_calls": [
{
"id": "call_2",
@@ -472,9 +479,7 @@ class TestResponseInputToHarmonyRenderParity:
# First reasoning + tool call
{
"type": "reasoning",
"content": [
{"type": "reasoning_text", "text": "I need current weather first."}
],
"content": [{"type": "reasoning_text", "text": first_reasoning}],
},
{
"type": "function_call",
@@ -492,7 +497,7 @@ class TestResponseInputToHarmonyRenderParity:
"content": [
{
"type": "reasoning_text",
"text": "Now I need the weekly forecast.",
"text": second_reasoning,
}
],
},
@@ -512,6 +517,95 @@ class TestResponseInputToHarmonyRenderParity:
for item in resp_input
]
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
[_system()] + resp_msgs
chat_completion_tokens = render_for_completion([_system()] + chat_msgs)
responses_tokens = render_for_completion([_system()] + resp_msgs)
assert chat_completion_tokens == responses_tokens
rendered_prompt = get_encoding().decode(chat_completion_tokens)
assert first_reasoning in rendered_prompt
assert second_reasoning in rendered_prompt
def test_completed_turns_drop_reasoning(self):
"""Validates that reasoning from completed turns is dropped, while
reasoning from the current in-progress tool-call turn is preserved
in both chat completions and responses previous_input_messages."""
first_turn_reasoning = "FIRST_TURN_REASONING"
second_turn_reasoning = "SECOND_TURN_REASONING"
chat_completion_msgs = []
for chat_message in [
{"role": "user", "content": "What is 2+2?"},
{
"role": "assistant",
"reasoning": first_turn_reasoning,
"content": "The answer is 4.",
},
{"role": "user", "content": "Now what is 3+3?"},
{
"role": "assistant",
"reasoning": second_turn_reasoning,
"tool_calls": [
{
"id": "call_1",
"function": {
"name": "calc",
"arguments": '{"a":3,"b":3}',
},
}
],
},
]:
chat_completion_msgs.extend(
parse_chat_input_to_harmony_message(chat_message)
)
responses_prev_input_msgs = []
for responses_message in [
{
"author": {"role": "user"},
"content": [{"type": "text", "text": "What is 2+2?"}],
},
{
"author": {"role": "assistant"},
"channel": "analysis",
"content": [{"type": "text", "text": first_turn_reasoning}],
},
{
"author": {"role": "assistant"},
"channel": "final",
"content": [{"type": "text", "text": "The answer is 4."}],
},
{
"author": {"role": "user"},
"content": [{"type": "text", "text": "Now what is 3+3?"}],
},
{
"author": {"role": "assistant"},
"channel": "analysis",
"content": [{"type": "text", "text": second_turn_reasoning}],
},
{
"author": {"role": "assistant"},
"channel": "commentary",
"recipient": "functions.calc",
"content_type": "json",
"content": [{"type": "text", "text": '{"a":3,"b":3}'}],
},
]:
responses_prev_input_msgs.extend(
response_previous_input_to_harmony(responses_message)
)
chat_completion_tokens = render_for_completion(
[_system()] + chat_completion_msgs
)
responses_tokens = render_for_completion(
[_system()] + responses_prev_input_msgs
)
assert chat_completion_tokens == responses_tokens
rendered_prompt = get_encoding().decode(responses_tokens)
assert first_turn_reasoning not in rendered_prompt
assert second_turn_reasoning in rendered_prompt
@@ -13,6 +13,7 @@ from openai_harmony import (
HarmonyEncodingName,
Message,
ReasoningEffort,
RenderConversationConfig,
Role,
StreamableParser,
SystemContent,
@@ -220,7 +221,6 @@ def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:
for chat_msg in chat_msgs:
msgs.extend(parse_chat_input_to_harmony_message(chat_msg, tool_id_names))
msgs = auto_drop_analysis_messages(msgs)
return msgs
@@ -447,9 +447,12 @@ def parse_chat_input_to_harmony_message(
def render_for_completion(messages: list[Message]) -> list[int]:
messages = auto_drop_analysis_messages(messages)
conversation = Conversation.from_messages(messages)
token_ids = get_encoding().render_conversation_for_completion(
conversation, Role.ASSISTANT
conversation,
Role.ASSISTANT,
config=RenderConversationConfig(auto_drop_analysis=False),
)
return token_ids
+1 -1
View File
@@ -67,7 +67,7 @@ def _parse_harmony_format_message(chat_msg: dict) -> Message:
contents = [TextContent(text="")]
if name:
msg = Message.from_author_and_contents(Author.new(Role(role), name), contents)
msg = Message(author=Author.new(Role(role), name), content=contents)
else:
msg = Message.from_role_and_contents(Role(role), contents)
@@ -1191,30 +1191,6 @@ class OpenAIServingResponses(OpenAIServing):
# instructions are ignored.
prev_msgs = self.msg_store[prev_response.id]
# FIXME(woosuk): The slice-delete-reappend cycle below is
# currently a no-op --- it removes messages then puts them all
# back unfiltered. It may be intentionally deferred (see FIXME
# above) or redundant if the Harmony encoder already strips
# analysis messages at render time. If analysis messages need
# to be dropped here, add a channel != "analysis" filter when
# re-appending, similar to auto_drop_analysis_messages in
# harmony_utils.py.
if len(prev_msgs) > 0:
last_msg = prev_msgs[-1]
assert isinstance(last_msg, OpenAIHarmonyMessage)
if last_msg.channel == "final":
prev_final_msg_idx = -1
for i in range(len(prev_msgs) - 2, -1, -1):
prev_msg_i = prev_msgs[i]
assert isinstance(prev_msg_i, OpenAIHarmonyMessage)
if prev_msg_i.channel == "final":
prev_final_msg_idx = i
break
recent_turn_msgs = prev_msgs[prev_final_msg_idx + 1 :]
del prev_msgs[prev_final_msg_idx + 1 :]
for msg in recent_turn_msgs:
assert isinstance(msg, OpenAIHarmonyMessage)
prev_msgs.append(msg)
messages.extend(prev_msgs)
# Append the new input.
# Responses API supports simple text inputs without chat format.