From f59db63732e5c4f72e64d524cecc87e6fe0c4453 Mon Sep 17 00:00:00 2001 From: yzong-rh Date: Tue, 23 Jun 2026 09:36:33 -0400 Subject: [PATCH] [Bugfix] GPT-OSS Autodrop reasoning in Response API and cleanup (#45048) Signed-off-by: Yifan Zong Co-authored-by: Ben Browning --- .../chat_completion/test_serving_chat.py | 38 ++++-- .../parser/test_harmony_render_parity.py | 112 ++++++++++++++++-- .../openai/parser/harmony_utils.py | 7 +- vllm/entrypoints/openai/responses/harmony.py | 2 +- vllm/entrypoints/openai/responses/serving.py | 24 ---- 5 files changed, 137 insertions(+), 46 deletions(-) diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 3802b7e3e5..7b480bdba6 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -1429,15 +1429,23 @@ class TestServingChatWithHarmony: input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony( req_2 ) + expected_input_messages_2 = [ + {"role": "system"}, + {"role": "user"}, + ] + if include_reasoning: + expected_input_messages_2.append( + { + "role": "assistant", + "channel": "analysis", + } + ) + expected_input_messages_2.append( + {"role": "assistant", "channel": "final", "content": final_str} + ) verify_harmony_messages( input_messages_2, - [ - {"role": "system"}, - {"role": "user"}, - # The analysis message should be dropped on subsequent inputs because - # of the subsequent assistant message to the final channel. - {"role": "assistant", "channel": "final", "content": final_str}, - ], + expected_input_messages_2, ) @pytest.mark.asyncio @@ -1635,7 +1643,6 @@ class TestServingChatWithHarmony: { "role": "assistant", "channel": "analysis", - "content": reasoning_str, }, { "role": "assistant", @@ -1684,6 +1691,11 @@ class TestServingChatWithHarmony: {"role": "system"}, {"role": "developer"}, {"role": "user"}, + { + "role": "assistant", + "channel": "analysis", + "content": reasoning_str, + }, { "role": "assistant", "channel": "commentary", @@ -1749,6 +1761,10 @@ class TestServingChatWithHarmony: {"role": "system"}, {"role": "developer"}, {"role": "user"}, + { + "role": "assistant", + "channel": "analysis", + }, {"role": "assistant"}, {"role": "tool"}, { @@ -1798,8 +1814,10 @@ class TestServingChatWithHarmony: [ {"role": "system"}, {"role": "user", "content": messages[0]["content"]}, - # The reasoning that would have resulted in an analysis message is - # dropped because of a later assistant message to the final channel. + { + "role": "assistant", + "channel": "analysis", + }, { "role": "assistant", "channel": "final", diff --git a/tests/entrypoints/openai/parser/test_harmony_render_parity.py b/tests/entrypoints/openai/parser/test_harmony_render_parity.py index 5b771ff7bb..5cb4461223 100644 --- a/tests/entrypoints/openai/parser/test_harmony_render_parity.py +++ b/tests/entrypoints/openai/parser/test_harmony_render_parity.py @@ -23,11 +23,15 @@ from openai.types.responses import ResponseFunctionToolCall from tests.entrypoints.openai.utils import verify_harmony_messages from vllm.entrypoints.openai.parser.harmony_utils import ( + get_encoding, get_system_message, parse_chat_input_to_harmony_message, render_for_completion, ) -from vllm.entrypoints.openai.responses.harmony import response_input_to_harmony +from vllm.entrypoints.openai.responses.harmony import ( + response_input_to_harmony, + response_previous_input_to_harmony, +) # Use a fixed date so the system message is deterministic across both paths. _DATE = "2025-01-01" @@ -395,6 +399,9 @@ class TestResponseInputToHarmonyRenderParity: reasoning trace. Reasoning traces in between commentary-channel tool calls must survive as analysis-channel messages in both paths. """ + first_reasoning = "I need current weather first." + second_reasoning = "Now I need the weekly forecast." + prev_call_1 = ResponseFunctionToolCall( id="fc_1", call_id="call_1", @@ -420,7 +427,7 @@ class TestResponseInputToHarmonyRenderParity: chat_msgs += parse_chat_input_to_harmony_message( { "role": "assistant", - "reasoning": "I need current weather first.", + "reasoning": first_reasoning, "tool_calls": [ { "id": "call_1", @@ -440,7 +447,7 @@ class TestResponseInputToHarmonyRenderParity: chat_msgs += parse_chat_input_to_harmony_message( { "role": "assistant", - "reasoning": "Now I need the weekly forecast.", + "reasoning": second_reasoning, "tool_calls": [ { "id": "call_2", @@ -472,9 +479,7 @@ class TestResponseInputToHarmonyRenderParity: # First reasoning + tool call { "type": "reasoning", - "content": [ - {"type": "reasoning_text", "text": "I need current weather first."} - ], + "content": [{"type": "reasoning_text", "text": first_reasoning}], }, { "type": "function_call", @@ -492,7 +497,7 @@ class TestResponseInputToHarmonyRenderParity: "content": [ { "type": "reasoning_text", - "text": "Now I need the weekly forecast.", + "text": second_reasoning, } ], }, @@ -512,6 +517,95 @@ class TestResponseInputToHarmonyRenderParity: for item in resp_input ] - assert render_for_completion([_system()] + chat_msgs) == render_for_completion( - [_system()] + resp_msgs + chat_completion_tokens = render_for_completion([_system()] + chat_msgs) + responses_tokens = render_for_completion([_system()] + resp_msgs) + + assert chat_completion_tokens == responses_tokens + + rendered_prompt = get_encoding().decode(chat_completion_tokens) + assert first_reasoning in rendered_prompt + assert second_reasoning in rendered_prompt + + def test_completed_turns_drop_reasoning(self): + """Validates that reasoning from completed turns is dropped, while + reasoning from the current in-progress tool-call turn is preserved + in both chat completions and responses previous_input_messages.""" + first_turn_reasoning = "FIRST_TURN_REASONING" + second_turn_reasoning = "SECOND_TURN_REASONING" + + chat_completion_msgs = [] + for chat_message in [ + {"role": "user", "content": "What is 2+2?"}, + { + "role": "assistant", + "reasoning": first_turn_reasoning, + "content": "The answer is 4.", + }, + {"role": "user", "content": "Now what is 3+3?"}, + { + "role": "assistant", + "reasoning": second_turn_reasoning, + "tool_calls": [ + { + "id": "call_1", + "function": { + "name": "calc", + "arguments": '{"a":3,"b":3}', + }, + } + ], + }, + ]: + chat_completion_msgs.extend( + parse_chat_input_to_harmony_message(chat_message) + ) + + responses_prev_input_msgs = [] + for responses_message in [ + { + "author": {"role": "user"}, + "content": [{"type": "text", "text": "What is 2+2?"}], + }, + { + "author": {"role": "assistant"}, + "channel": "analysis", + "content": [{"type": "text", "text": first_turn_reasoning}], + }, + { + "author": {"role": "assistant"}, + "channel": "final", + "content": [{"type": "text", "text": "The answer is 4."}], + }, + { + "author": {"role": "user"}, + "content": [{"type": "text", "text": "Now what is 3+3?"}], + }, + { + "author": {"role": "assistant"}, + "channel": "analysis", + "content": [{"type": "text", "text": second_turn_reasoning}], + }, + { + "author": {"role": "assistant"}, + "channel": "commentary", + "recipient": "functions.calc", + "content_type": "json", + "content": [{"type": "text", "text": '{"a":3,"b":3}'}], + }, + ]: + responses_prev_input_msgs.extend( + response_previous_input_to_harmony(responses_message) + ) + + chat_completion_tokens = render_for_completion( + [_system()] + chat_completion_msgs ) + responses_tokens = render_for_completion( + [_system()] + responses_prev_input_msgs + ) + + assert chat_completion_tokens == responses_tokens + + rendered_prompt = get_encoding().decode(responses_tokens) + assert first_turn_reasoning not in rendered_prompt + assert second_turn_reasoning in rendered_prompt diff --git a/vllm/entrypoints/openai/parser/harmony_utils.py b/vllm/entrypoints/openai/parser/harmony_utils.py index 82316efb86..a18a93704c 100644 --- a/vllm/entrypoints/openai/parser/harmony_utils.py +++ b/vllm/entrypoints/openai/parser/harmony_utils.py @@ -13,6 +13,7 @@ from openai_harmony import ( HarmonyEncodingName, Message, ReasoningEffort, + RenderConversationConfig, Role, StreamableParser, SystemContent, @@ -220,7 +221,6 @@ def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]: for chat_msg in chat_msgs: msgs.extend(parse_chat_input_to_harmony_message(chat_msg, tool_id_names)) - msgs = auto_drop_analysis_messages(msgs) return msgs @@ -447,9 +447,12 @@ def parse_chat_input_to_harmony_message( def render_for_completion(messages: list[Message]) -> list[int]: + messages = auto_drop_analysis_messages(messages) conversation = Conversation.from_messages(messages) token_ids = get_encoding().render_conversation_for_completion( - conversation, Role.ASSISTANT + conversation, + Role.ASSISTANT, + config=RenderConversationConfig(auto_drop_analysis=False), ) return token_ids diff --git a/vllm/entrypoints/openai/responses/harmony.py b/vllm/entrypoints/openai/responses/harmony.py index 8dee0d993d..562b1d201e 100644 --- a/vllm/entrypoints/openai/responses/harmony.py +++ b/vllm/entrypoints/openai/responses/harmony.py @@ -67,7 +67,7 @@ def _parse_harmony_format_message(chat_msg: dict) -> Message: contents = [TextContent(text="")] if name: - msg = Message.from_author_and_contents(Author.new(Role(role), name), contents) + msg = Message(author=Author.new(Role(role), name), content=contents) else: msg = Message.from_role_and_contents(Role(role), contents) diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 0f37f3f0f3..62af1953dd 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -1191,30 +1191,6 @@ class OpenAIServingResponses(OpenAIServing): # instructions are ignored. prev_msgs = self.msg_store[prev_response.id] - # FIXME(woosuk): The slice-delete-reappend cycle below is - # currently a no-op --- it removes messages then puts them all - # back unfiltered. It may be intentionally deferred (see FIXME - # above) or redundant if the Harmony encoder already strips - # analysis messages at render time. If analysis messages need - # to be dropped here, add a channel != "analysis" filter when - # re-appending, similar to auto_drop_analysis_messages in - # harmony_utils.py. - if len(prev_msgs) > 0: - last_msg = prev_msgs[-1] - assert isinstance(last_msg, OpenAIHarmonyMessage) - if last_msg.channel == "final": - prev_final_msg_idx = -1 - for i in range(len(prev_msgs) - 2, -1, -1): - prev_msg_i = prev_msgs[i] - assert isinstance(prev_msg_i, OpenAIHarmonyMessage) - if prev_msg_i.channel == "final": - prev_final_msg_idx = i - break - recent_turn_msgs = prev_msgs[prev_final_msg_idx + 1 :] - del prev_msgs[prev_final_msg_idx + 1 :] - for msg in recent_turn_msgs: - assert isinstance(msg, OpenAIHarmonyMessage) - prev_msgs.append(msg) messages.extend(prev_msgs) # Append the new input. # Responses API supports simple text inputs without chat format.