[Bugfix] GPT-OSS Autodrop reasoning in Response API and cleanup (#45048)
Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: Ben Browning <bbrownin@redhat.com>
This commit is contained in:
@@ -1429,15 +1429,23 @@ class TestServingChatWithHarmony:
|
||||
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_2
|
||||
)
|
||||
expected_input_messages_2 = [
|
||||
{"role": "system"},
|
||||
{"role": "user"},
|
||||
]
|
||||
if include_reasoning:
|
||||
expected_input_messages_2.append(
|
||||
{
|
||||
"role": "assistant",
|
||||
"channel": "analysis",
|
||||
}
|
||||
)
|
||||
expected_input_messages_2.append(
|
||||
{"role": "assistant", "channel": "final", "content": final_str}
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_2,
|
||||
[
|
||||
{"role": "system"},
|
||||
{"role": "user"},
|
||||
# The analysis message should be dropped on subsequent inputs because
|
||||
# of the subsequent assistant message to the final channel.
|
||||
{"role": "assistant", "channel": "final", "content": final_str},
|
||||
],
|
||||
expected_input_messages_2,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1635,7 +1643,6 @@ class TestServingChatWithHarmony:
|
||||
{
|
||||
"role": "assistant",
|
||||
"channel": "analysis",
|
||||
"content": reasoning_str,
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
@@ -1684,6 +1691,11 @@ class TestServingChatWithHarmony:
|
||||
{"role": "system"},
|
||||
{"role": "developer"},
|
||||
{"role": "user"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"channel": "analysis",
|
||||
"content": reasoning_str,
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"channel": "commentary",
|
||||
@@ -1749,6 +1761,10 @@ class TestServingChatWithHarmony:
|
||||
{"role": "system"},
|
||||
{"role": "developer"},
|
||||
{"role": "user"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"channel": "analysis",
|
||||
},
|
||||
{"role": "assistant"},
|
||||
{"role": "tool"},
|
||||
{
|
||||
@@ -1798,8 +1814,10 @@ class TestServingChatWithHarmony:
|
||||
[
|
||||
{"role": "system"},
|
||||
{"role": "user", "content": messages[0]["content"]},
|
||||
# The reasoning that would have resulted in an analysis message is
|
||||
# dropped because of a later assistant message to the final channel.
|
||||
{
|
||||
"role": "assistant",
|
||||
"channel": "analysis",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"channel": "final",
|
||||
|
||||
@@ -23,11 +23,15 @@ from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
from tests.entrypoints.openai.utils import verify_harmony_messages
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
get_encoding,
|
||||
get_system_message,
|
||||
parse_chat_input_to_harmony_message,
|
||||
render_for_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.harmony import response_input_to_harmony
|
||||
from vllm.entrypoints.openai.responses.harmony import (
|
||||
response_input_to_harmony,
|
||||
response_previous_input_to_harmony,
|
||||
)
|
||||
|
||||
# Use a fixed date so the system message is deterministic across both paths.
|
||||
_DATE = "2025-01-01"
|
||||
@@ -395,6 +399,9 @@ class TestResponseInputToHarmonyRenderParity:
|
||||
reasoning trace. Reasoning traces in between commentary-channel tool
|
||||
calls must survive as analysis-channel messages in both paths.
|
||||
"""
|
||||
first_reasoning = "I need current weather first."
|
||||
second_reasoning = "Now I need the weekly forecast."
|
||||
|
||||
prev_call_1 = ResponseFunctionToolCall(
|
||||
id="fc_1",
|
||||
call_id="call_1",
|
||||
@@ -420,7 +427,7 @@ class TestResponseInputToHarmonyRenderParity:
|
||||
chat_msgs += parse_chat_input_to_harmony_message(
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning": "I need current weather first.",
|
||||
"reasoning": first_reasoning,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
@@ -440,7 +447,7 @@ class TestResponseInputToHarmonyRenderParity:
|
||||
chat_msgs += parse_chat_input_to_harmony_message(
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning": "Now I need the weekly forecast.",
|
||||
"reasoning": second_reasoning,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_2",
|
||||
@@ -472,9 +479,7 @@ class TestResponseInputToHarmonyRenderParity:
|
||||
# First reasoning + tool call
|
||||
{
|
||||
"type": "reasoning",
|
||||
"content": [
|
||||
{"type": "reasoning_text", "text": "I need current weather first."}
|
||||
],
|
||||
"content": [{"type": "reasoning_text", "text": first_reasoning}],
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
@@ -492,7 +497,7 @@ class TestResponseInputToHarmonyRenderParity:
|
||||
"content": [
|
||||
{
|
||||
"type": "reasoning_text",
|
||||
"text": "Now I need the weekly forecast.",
|
||||
"text": second_reasoning,
|
||||
}
|
||||
],
|
||||
},
|
||||
@@ -512,6 +517,95 @@ class TestResponseInputToHarmonyRenderParity:
|
||||
for item in resp_input
|
||||
]
|
||||
|
||||
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
|
||||
[_system()] + resp_msgs
|
||||
chat_completion_tokens = render_for_completion([_system()] + chat_msgs)
|
||||
responses_tokens = render_for_completion([_system()] + resp_msgs)
|
||||
|
||||
assert chat_completion_tokens == responses_tokens
|
||||
|
||||
rendered_prompt = get_encoding().decode(chat_completion_tokens)
|
||||
assert first_reasoning in rendered_prompt
|
||||
assert second_reasoning in rendered_prompt
|
||||
|
||||
def test_completed_turns_drop_reasoning(self):
|
||||
"""Validates that reasoning from completed turns is dropped, while
|
||||
reasoning from the current in-progress tool-call turn is preserved
|
||||
in both chat completions and responses previous_input_messages."""
|
||||
first_turn_reasoning = "FIRST_TURN_REASONING"
|
||||
second_turn_reasoning = "SECOND_TURN_REASONING"
|
||||
|
||||
chat_completion_msgs = []
|
||||
for chat_message in [
|
||||
{"role": "user", "content": "What is 2+2?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning": first_turn_reasoning,
|
||||
"content": "The answer is 4.",
|
||||
},
|
||||
{"role": "user", "content": "Now what is 3+3?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning": second_turn_reasoning,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"function": {
|
||||
"name": "calc",
|
||||
"arguments": '{"a":3,"b":3}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
]:
|
||||
chat_completion_msgs.extend(
|
||||
parse_chat_input_to_harmony_message(chat_message)
|
||||
)
|
||||
|
||||
responses_prev_input_msgs = []
|
||||
for responses_message in [
|
||||
{
|
||||
"author": {"role": "user"},
|
||||
"content": [{"type": "text", "text": "What is 2+2?"}],
|
||||
},
|
||||
{
|
||||
"author": {"role": "assistant"},
|
||||
"channel": "analysis",
|
||||
"content": [{"type": "text", "text": first_turn_reasoning}],
|
||||
},
|
||||
{
|
||||
"author": {"role": "assistant"},
|
||||
"channel": "final",
|
||||
"content": [{"type": "text", "text": "The answer is 4."}],
|
||||
},
|
||||
{
|
||||
"author": {"role": "user"},
|
||||
"content": [{"type": "text", "text": "Now what is 3+3?"}],
|
||||
},
|
||||
{
|
||||
"author": {"role": "assistant"},
|
||||
"channel": "analysis",
|
||||
"content": [{"type": "text", "text": second_turn_reasoning}],
|
||||
},
|
||||
{
|
||||
"author": {"role": "assistant"},
|
||||
"channel": "commentary",
|
||||
"recipient": "functions.calc",
|
||||
"content_type": "json",
|
||||
"content": [{"type": "text", "text": '{"a":3,"b":3}'}],
|
||||
},
|
||||
]:
|
||||
responses_prev_input_msgs.extend(
|
||||
response_previous_input_to_harmony(responses_message)
|
||||
)
|
||||
|
||||
chat_completion_tokens = render_for_completion(
|
||||
[_system()] + chat_completion_msgs
|
||||
)
|
||||
responses_tokens = render_for_completion(
|
||||
[_system()] + responses_prev_input_msgs
|
||||
)
|
||||
|
||||
assert chat_completion_tokens == responses_tokens
|
||||
|
||||
rendered_prompt = get_encoding().decode(responses_tokens)
|
||||
assert first_turn_reasoning not in rendered_prompt
|
||||
assert second_turn_reasoning in rendered_prompt
|
||||
|
||||
@@ -13,6 +13,7 @@ from openai_harmony import (
|
||||
HarmonyEncodingName,
|
||||
Message,
|
||||
ReasoningEffort,
|
||||
RenderConversationConfig,
|
||||
Role,
|
||||
StreamableParser,
|
||||
SystemContent,
|
||||
@@ -220,7 +221,6 @@ def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:
|
||||
for chat_msg in chat_msgs:
|
||||
msgs.extend(parse_chat_input_to_harmony_message(chat_msg, tool_id_names))
|
||||
|
||||
msgs = auto_drop_analysis_messages(msgs)
|
||||
return msgs
|
||||
|
||||
|
||||
@@ -447,9 +447,12 @@ def parse_chat_input_to_harmony_message(
|
||||
|
||||
|
||||
def render_for_completion(messages: list[Message]) -> list[int]:
|
||||
messages = auto_drop_analysis_messages(messages)
|
||||
conversation = Conversation.from_messages(messages)
|
||||
token_ids = get_encoding().render_conversation_for_completion(
|
||||
conversation, Role.ASSISTANT
|
||||
conversation,
|
||||
Role.ASSISTANT,
|
||||
config=RenderConversationConfig(auto_drop_analysis=False),
|
||||
)
|
||||
return token_ids
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@ def _parse_harmony_format_message(chat_msg: dict) -> Message:
|
||||
contents = [TextContent(text="")]
|
||||
|
||||
if name:
|
||||
msg = Message.from_author_and_contents(Author.new(Role(role), name), contents)
|
||||
msg = Message(author=Author.new(Role(role), name), content=contents)
|
||||
else:
|
||||
msg = Message.from_role_and_contents(Role(role), contents)
|
||||
|
||||
|
||||
@@ -1191,30 +1191,6 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
# instructions are ignored.
|
||||
prev_msgs = self.msg_store[prev_response.id]
|
||||
|
||||
# FIXME(woosuk): The slice-delete-reappend cycle below is
|
||||
# currently a no-op --- it removes messages then puts them all
|
||||
# back unfiltered. It may be intentionally deferred (see FIXME
|
||||
# above) or redundant if the Harmony encoder already strips
|
||||
# analysis messages at render time. If analysis messages need
|
||||
# to be dropped here, add a channel != "analysis" filter when
|
||||
# re-appending, similar to auto_drop_analysis_messages in
|
||||
# harmony_utils.py.
|
||||
if len(prev_msgs) > 0:
|
||||
last_msg = prev_msgs[-1]
|
||||
assert isinstance(last_msg, OpenAIHarmonyMessage)
|
||||
if last_msg.channel == "final":
|
||||
prev_final_msg_idx = -1
|
||||
for i in range(len(prev_msgs) - 2, -1, -1):
|
||||
prev_msg_i = prev_msgs[i]
|
||||
assert isinstance(prev_msg_i, OpenAIHarmonyMessage)
|
||||
if prev_msg_i.channel == "final":
|
||||
prev_final_msg_idx = i
|
||||
break
|
||||
recent_turn_msgs = prev_msgs[prev_final_msg_idx + 1 :]
|
||||
del prev_msgs[prev_final_msg_idx + 1 :]
|
||||
for msg in recent_turn_msgs:
|
||||
assert isinstance(msg, OpenAIHarmonyMessage)
|
||||
prev_msgs.append(msg)
|
||||
messages.extend(prev_msgs)
|
||||
# Append the new input.
|
||||
# Responses API supports simple text inputs without chat format.
|
||||
|
||||
Reference in New Issue
Block a user