diff --git a/tests/entrypoints/openai/parser/test_harmony_render_parity.py b/tests/entrypoints/openai/parser/test_harmony_render_parity.py new file mode 100644 index 0000000000..b5ba334499 --- /dev/null +++ b/tests/entrypoints/openai/parser/test_harmony_render_parity.py @@ -0,0 +1,492 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Cross-API render parity tests. + +Verifies that the chat completion input path (parse_chat_input_to_harmony_message) +and the responses API input path (response_input_to_harmony) produce identical +Harmony messages and identical rendered token sequences when given equivalent +conversation representations. + +The chat completion API encodes reasoning and tool calls as fields on a single +assistant message dict; the responses API encodes them as separate typed items +in request.input. Both paths must converge on the same Harmony message list and +therefore the same rendered prompt. + +Each test: + 1. Builds Harmony messages from each path for a single message or sequence. + 2. Asserts message-level properties (role, channel, recipient, content) + using verify_harmony_messages. + 3. Asserts that render_for_completion produces identical token sequences. +""" + +from openai.types.responses import ResponseFunctionToolCall + +from tests.entrypoints.openai.utils import verify_harmony_messages +from vllm.entrypoints.openai.parser.harmony_utils import ( + get_system_message, + parse_chat_input_to_harmony_message, + render_for_completion, +) +from vllm.entrypoints.openai.responses.harmony import response_input_to_harmony + +# Use a fixed date so the system message is deterministic across both paths. +_DATE = "2025-01-01" + + +def _system(): + return get_system_message(start_date=_DATE) + + +class TestResponseInputToHarmonyRenderParity: + """Each test drives the same conversation through both APIs and asserts + identical Harmony messages and rendered token sequences.""" + + # ----------------------------------------------------------------------- + # Single-message cases + # ----------------------------------------------------------------------- + + def test_user_message(self): + chat_msgs = parse_chat_input_to_harmony_message( + {"role": "user", "content": "What's the weather in Paris?"} + ) + resp_msgs = [ + response_input_to_harmony( + { + "type": "message", + "role": "user", + "content": "What's the weather in Paris?", + }, + prev_responses=[], + ) + ] + + expected = [{"role": "user", "content": "What's the weather in Paris?"}] + verify_harmony_messages(chat_msgs, expected) + verify_harmony_messages(resp_msgs, expected) + + assert render_for_completion([_system()] + chat_msgs) == render_for_completion( + [_system()] + resp_msgs + ) + + def test_assistant_final_message(self): + chat_msgs = parse_chat_input_to_harmony_message( + {"role": "assistant", "content": "It is 18°C in Paris."} + ) + resp_msgs = [ + response_input_to_harmony( + { + "type": "message", + "role": "assistant", + "content": "It is 18°C in Paris.", + }, + prev_responses=[], + ) + ] + + expected = [ + {"role": "assistant", "channel": "final", "content": "It is 18°C in Paris."} + ] + verify_harmony_messages(chat_msgs, expected) + verify_harmony_messages(resp_msgs, expected) + + assert render_for_completion([_system()] + chat_msgs) == render_for_completion( + [_system()] + resp_msgs + ) + + def test_reasoning_item(self): + # Chat path: assistant message with only a reasoning field and no content. + chat_msgs = parse_chat_input_to_harmony_message( + { + "role": "assistant", + "reasoning": "I should call get_weather.", + "content": "", + } + ) + resp_msgs = [ + response_input_to_harmony( + { + "type": "reasoning", + "content": [ + {"type": "reasoning_text", "text": "I should call get_weather."} + ], + }, + prev_responses=[], + ) + ] + + expected = [ + { + "role": "assistant", + "channel": "analysis", + "content": "I should call get_weather.", + } + ] + verify_harmony_messages(chat_msgs, expected) + verify_harmony_messages(resp_msgs, expected) + + assert render_for_completion([_system()] + chat_msgs) == render_for_completion( + [_system()] + resp_msgs + ) + + def test_function_call(self): + chat_msgs = parse_chat_input_to_harmony_message( + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "function": { + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + } + ], + } + ) + resp_msgs = [ + response_input_to_harmony( + { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + prev_responses=[], + ) + ] + + expected = [ + { + "role": "assistant", + "channel": "commentary", + "recipient": "functions.get_weather", + "content": '{"location": "Paris"}', + "content_type": "json", + } + ] + verify_harmony_messages(chat_msgs, expected) + verify_harmony_messages(resp_msgs, expected) + + assert render_for_completion([_system()] + chat_msgs) == render_for_completion( + [_system()] + resp_msgs + ) + + def test_tool_output(self): + prev_call = ResponseFunctionToolCall( + id="fc_1", + call_id="call_1", + name="get_weather", + arguments='{"location": "Paris"}', + type="function_call", + ) + + chat_msgs = parse_chat_input_to_harmony_message( + {"role": "tool", "tool_call_id": "call_1", "content": "18°C, clear skies."}, + tool_id_names={"call_1": "get_weather"}, + ) + resp_msgs = [ + response_input_to_harmony( + { + "type": "function_call_output", + "call_id": "call_1", + "output": "18°C, clear skies.", + }, + prev_responses=[prev_call], + ) + ] + + expected = [ + { + "role": "tool", + "author_name": "functions.get_weather", + "channel": "commentary", + "recipient": "assistant", + "content": "18°C, clear skies.", + } + ] + verify_harmony_messages(chat_msgs, expected) + verify_harmony_messages(resp_msgs, expected) + + assert render_for_completion([_system()] + chat_msgs) == render_for_completion( + [_system()] + resp_msgs + ) + + # ----------------------------------------------------------------------- + # Combined and multi-turn cases + # ----------------------------------------------------------------------- + + def test_reasoning_combined_with_function_call(self): + """Chat API packs reasoning + tool_calls into one dict; responses API + represents them as two separate items. Both must produce the same two + Harmony messages in the same order: analysis then commentary.""" + chat_msgs = parse_chat_input_to_harmony_message( + { + "role": "assistant", + "reasoning": "I should get the weather for Paris.", + "tool_calls": [ + { + "id": "call_1", + "function": { + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + } + ], + } + ) + resp_msgs = [ + response_input_to_harmony( + { + "type": "reasoning", + "content": [ + { + "type": "reasoning_text", + "text": "I should get the weather for Paris.", + } + ], + }, + prev_responses=[], + ), + response_input_to_harmony( + { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + prev_responses=[], + ), + ] + + expected = [ + { + "role": "assistant", + "channel": "analysis", + "content": "I should get the weather for Paris.", + }, + { + "role": "assistant", + "channel": "commentary", + "recipient": "functions.get_weather", + "content": '{"location": "Paris"}', + "content_type": "json", + }, + ] + verify_harmony_messages(chat_msgs, expected) + verify_harmony_messages(resp_msgs, expected) + + assert render_for_completion([_system()] + chat_msgs) == render_for_completion( + [_system()] + resp_msgs + ) + + def test_full_multi_turn_tool_call_conversation(self): + """Full conversation: user -> reasoning + tool_call -> tool_output -> final. + + Both APIs must render the complete conversation to identical token sequences. + This exercises the entire input pipeline including all message types and + the Rust harmony encoder. + """ + prev_call = ResponseFunctionToolCall( + id="fc_1", + call_id="call_1", + name="get_weather", + arguments='{"location": "Paris"}', + type="function_call", + ) + + # --- Chat completion API path --- + tool_id_names = {"call_1": "get_weather"} + chat_msgs = [] + chat_msgs += parse_chat_input_to_harmony_message( + {"role": "user", "content": "What's the weather in Paris?"} + ) + chat_msgs += parse_chat_input_to_harmony_message( + { + "role": "assistant", + "reasoning": "I should call get_weather for Paris.", + "tool_calls": [ + { + "id": "call_1", + "function": { + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + } + ], + } + ) + chat_msgs += parse_chat_input_to_harmony_message( + {"role": "tool", "tool_call_id": "call_1", "content": "18°C, clear skies."}, + tool_id_names=tool_id_names, + ) + chat_msgs += parse_chat_input_to_harmony_message( + { + "role": "assistant", + "content": "It is currently 18°C in Paris with clear skies.", + } + ) + + # --- Responses API path --- + resp_input = [ + { + "type": "message", + "role": "user", + "content": "What's the weather in Paris?", + }, + { + "type": "reasoning", + "content": [ + { + "type": "reasoning_text", + "text": "I should call get_weather for Paris.", + } + ], + }, + { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + { + "type": "function_call_output", + "call_id": "call_1", + "output": "18°C, clear skies.", + }, + { + "type": "message", + "role": "assistant", + "content": "It is currently 18°C in Paris with clear skies.", + }, + ] + resp_msgs = [ + response_input_to_harmony(item, prev_responses=[prev_call]) + for item in resp_input + ] + + assert render_for_completion([_system()] + chat_msgs) == render_for_completion( + [_system()] + resp_msgs + ) + + def test_multi_turn_two_tool_calls_with_reasoning_between(self): + """Validates parity for a chain of two tool calls, each with its own + reasoning trace. Reasoning traces in between commentary-channel tool + calls must survive as analysis-channel messages in both paths. + """ + prev_call_1 = ResponseFunctionToolCall( + id="fc_1", + call_id="call_1", + name="get_weather", + arguments='{"location": "Paris"}', + type="function_call", + ) + prev_call_2 = ResponseFunctionToolCall( + id="fc_2", + call_id="call_2", + name="get_forecast", + arguments='{"location": "Paris", "days": 7}', + type="function_call", + ) + + # --- Chat completion API path --- + tool_id_names = {"call_1": "get_weather", "call_2": "get_forecast"} + chat_msgs = [] + chat_msgs += parse_chat_input_to_harmony_message( + {"role": "user", "content": "What's the weather and forecast for Paris?"} + ) + # First reasoning + tool call + chat_msgs += parse_chat_input_to_harmony_message( + { + "role": "assistant", + "reasoning": "I need current weather first.", + "tool_calls": [ + { + "id": "call_1", + "function": { + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + } + ], + } + ) + chat_msgs += parse_chat_input_to_harmony_message( + {"role": "tool", "tool_call_id": "call_1", "content": "18°C, clear skies."}, + tool_id_names=tool_id_names, + ) + # Second reasoning + tool call + chat_msgs += parse_chat_input_to_harmony_message( + { + "role": "assistant", + "reasoning": "Now I need the weekly forecast.", + "tool_calls": [ + { + "id": "call_2", + "function": { + "name": "get_forecast", + "arguments": '{"location": "Paris", "days": 7}', + }, + } + ], + } + ) + chat_msgs += parse_chat_input_to_harmony_message( + { + "role": "tool", + "tool_call_id": "call_2", + "content": "Mon 17°C, Tue 19°C, Wed 16°C", + }, + tool_id_names=tool_id_names, + ) + + # --- Responses API path --- + prev_responses = [prev_call_1, prev_call_2] + resp_input = [ + { + "type": "message", + "role": "user", + "content": "What's the weather and forecast for Paris?", + }, + # First reasoning + tool call + { + "type": "reasoning", + "content": [ + {"type": "reasoning_text", "text": "I need current weather first."} + ], + }, + { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + { + "type": "function_call_output", + "call_id": "call_1", + "output": "18°C, clear skies.", + }, + # Second reasoning + tool call + { + "type": "reasoning", + "content": [ + { + "type": "reasoning_text", + "text": "Now I need the weekly forecast.", + } + ], + }, + { + "type": "function_call", + "name": "get_forecast", + "arguments": '{"location": "Paris", "days": 7}', + }, + { + "type": "function_call_output", + "call_id": "call_2", + "output": "Mon 17°C, Tue 19°C, Wed 16°C", + }, + ] + resp_msgs = [ + response_input_to_harmony(item, prev_responses=prev_responses) + for item in resp_input + ] + + assert render_for_completion([_system()] + chat_msgs) == render_for_completion( + [_system()] + resp_msgs + ) diff --git a/tests/entrypoints/openai/responses/test_response_input_to_harmony.py b/tests/entrypoints/openai/responses/test_response_input_to_harmony.py new file mode 100644 index 0000000000..8efd015773 --- /dev/null +++ b/tests/entrypoints/openai/responses/test_response_input_to_harmony.py @@ -0,0 +1,274 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for response_input_to_harmony. + +Covers every type branch in the function and verifies that each produced +Harmony Message has the correct role, channel, recipient, content_type, +author name, and text content. +""" + +import pytest +from openai.types.responses import ResponseFunctionToolCall, ResponseReasoningItem +from openai.types.responses.response_reasoning_item import ( + Content as ReasoningTextContent, +) +from openai_harmony import Role + +from vllm.entrypoints.openai.responses.harmony import response_input_to_harmony + +# --------------------------------------------------------------------------- +# Shared fixtures +# --------------------------------------------------------------------------- + +_PREV_CALL = ResponseFunctionToolCall( + id="fc_test", + call_id="call_test", + name="get_weather", + arguments='{"location": "Paris"}', + type="function_call", +) + +_REASONING_ITEM = ResponseReasoningItem( + id="rs_test", + type="reasoning", + content=[ReasoningTextContent(type="reasoning_text", text="Thinking hard.")], + summary=[], + status=None, +) + + +class TestResponseInputToHarmonyMessage: + """Unit tests for every message type handled by response_input_to_harmony.""" + + # ----------------------------------------------------------------------- + # type="message" (or no type key) + # ----------------------------------------------------------------------- + + def test_user_message_string_content(self): + msg = response_input_to_harmony( + {"type": "message", "role": "user", "content": "Hello"}, + prev_responses=[], + ) + + assert msg.author.role == Role.USER + assert msg.content[0].text == "Hello" + assert msg.channel is None + + def test_no_type_key_defaults_to_message_branch(self): + """Omitting 'type' should fall through to the message branch.""" + msg = response_input_to_harmony( + {"role": "user", "content": "Hello"}, + prev_responses=[], + ) + + assert msg.author.role == Role.USER + assert msg.content[0].text == "Hello" + + def test_system_message(self): + msg = response_input_to_harmony( + {"type": "message", "role": "system", "content": "Be helpful."}, + prev_responses=[], + ) + + assert msg.author.role == Role.SYSTEM + assert msg.content[0].text == "Be helpful." + assert msg.channel is None + + def test_assistant_message_gets_final_channel(self): + msg = response_input_to_harmony( + {"type": "message", "role": "assistant", "content": "The answer is 42."}, + prev_responses=[], + ) + + assert msg.author.role == Role.ASSISTANT + assert msg.channel == "final" + assert msg.content[0].text == "The answer is 42." + + def test_developer_message_gets_instructions_prefix(self): + msg = response_input_to_harmony( + {"type": "message", "role": "developer", "content": "Be concise."}, + prev_responses=[], + ) + + assert msg.author.role == Role.DEVELOPER + assert msg.content[0].text == "Instructions:\nBe concise." + assert msg.channel is None + + def test_message_with_array_content(self): + msg = response_input_to_harmony( + { + "type": "message", + "role": "user", + "content": [ + {"type": "text", "text": "Part one. "}, + {"type": "text", "text": "Part two."}, + ], + }, + prev_responses=[], + ) + + assert msg.author.role == Role.USER + assert len(msg.content) == 2 + assert msg.content[0].text == "Part one. " + assert msg.content[1].text == "Part two." + + def test_developer_message_array_content_gets_prefix_on_each_part(self): + msg = response_input_to_harmony( + { + "type": "message", + "role": "developer", + "content": [ + {"type": "text", "text": "Rule 1."}, + {"type": "text", "text": "Rule 2."}, + ], + }, + prev_responses=[], + ) + + assert msg.content[0].text == "Instructions:\nRule 1." + assert msg.content[1].text == "Instructions:\nRule 2." + + # ----------------------------------------------------------------------- + # type="reasoning" + # ----------------------------------------------------------------------- + + def test_reasoning_gets_analysis_channel(self): + msg = response_input_to_harmony( + { + "type": "reasoning", + "content": [ + {"type": "reasoning_text", "text": "I should call get_weather."} + ], + }, + prev_responses=[], + ) + + assert msg.author.role == Role.ASSISTANT + assert msg.channel == "analysis" + assert msg.content[0].text == "I should call get_weather." + + def test_reasoning_pydantic_model_input(self): + """A Pydantic ResponseReasoningItem should be model_dump()'d before parsing.""" + msg = response_input_to_harmony(_REASONING_ITEM, prev_responses=[]) + + assert msg.author.role == Role.ASSISTANT + assert msg.channel == "analysis" + assert msg.content[0].text == "Thinking hard." + + # ----------------------------------------------------------------------- + # type="function_call" + # ----------------------------------------------------------------------- + + def test_function_call_channel_recipient_and_content_type(self): + msg = response_input_to_harmony( + { + "type": "function_call", + "name": "get_weather", + "arguments": '{"location": "Paris"}', + }, + prev_responses=[], + ) + + assert msg.author.role == Role.ASSISTANT + assert msg.channel == "commentary" + assert msg.recipient == "functions.get_weather" + assert msg.content_type == "json" + assert msg.content[0].text == '{"location": "Paris"}' + + def test_function_call_empty_arguments(self): + msg = response_input_to_harmony( + {"type": "function_call", "name": "ping", "arguments": ""}, + prev_responses=[], + ) + + assert msg.recipient == "functions.ping" + assert msg.content[0].text == "" + + # ----------------------------------------------------------------------- + # type="function_call_output" + # ----------------------------------------------------------------------- + + def test_function_call_output_channel_recipient_and_author_name(self): + msg = response_input_to_harmony( + {"type": "function_call_output", "call_id": "call_test", "output": "18°C"}, + prev_responses=[_PREV_CALL], + ) + + assert msg.author.role == Role.TOOL + assert msg.author.name == "functions.get_weather" + assert msg.channel == "commentary" + assert msg.recipient == "assistant" + assert msg.content[0].text == "18°C" + + def test_function_call_output_uses_most_recent_matching_call(self): + """When multiple prev_responses share a call_id, the last one wins + because the search is reversed.""" + earlier = ResponseFunctionToolCall( + id="fc_old", + call_id="call_test", + name="old_func", + arguments="{}", + type="function_call", + ) + later = ResponseFunctionToolCall( + id="fc_new", + call_id="call_test", + name="get_weather", + arguments="{}", + type="function_call", + ) + + msg = response_input_to_harmony( + { + "type": "function_call_output", + "call_id": "call_test", + "output": "result", + }, + prev_responses=[earlier, later], + ) + + assert msg.author.name == "functions.get_weather" + + def test_function_call_output_skips_non_function_call_items_in_prev_responses( + self, + ): + """ResponseReasoningItem entries in prev_responses should be ignored.""" + msg = response_input_to_harmony( + { + "type": "function_call_output", + "call_id": "call_test", + "output": "18°C", + }, + prev_responses=[_REASONING_ITEM, _PREV_CALL], + ) + + assert msg.author.name == "functions.get_weather" + + def test_function_call_output_raises_if_no_matching_call(self): + with pytest.raises(ValueError, match="No call message found for"): + response_input_to_harmony( + { + "type": "function_call_output", + "call_id": "no_such_id", + "output": "x", + }, + prev_responses=[_PREV_CALL], + ) + + def test_function_call_output_raises_on_empty_prev_responses(self): + with pytest.raises(ValueError, match="No call message found for"): + response_input_to_harmony( + {"type": "function_call_output", "call_id": "call_test", "output": "x"}, + prev_responses=[], + ) + + # ----------------------------------------------------------------------- + # Error cases + # ----------------------------------------------------------------------- + + def test_unknown_type_raises_value_error(self): + with pytest.raises(ValueError, match="Unknown input type"): + response_input_to_harmony( + {"type": "image_url", "url": "https://example.com/img.png"}, + prev_responses=[], + ) diff --git a/vllm/entrypoints/openai/responses/harmony.py b/vllm/entrypoints/openai/responses/harmony.py index faab2f7f4c..313a621d08 100644 --- a/vllm/entrypoints/openai/responses/harmony.py +++ b/vllm/entrypoints/openai/responses/harmony.py @@ -175,6 +175,8 @@ def response_input_to_harmony( Author.new(Role.TOOL, f"functions.{call_response.name}"), response_msg["output"], ) + msg = msg.with_channel("commentary") + msg = msg.with_recipient("assistant") elif response_msg["type"] == "reasoning": content = response_msg.get("content") if content and len(content) >= 1: