[Bugfix] Fix empty channel/recipient in harmony for /v1/responses (#35540)

Signed-off-by: kg6-sleipnir <christopherhazen42@gmail.com>
Signed-off-by: chazen <45186108+kg6-sleipnir@users.noreply.github.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
chazen
2026-05-12 08:45:51 +00:00
committed by GitHub
co-authored by Chauncey
parent ef34592a1a
commit 7f65f84428
3 changed files with 768 additions and 0 deletions
@@ -0,0 +1,492 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cross-API render parity tests.
Verifies that the chat completion input path (parse_chat_input_to_harmony_message)
and the responses API input path (response_input_to_harmony) produce identical
Harmony messages and identical rendered token sequences when given equivalent
conversation representations.
The chat completion API encodes reasoning and tool calls as fields on a single
assistant message dict; the responses API encodes them as separate typed items
in request.input. Both paths must converge on the same Harmony message list and
therefore the same rendered prompt.
Each test:
1. Builds Harmony messages from each path for a single message or sequence.
2. Asserts message-level properties (role, channel, recipient, content)
using verify_harmony_messages.
3. Asserts that render_for_completion produces identical token sequences.
"""
from openai.types.responses import ResponseFunctionToolCall
from tests.entrypoints.openai.utils import verify_harmony_messages
from vllm.entrypoints.openai.parser.harmony_utils import (
get_system_message,
parse_chat_input_to_harmony_message,
render_for_completion,
)
from vllm.entrypoints.openai.responses.harmony import response_input_to_harmony
# Use a fixed date so the system message is deterministic across both paths.
_DATE = "2025-01-01"
def _system():
return get_system_message(start_date=_DATE)
class TestResponseInputToHarmonyRenderParity:
"""Each test drives the same conversation through both APIs and asserts
identical Harmony messages and rendered token sequences."""
# -----------------------------------------------------------------------
# Single-message cases
# -----------------------------------------------------------------------
def test_user_message(self):
chat_msgs = parse_chat_input_to_harmony_message(
{"role": "user", "content": "What's the weather in Paris?"}
)
resp_msgs = [
response_input_to_harmony(
{
"type": "message",
"role": "user",
"content": "What's the weather in Paris?",
},
prev_responses=[],
)
]
expected = [{"role": "user", "content": "What's the weather in Paris?"}]
verify_harmony_messages(chat_msgs, expected)
verify_harmony_messages(resp_msgs, expected)
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
[_system()] + resp_msgs
)
def test_assistant_final_message(self):
chat_msgs = parse_chat_input_to_harmony_message(
{"role": "assistant", "content": "It is 18°C in Paris."}
)
resp_msgs = [
response_input_to_harmony(
{
"type": "message",
"role": "assistant",
"content": "It is 18°C in Paris.",
},
prev_responses=[],
)
]
expected = [
{"role": "assistant", "channel": "final", "content": "It is 18°C in Paris."}
]
verify_harmony_messages(chat_msgs, expected)
verify_harmony_messages(resp_msgs, expected)
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
[_system()] + resp_msgs
)
def test_reasoning_item(self):
# Chat path: assistant message with only a reasoning field and no content.
chat_msgs = parse_chat_input_to_harmony_message(
{
"role": "assistant",
"reasoning": "I should call get_weather.",
"content": "",
}
)
resp_msgs = [
response_input_to_harmony(
{
"type": "reasoning",
"content": [
{"type": "reasoning_text", "text": "I should call get_weather."}
],
},
prev_responses=[],
)
]
expected = [
{
"role": "assistant",
"channel": "analysis",
"content": "I should call get_weather.",
}
]
verify_harmony_messages(chat_msgs, expected)
verify_harmony_messages(resp_msgs, expected)
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
[_system()] + resp_msgs
)
def test_function_call(self):
chat_msgs = parse_chat_input_to_harmony_message(
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"function": {
"name": "get_weather",
"arguments": '{"location": "Paris"}',
},
}
],
}
)
resp_msgs = [
response_input_to_harmony(
{
"type": "function_call",
"name": "get_weather",
"arguments": '{"location": "Paris"}',
},
prev_responses=[],
)
]
expected = [
{
"role": "assistant",
"channel": "commentary",
"recipient": "functions.get_weather",
"content": '{"location": "Paris"}',
"content_type": "json",
}
]
verify_harmony_messages(chat_msgs, expected)
verify_harmony_messages(resp_msgs, expected)
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
[_system()] + resp_msgs
)
def test_tool_output(self):
prev_call = ResponseFunctionToolCall(
id="fc_1",
call_id="call_1",
name="get_weather",
arguments='{"location": "Paris"}',
type="function_call",
)
chat_msgs = parse_chat_input_to_harmony_message(
{"role": "tool", "tool_call_id": "call_1", "content": "18°C, clear skies."},
tool_id_names={"call_1": "get_weather"},
)
resp_msgs = [
response_input_to_harmony(
{
"type": "function_call_output",
"call_id": "call_1",
"output": "18°C, clear skies.",
},
prev_responses=[prev_call],
)
]
expected = [
{
"role": "tool",
"author_name": "functions.get_weather",
"channel": "commentary",
"recipient": "assistant",
"content": "18°C, clear skies.",
}
]
verify_harmony_messages(chat_msgs, expected)
verify_harmony_messages(resp_msgs, expected)
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
[_system()] + resp_msgs
)
# -----------------------------------------------------------------------
# Combined and multi-turn cases
# -----------------------------------------------------------------------
def test_reasoning_combined_with_function_call(self):
"""Chat API packs reasoning + tool_calls into one dict; responses API
represents them as two separate items. Both must produce the same two
Harmony messages in the same order: analysis then commentary."""
chat_msgs = parse_chat_input_to_harmony_message(
{
"role": "assistant",
"reasoning": "I should get the weather for Paris.",
"tool_calls": [
{
"id": "call_1",
"function": {
"name": "get_weather",
"arguments": '{"location": "Paris"}',
},
}
],
}
)
resp_msgs = [
response_input_to_harmony(
{
"type": "reasoning",
"content": [
{
"type": "reasoning_text",
"text": "I should get the weather for Paris.",
}
],
},
prev_responses=[],
),
response_input_to_harmony(
{
"type": "function_call",
"name": "get_weather",
"arguments": '{"location": "Paris"}',
},
prev_responses=[],
),
]
expected = [
{
"role": "assistant",
"channel": "analysis",
"content": "I should get the weather for Paris.",
},
{
"role": "assistant",
"channel": "commentary",
"recipient": "functions.get_weather",
"content": '{"location": "Paris"}',
"content_type": "json",
},
]
verify_harmony_messages(chat_msgs, expected)
verify_harmony_messages(resp_msgs, expected)
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
[_system()] + resp_msgs
)
def test_full_multi_turn_tool_call_conversation(self):
"""Full conversation: user -> reasoning + tool_call -> tool_output -> final.
Both APIs must render the complete conversation to identical token sequences.
This exercises the entire input pipeline including all message types and
the Rust harmony encoder.
"""
prev_call = ResponseFunctionToolCall(
id="fc_1",
call_id="call_1",
name="get_weather",
arguments='{"location": "Paris"}',
type="function_call",
)
# --- Chat completion API path ---
tool_id_names = {"call_1": "get_weather"}
chat_msgs = []
chat_msgs += parse_chat_input_to_harmony_message(
{"role": "user", "content": "What's the weather in Paris?"}
)
chat_msgs += parse_chat_input_to_harmony_message(
{
"role": "assistant",
"reasoning": "I should call get_weather for Paris.",
"tool_calls": [
{
"id": "call_1",
"function": {
"name": "get_weather",
"arguments": '{"location": "Paris"}',
},
}
],
}
)
chat_msgs += parse_chat_input_to_harmony_message(
{"role": "tool", "tool_call_id": "call_1", "content": "18°C, clear skies."},
tool_id_names=tool_id_names,
)
chat_msgs += parse_chat_input_to_harmony_message(
{
"role": "assistant",
"content": "It is currently 18°C in Paris with clear skies.",
}
)
# --- Responses API path ---
resp_input = [
{
"type": "message",
"role": "user",
"content": "What's the weather in Paris?",
},
{
"type": "reasoning",
"content": [
{
"type": "reasoning_text",
"text": "I should call get_weather for Paris.",
}
],
},
{
"type": "function_call",
"name": "get_weather",
"arguments": '{"location": "Paris"}',
},
{
"type": "function_call_output",
"call_id": "call_1",
"output": "18°C, clear skies.",
},
{
"type": "message",
"role": "assistant",
"content": "It is currently 18°C in Paris with clear skies.",
},
]
resp_msgs = [
response_input_to_harmony(item, prev_responses=[prev_call])
for item in resp_input
]
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
[_system()] + resp_msgs
)
def test_multi_turn_two_tool_calls_with_reasoning_between(self):
"""Validates parity for a chain of two tool calls, each with its own
reasoning trace. Reasoning traces in between commentary-channel tool
calls must survive as analysis-channel messages in both paths.
"""
prev_call_1 = ResponseFunctionToolCall(
id="fc_1",
call_id="call_1",
name="get_weather",
arguments='{"location": "Paris"}',
type="function_call",
)
prev_call_2 = ResponseFunctionToolCall(
id="fc_2",
call_id="call_2",
name="get_forecast",
arguments='{"location": "Paris", "days": 7}',
type="function_call",
)
# --- Chat completion API path ---
tool_id_names = {"call_1": "get_weather", "call_2": "get_forecast"}
chat_msgs = []
chat_msgs += parse_chat_input_to_harmony_message(
{"role": "user", "content": "What's the weather and forecast for Paris?"}
)
# First reasoning + tool call
chat_msgs += parse_chat_input_to_harmony_message(
{
"role": "assistant",
"reasoning": "I need current weather first.",
"tool_calls": [
{
"id": "call_1",
"function": {
"name": "get_weather",
"arguments": '{"location": "Paris"}',
},
}
],
}
)
chat_msgs += parse_chat_input_to_harmony_message(
{"role": "tool", "tool_call_id": "call_1", "content": "18°C, clear skies."},
tool_id_names=tool_id_names,
)
# Second reasoning + tool call
chat_msgs += parse_chat_input_to_harmony_message(
{
"role": "assistant",
"reasoning": "Now I need the weekly forecast.",
"tool_calls": [
{
"id": "call_2",
"function": {
"name": "get_forecast",
"arguments": '{"location": "Paris", "days": 7}',
},
}
],
}
)
chat_msgs += parse_chat_input_to_harmony_message(
{
"role": "tool",
"tool_call_id": "call_2",
"content": "Mon 17°C, Tue 19°C, Wed 16°C",
},
tool_id_names=tool_id_names,
)
# --- Responses API path ---
prev_responses = [prev_call_1, prev_call_2]
resp_input = [
{
"type": "message",
"role": "user",
"content": "What's the weather and forecast for Paris?",
},
# First reasoning + tool call
{
"type": "reasoning",
"content": [
{"type": "reasoning_text", "text": "I need current weather first."}
],
},
{
"type": "function_call",
"name": "get_weather",
"arguments": '{"location": "Paris"}',
},
{
"type": "function_call_output",
"call_id": "call_1",
"output": "18°C, clear skies.",
},
# Second reasoning + tool call
{
"type": "reasoning",
"content": [
{
"type": "reasoning_text",
"text": "Now I need the weekly forecast.",
}
],
},
{
"type": "function_call",
"name": "get_forecast",
"arguments": '{"location": "Paris", "days": 7}',
},
{
"type": "function_call_output",
"call_id": "call_2",
"output": "Mon 17°C, Tue 19°C, Wed 16°C",
},
]
resp_msgs = [
response_input_to_harmony(item, prev_responses=prev_responses)
for item in resp_input
]
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
[_system()] + resp_msgs
)
@@ -0,0 +1,274 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for response_input_to_harmony.
Covers every type branch in the function and verifies that each produced
Harmony Message has the correct role, channel, recipient, content_type,
author name, and text content.
"""
import pytest
from openai.types.responses import ResponseFunctionToolCall, ResponseReasoningItem
from openai.types.responses.response_reasoning_item import (
Content as ReasoningTextContent,
)
from openai_harmony import Role
from vllm.entrypoints.openai.responses.harmony import response_input_to_harmony
# ---------------------------------------------------------------------------
# Shared fixtures
# ---------------------------------------------------------------------------
_PREV_CALL = ResponseFunctionToolCall(
id="fc_test",
call_id="call_test",
name="get_weather",
arguments='{"location": "Paris"}',
type="function_call",
)
_REASONING_ITEM = ResponseReasoningItem(
id="rs_test",
type="reasoning",
content=[ReasoningTextContent(type="reasoning_text", text="Thinking hard.")],
summary=[],
status=None,
)
class TestResponseInputToHarmonyMessage:
"""Unit tests for every message type handled by response_input_to_harmony."""
# -----------------------------------------------------------------------
# type="message" (or no type key)
# -----------------------------------------------------------------------
def test_user_message_string_content(self):
msg = response_input_to_harmony(
{"type": "message", "role": "user", "content": "Hello"},
prev_responses=[],
)
assert msg.author.role == Role.USER
assert msg.content[0].text == "Hello"
assert msg.channel is None
def test_no_type_key_defaults_to_message_branch(self):
"""Omitting 'type' should fall through to the message branch."""
msg = response_input_to_harmony(
{"role": "user", "content": "Hello"},
prev_responses=[],
)
assert msg.author.role == Role.USER
assert msg.content[0].text == "Hello"
def test_system_message(self):
msg = response_input_to_harmony(
{"type": "message", "role": "system", "content": "Be helpful."},
prev_responses=[],
)
assert msg.author.role == Role.SYSTEM
assert msg.content[0].text == "Be helpful."
assert msg.channel is None
def test_assistant_message_gets_final_channel(self):
msg = response_input_to_harmony(
{"type": "message", "role": "assistant", "content": "The answer is 42."},
prev_responses=[],
)
assert msg.author.role == Role.ASSISTANT
assert msg.channel == "final"
assert msg.content[0].text == "The answer is 42."
def test_developer_message_gets_instructions_prefix(self):
msg = response_input_to_harmony(
{"type": "message", "role": "developer", "content": "Be concise."},
prev_responses=[],
)
assert msg.author.role == Role.DEVELOPER
assert msg.content[0].text == "Instructions:\nBe concise."
assert msg.channel is None
def test_message_with_array_content(self):
msg = response_input_to_harmony(
{
"type": "message",
"role": "user",
"content": [
{"type": "text", "text": "Part one. "},
{"type": "text", "text": "Part two."},
],
},
prev_responses=[],
)
assert msg.author.role == Role.USER
assert len(msg.content) == 2
assert msg.content[0].text == "Part one. "
assert msg.content[1].text == "Part two."
def test_developer_message_array_content_gets_prefix_on_each_part(self):
msg = response_input_to_harmony(
{
"type": "message",
"role": "developer",
"content": [
{"type": "text", "text": "Rule 1."},
{"type": "text", "text": "Rule 2."},
],
},
prev_responses=[],
)
assert msg.content[0].text == "Instructions:\nRule 1."
assert msg.content[1].text == "Instructions:\nRule 2."
# -----------------------------------------------------------------------
# type="reasoning"
# -----------------------------------------------------------------------
def test_reasoning_gets_analysis_channel(self):
msg = response_input_to_harmony(
{
"type": "reasoning",
"content": [
{"type": "reasoning_text", "text": "I should call get_weather."}
],
},
prev_responses=[],
)
assert msg.author.role == Role.ASSISTANT
assert msg.channel == "analysis"
assert msg.content[0].text == "I should call get_weather."
def test_reasoning_pydantic_model_input(self):
"""A Pydantic ResponseReasoningItem should be model_dump()'d before parsing."""
msg = response_input_to_harmony(_REASONING_ITEM, prev_responses=[])
assert msg.author.role == Role.ASSISTANT
assert msg.channel == "analysis"
assert msg.content[0].text == "Thinking hard."
# -----------------------------------------------------------------------
# type="function_call"
# -----------------------------------------------------------------------
def test_function_call_channel_recipient_and_content_type(self):
msg = response_input_to_harmony(
{
"type": "function_call",
"name": "get_weather",
"arguments": '{"location": "Paris"}',
},
prev_responses=[],
)
assert msg.author.role == Role.ASSISTANT
assert msg.channel == "commentary"
assert msg.recipient == "functions.get_weather"
assert msg.content_type == "json"
assert msg.content[0].text == '{"location": "Paris"}'
def test_function_call_empty_arguments(self):
msg = response_input_to_harmony(
{"type": "function_call", "name": "ping", "arguments": ""},
prev_responses=[],
)
assert msg.recipient == "functions.ping"
assert msg.content[0].text == ""
# -----------------------------------------------------------------------
# type="function_call_output"
# -----------------------------------------------------------------------
def test_function_call_output_channel_recipient_and_author_name(self):
msg = response_input_to_harmony(
{"type": "function_call_output", "call_id": "call_test", "output": "18°C"},
prev_responses=[_PREV_CALL],
)
assert msg.author.role == Role.TOOL
assert msg.author.name == "functions.get_weather"
assert msg.channel == "commentary"
assert msg.recipient == "assistant"
assert msg.content[0].text == "18°C"
def test_function_call_output_uses_most_recent_matching_call(self):
"""When multiple prev_responses share a call_id, the last one wins
because the search is reversed."""
earlier = ResponseFunctionToolCall(
id="fc_old",
call_id="call_test",
name="old_func",
arguments="{}",
type="function_call",
)
later = ResponseFunctionToolCall(
id="fc_new",
call_id="call_test",
name="get_weather",
arguments="{}",
type="function_call",
)
msg = response_input_to_harmony(
{
"type": "function_call_output",
"call_id": "call_test",
"output": "result",
},
prev_responses=[earlier, later],
)
assert msg.author.name == "functions.get_weather"
def test_function_call_output_skips_non_function_call_items_in_prev_responses(
self,
):
"""ResponseReasoningItem entries in prev_responses should be ignored."""
msg = response_input_to_harmony(
{
"type": "function_call_output",
"call_id": "call_test",
"output": "18°C",
},
prev_responses=[_REASONING_ITEM, _PREV_CALL],
)
assert msg.author.name == "functions.get_weather"
def test_function_call_output_raises_if_no_matching_call(self):
with pytest.raises(ValueError, match="No call message found for"):
response_input_to_harmony(
{
"type": "function_call_output",
"call_id": "no_such_id",
"output": "x",
},
prev_responses=[_PREV_CALL],
)
def test_function_call_output_raises_on_empty_prev_responses(self):
with pytest.raises(ValueError, match="No call message found for"):
response_input_to_harmony(
{"type": "function_call_output", "call_id": "call_test", "output": "x"},
prev_responses=[],
)
# -----------------------------------------------------------------------
# Error cases
# -----------------------------------------------------------------------
def test_unknown_type_raises_value_error(self):
with pytest.raises(ValueError, match="Unknown input type"):
response_input_to_harmony(
{"type": "image_url", "url": "https://example.com/img.png"},
prev_responses=[],
)
@@ -175,6 +175,8 @@ def response_input_to_harmony(
Author.new(Role.TOOL, f"functions.{call_response.name}"),
response_msg["output"],
)
msg = msg.with_channel("commentary")
msg = msg.with_recipient("assistant")
elif response_msg["type"] == "reasoning":
content = response_msg.get("content")
if content and len(content) >= 1: