[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-06-11 21:18:30 +00:00
committed by GitHub
parent 5a6c7b7ab5
commit f712fd0d7d
16 changed files with 822 additions and 816 deletions
@@ -38,12 +38,12 @@ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.exceptions import VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser import HarmonyParser
from vllm.renderers.hf import HfRenderer
from vllm.renderers.mistral import MistralRenderer
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.tool_parsers import ToolParserManager
from vllm.v1.engine.async_llm import AsyncLLM
GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
@@ -575,7 +575,13 @@ def _build_serving_render(
)
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
def _build_serving_chat(
engine: AsyncLLM,
*,
reasoning_parser: str = "",
tool_parser: str | None = None,
enable_auto_tools: bool = False,
) -> OpenAIServingChat:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
@@ -590,6 +596,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
reasoning_parser=reasoning_parser,
tool_parser=tool_parser,
enable_auto_tools=enable_auto_tools,
)
return serving_chat
@@ -637,7 +646,7 @@ async def test_serving_chat_returns_correct_model_name():
serving_chat = _build_serving_chat(mock_engine)
messages = [{"role": "user", "content": "what is 1+1?"}]
async def return_model_name(*args):
async def return_model_name(*args, **kwargs):
return args[3]
serving_chat.chat_completion_full_generator = return_model_name
@@ -1210,15 +1219,21 @@ class TestServingChatWithHarmony:
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
mock_engine.model_config.hf_text_config = MockHFConfig(model_type="gpt_oss")
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
return mock_engine
@pytest.fixture()
def serving_chat(self, mock_engine) -> OpenAIServingChat:
chat = _build_serving_chat(mock_engine)
chat.use_harmony = True
chat.tool_parser = ToolParserManager.get_tool_parser("openai")
chat = _build_serving_chat(
mock_engine,
reasoning_parser="openai_gptoss",
tool_parser="openai",
enable_auto_tools=True,
)
assert chat.parser_cls is HarmonyParser
return chat
def mock_request_output_from_req_and_token_ids(
@@ -1277,6 +1292,7 @@ class TestServingChatWithHarmony:
stream: bool = False,
) -> ChatCompletionResponse:
harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
tokenizer = get_tokenizer(GPT_OSS_MODEL_NAME)
async def result_generator():
if stream:
@@ -1304,11 +1320,12 @@ class TestServingChatWithHarmony:
request_id=req.request_id,
model_name=req.model,
conversation=[],
tokenizer=get_tokenizer(req.model),
tokenizer=tokenizer,
request_metadata=RequestResponseMetadata(
request_id=req.request_id,
model_name=req.model,
),
chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req),
)
if stream:
@@ -1316,11 +1333,18 @@ class TestServingChatWithHarmony:
return await result
@pytest.mark.asyncio
async def test_simple_chat(self, serving_chat, stream):
@pytest.mark.parametrize(
"include_reasoning", [True, False], ids=["with_reasoning", "no_reasoning"]
)
async def test_simple_chat(self, serving_chat, stream, include_reasoning):
messages = [{"role": "user", "content": "what is 1+1?"}]
# Test the Harmony messages for the first turn's input
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
req = ChatCompletionRequest(
model=MODEL_NAME,
messages=messages,
include_reasoning=include_reasoning,
)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
@@ -1342,7 +1366,11 @@ class TestServingChatWithHarmony:
response = await self.generate_response_from_harmony_str(
serving_chat, req, response_str, stream=stream
)
verify_chat_response(response, content=final_str, reasoning=reasoning_str)
verify_chat_response(
response,
content=final_str,
reasoning=reasoning_str if include_reasoning else None,
)
# Add the output messages from the first turn as input to the second turn
for choice in response.choices:
@@ -11,12 +11,10 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
auto_drop_analysis_messages,
create_tool_definition,
extract_function_from_recipient,
get_encoding,
get_system_message,
has_custom_tools,
is_function_recipient,
parse_chat_input_to_harmony_message,
parse_chat_output,
)
from vllm.entrypoints.openai.responses.harmony import (
response_input_to_harmony,
@@ -941,110 +939,6 @@ class TestAutoDropAnalysisMessages:
assert cleaned_messages == messages[1:]
class TestParseChatOutput:
def test_parse_chat_output_interrupted_first_message(self) -> None:
harmony_str = "<|channel|>final<|message|>I'm in the middle of answering"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "I'm in the middle of answering"
def test_parse_chat_output_interrupted_reasoning_first_message(self) -> None:
harmony_str = "<|channel|>analysis<|message|>I'm in the middle of thinking"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I'm in the middle of thinking"
assert final_content is None
def test_parse_chat_output_complete_reasoning_interrupted_content(self) -> None:
harmony_str = (
"<|channel|>analysis<|message|>I'm thinking.<|end|>"
"<|start|>assistant<|channel|>final"
"<|message|>I'm in the middle of answering"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I'm thinking."
assert final_content == "I'm in the middle of answering"
def test_parse_chat_output_complete_content(self) -> None:
harmony_str = "<|channel|>final<|message|>The answer is 4.<|end|>"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "The answer is 4."
def test_parse_chat_output_complete_commentary(self) -> None:
harmony_str = (
"<|channel|>commentary<|message|>I need to call some tools.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "I need to call some tools."
def test_parse_chat_output_complete_reasoning(self) -> None:
harmony_str = (
"<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I've thought hard about this."
assert final_content is None
def test_parse_chat_output_complete_reasoning_and_content(self) -> None:
harmony_str = (
"<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
"<|start|>assistant<|channel|>final<|message|>The answer is 4.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning == "I've thought hard about this."
assert final_content == "The answer is 4."
def test_parse_chat_output_commentary_with_recipient_excluded(self) -> None:
"""Commentary with a recipient (tool call) should not appear in
final_content — those are handled separately by the tool parser.
The first message is a preamble (visible), the second is a tool
call (excluded). Only the preamble should appear in final_content.
"""
harmony_str = (
"<|channel|>commentary"
"<|message|>Let me check the weather.<|end|>"
"<|start|>assistant to=functions.get_weather"
"<|channel|>commentary"
'<|message|>{"location": "SF"}<|end|>'
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "Let me check the weather."
def test_parse_chat_output_interrupted_preamble(self) -> None:
"""Partial/interrupted preamble (commentary without recipient) should
appear in final_content, not reasoning."""
harmony_str = "<|channel|>commentary<|message|>I'll search for that"
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "I'll search for that"
def test_parse_chat_output_preamble_then_final(self) -> None:
"""Preamble followed by a final message should both appear in
final_content, joined by newline."""
harmony_str = (
"<|channel|>commentary"
"<|message|>Let me look that up.<|end|>"
"<|start|>assistant<|channel|>final"
"<|message|>The answer is 42.<|end|>"
)
token_ids = get_encoding().encode(harmony_str, allowed_special="all")
reasoning, final_content, _ = parse_chat_output(token_ids)
assert reasoning is None
assert final_content == "Let me look that up.\nThe answer is 42."
def test_has_custom_tools() -> None:
assert not has_custom_tools(set())
assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"})