[Refactor] Chat Completions Harmony Refactor, non-streaming path. (#45171)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-06-11 21:18:30 +00:00
committed by GitHub
parent 5a6c7b7ab5
commit f712fd0d7d
16 changed files with 822 additions and 816 deletions
@@ -38,12 +38,12 @@ from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.exceptions import VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser import HarmonyParser
from vllm.renderers.hf import HfRenderer
from vllm.renderers.mistral import MistralRenderer
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.tool_parsers import ToolParserManager
from vllm.v1.engine.async_llm import AsyncLLM
GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
@@ -575,7 +575,13 @@ def _build_serving_render(
)
def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
def _build_serving_chat(
engine: AsyncLLM,
*,
reasoning_parser: str = "",
tool_parser: str | None = None,
enable_auto_tools: bool = False,
) -> OpenAIServingChat:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
@@ -590,6 +596,9 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
reasoning_parser=reasoning_parser,
tool_parser=tool_parser,
enable_auto_tools=enable_auto_tools,
)
return serving_chat
@@ -637,7 +646,7 @@ async def test_serving_chat_returns_correct_model_name():
serving_chat = _build_serving_chat(mock_engine)
messages = [{"role": "user", "content": "what is 1+1?"}]
async def return_model_name(*args):
async def return_model_name(*args, **kwargs):
return args[3]
serving_chat.chat_completion_full_generator = return_model_name
@@ -1210,15 +1219,21 @@ class TestServingChatWithHarmony:
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
mock_engine.model_config.hf_text_config = MockHFConfig(model_type="gpt_oss")
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
return mock_engine
@pytest.fixture()
def serving_chat(self, mock_engine) -> OpenAIServingChat:
chat = _build_serving_chat(mock_engine)
chat.use_harmony = True
chat.tool_parser = ToolParserManager.get_tool_parser("openai")
chat = _build_serving_chat(
mock_engine,
reasoning_parser="openai_gptoss",
tool_parser="openai",
enable_auto_tools=True,
)
assert chat.parser_cls is HarmonyParser
return chat
def mock_request_output_from_req_and_token_ids(
@@ -1277,6 +1292,7 @@ class TestServingChatWithHarmony:
stream: bool = False,
) -> ChatCompletionResponse:
harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
tokenizer = get_tokenizer(GPT_OSS_MODEL_NAME)
async def result_generator():
if stream:
@@ -1304,11 +1320,12 @@ class TestServingChatWithHarmony:
request_id=req.request_id,
model_name=req.model,
conversation=[],
tokenizer=get_tokenizer(req.model),
tokenizer=tokenizer,
request_metadata=RequestResponseMetadata(
request_id=req.request_id,
model_name=req.model,
),
chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req),
)
if stream:
@@ -1316,11 +1333,18 @@ class TestServingChatWithHarmony:
return await result
@pytest.mark.asyncio
async def test_simple_chat(self, serving_chat, stream):
@pytest.mark.parametrize(
"include_reasoning", [True, False], ids=["with_reasoning", "no_reasoning"]
)
async def test_simple_chat(self, serving_chat, stream, include_reasoning):
messages = [{"role": "user", "content": "what is 1+1?"}]
# Test the Harmony messages for the first turn's input
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
req = ChatCompletionRequest(
model=MODEL_NAME,
messages=messages,
include_reasoning=include_reasoning,
)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
@@ -1342,7 +1366,11 @@ class TestServingChatWithHarmony:
response = await self.generate_response_from_harmony_str(
serving_chat, req, response_str, stream=stream
)
verify_chat_response(response, content=final_str, reasoning=reasoning_str)
verify_chat_response(
response,
content=final_str,
reasoning=reasoning_str if include_reasoning else None,
)
# Add the output messages from the first turn as input to the second turn
for choice in response.choices: