diff --git a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py index 467e23e7d3..f1de27bf75 100644 --- a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py +++ b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py @@ -3,10 +3,20 @@ import json from collections.abc import AsyncGenerator +from unittest.mock import MagicMock import httpx import pytest +from tests.entrypoints.openai.chat_completion.test_serving_chat import ( + BASE_MODEL_PATHS, + MockHFConfig, + MockModelConfig, + _build_renderer, +) +from tests.entrypoints.openai.chat_completion.test_serving_chat import ( + MODEL_NAME as MOCK_MODEL_NAME, +) from tests.utils import RemoteOpenAIServer from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata from vllm.entrypoints.openai.chat_completion.batch_serving import ( @@ -15,7 +25,10 @@ from vllm.entrypoints.openai.chat_completion.batch_serving import ( from vllm.entrypoints.openai.chat_completion.protocol import ( BatchChatCompletionRequest, ) +from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.outputs import CompletionOutput, RequestOutput +from vllm.renderers.online_renderer import OnlineRenderer +from vllm.v1.engine.async_llm import AsyncLLM # any model with a chat template defined in tokenizer_config should work here MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct" @@ -322,3 +335,53 @@ async def test_batched_echo_prepends_matching_assistant_prefix() -> None: ) assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER" + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_batched_harmony_response_format_uses_structural_tag() -> None: + """Harmony must constrain only the final channel of every conversation.""" + mock_engine = MagicMock(spec=AsyncLLM) + mock_engine.model_config = MockModelConfig() + mock_engine.errored = False + mock_engine.input_processor = MagicMock() + mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss") + mock_engine.renderer = _build_renderer(mock_engine.model_config) + mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok") + + models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS) + online_renderer = OnlineRenderer( + model_config=mock_engine.model_config, + renderer=mock_engine.renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + reasoning_parser="openai_gptoss", + ) + serving = OpenAIServingChatBatch( + mock_engine, + models, + response_role="assistant", + online_renderer=online_renderer, + chat_template=None, + chat_template_content_format="auto", + request_logger=None, + ) + + request = BatchChatCompletionRequest( + model=MOCK_MODEL_NAME, + messages=[ + [{"role": "user", "content": "France?"}], + [{"role": "user", "content": "Japan?"}], + ], + response_format={ + "type": "json_schema", + "json_schema": {"name": "city", "schema": {"type": "object"}}, + }, + ) + await serving.create_batch_chat_completion(request) + + calls = mock_engine.generate.call_args_list + assert len(calls) == 2 + for call in calls: + assert call.args[1].structured_outputs.structural_tag is not None diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py index a8a720d1e2..3fde958a3d 100644 --- a/vllm/entrypoints/openai/chat_completion/batch_serving.py +++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py @@ -91,6 +91,7 @@ class OpenAIServingChatBatch(OpenAIServingChat): for messages in request.messages: single_request = request.to_chat_completion_request(messages) if renderer.use_harmony: + renderer.adjust_harmony_request(single_request) conversation, engine_prompts = renderer._make_request_with_harmony( single_request, should_include_tools=tool_dicts is not None ) diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 139b783aaa..be0056a350 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -259,15 +259,7 @@ class OnlineRenderer: ) else: # For GPT-OSS. - if self.parser is not None: - # HarmonyParser doesn't need chat_template_kwargs - # TODO: Unify adjust_request() call with non-harmony branch - self.parser( - self.renderer.get_tokenizer(), - request.tools, - model_config=self.model_config, - ).adjust_request(request=request) - + self.adjust_harmony_request(request) should_include_tools = tool_dicts is not None conversation, engine_inputs = self._make_request_with_harmony( request, should_include_tools @@ -355,14 +347,7 @@ class OnlineRenderer: previous_response_outputs: list[ResponseOutputItem] | None, tool_server: "ToolServer | None", ) -> ResponsesRenderResult | ErrorResponse: - if self.parser is not None: - # HarmonyParser doesn't need chat_template_kwargs - # TODO: Unify adjust_request() call with non-harmony branch - self.parser( - self.renderer.get_tokenizer(), - request.tools, - model_config=self.model_config, - ).adjust_request(request=request) + self.adjust_harmony_request(request) if previous_messages is not None and any( not isinstance(message, OpenAIMessage) for message in previous_messages @@ -488,6 +473,24 @@ class OnlineRenderer: engine_input=engine_inputs[0], ) + def adjust_harmony_request( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> None: + """Apply the Harmony parser's ``adjust_request`` to ``request`` in place. + + Every Harmony render path must call this before sampling params are + built from ``request``. + """ + if self.parser is None: + return + # HarmonyParser doesn't need chat_template_kwargs + # TODO: Unify adjust_request() call with non-harmony branch + self.parser( + self.renderer.get_tokenizer(), + request.tools, + model_config=self.model_config, + ).adjust_request(request=request) + def _make_request_with_harmony( self, request: ChatCompletionRequest,