[Bugfix][Frontend] Apply Harmony adjust_request in batched chat completions (#58958)

Signed-off-by: sfeng33 <4florafeng@gmail.com>

tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py

@@ -3,10 +3,20 @@
 
 import json
 from collections.abc import AsyncGenerator
+from unittest.mock import MagicMock
 
 import httpx
 import pytest
 
+from tests.entrypoints.openai.chat_completion.test_serving_chat import (
+    BASE_MODEL_PATHS,
+    MockHFConfig,
+    MockModelConfig,
+    _build_renderer,
+)
+from tests.entrypoints.openai.chat_completion.test_serving_chat import (
+    MODEL_NAME as MOCK_MODEL_NAME,
+)
 from tests.utils import RemoteOpenAIServer
 from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
 from vllm.entrypoints.openai.chat_completion.batch_serving import (
@@ -15,7 +25,10 @@ from vllm.entrypoints.openai.chat_completion.batch_serving import (
 from vllm.entrypoints.openai.chat_completion.protocol import (
     BatchChatCompletionRequest,
 )
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.outputs import CompletionOutput, RequestOutput
+from vllm.renderers.online_renderer import OnlineRenderer
+from vllm.v1.engine.async_llm import AsyncLLM
 
 # any model with a chat template defined in tokenizer_config should work here
 MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
@@ -322,3 +335,53 @@ async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
     )
 
     assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip_global_cleanup
+async def test_batched_harmony_response_format_uses_structural_tag() -> None:
+    """Harmony must constrain only the final channel of every conversation."""
+    mock_engine = MagicMock(spec=AsyncLLM)
+    mock_engine.model_config = MockModelConfig()
+    mock_engine.errored = False
+    mock_engine.input_processor = MagicMock()
+    mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
+    mock_engine.renderer = _build_renderer(mock_engine.model_config)
+    mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
+
+    models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
+    online_renderer = OnlineRenderer(
+        model_config=mock_engine.model_config,
+        renderer=mock_engine.renderer,
+        request_logger=None,
+        chat_template=None,
+        chat_template_content_format="auto",
+        reasoning_parser="openai_gptoss",
+    )
+    serving = OpenAIServingChatBatch(
+        mock_engine,
+        models,
+        response_role="assistant",
+        online_renderer=online_renderer,
+        chat_template=None,
+        chat_template_content_format="auto",
+        request_logger=None,
+    )
+
+    request = BatchChatCompletionRequest(
+        model=MOCK_MODEL_NAME,
+        messages=[
+            [{"role": "user", "content": "France?"}],
+            [{"role": "user", "content": "Japan?"}],
+        ],
+        response_format={
+            "type": "json_schema",
+            "json_schema": {"name": "city", "schema": {"type": "object"}},
+        },
+    )
+    await serving.create_batch_chat_completion(request)
+
+    calls = mock_engine.generate.call_args_list
+    assert len(calls) == 2
+    for call in calls:
+        assert call.args[1].structured_outputs.structural_tag is not None

vllm/entrypoints/openai/chat_completion/batch_serving.py

@@ -91,6 +91,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
         for messages in request.messages:
             single_request = request.to_chat_completion_request(messages)
             if renderer.use_harmony:
+                renderer.adjust_harmony_request(single_request)
                 conversation, engine_prompts = renderer._make_request_with_harmony(
                     single_request, should_include_tools=tool_dicts is not None
                 )

vllm/renderers/online_renderer.py

@@ -259,15 +259,7 @@ class OnlineRenderer:
             )
         else:
             # For GPT-OSS.
-            if self.parser is not None:
-                # HarmonyParser doesn't need chat_template_kwargs
-                # TODO: Unify adjust_request() call with non-harmony branch
-                self.parser(
-                    self.renderer.get_tokenizer(),
-                    request.tools,
-                    model_config=self.model_config,
-                ).adjust_request(request=request)
-
+            self.adjust_harmony_request(request)
             should_include_tools = tool_dicts is not None
             conversation, engine_inputs = self._make_request_with_harmony(
                 request, should_include_tools
@@ -355,14 +347,7 @@ class OnlineRenderer:
         previous_response_outputs: list[ResponseOutputItem] | None,
         tool_server: "ToolServer | None",
     ) -> ResponsesRenderResult | ErrorResponse:
-        if self.parser is not None:
-            # HarmonyParser doesn't need chat_template_kwargs
-            # TODO: Unify adjust_request() call with non-harmony branch
-            self.parser(
-                self.renderer.get_tokenizer(),
-                request.tools,
-                model_config=self.model_config,
-            ).adjust_request(request=request)
+        self.adjust_harmony_request(request)
 
         if previous_messages is not None and any(
             not isinstance(message, OpenAIMessage) for message in previous_messages
@@ -488,6 +473,24 @@ class OnlineRenderer:
             engine_input=engine_inputs[0],
         )
 
+    def adjust_harmony_request(
+        self, request: ChatCompletionRequest | ResponsesRequest
+    ) -> None:
+        """Apply the Harmony parser's ``adjust_request`` to ``request`` in place.
+
+        Every Harmony render path must call this before sampling params are
+        built from ``request``.
+        """
+        if self.parser is None:
+            return
+        # HarmonyParser doesn't need chat_template_kwargs
+        # TODO: Unify adjust_request() call with non-harmony branch
+        self.parser(
+            self.renderer.get_tokenizer(),
+            request.tools,
+            model_config=self.model_config,
+        ).adjust_request(request=request)
+
     def _make_request_with_harmony(
         self,
         request: ChatCompletionRequest,