[BugFix][Frontend] Pass reasoning_ended through render → generate (#60059) (#60062)

Signed-off-by: Yida Weng <84164537+YidaWeng@users.noreply.github.com>
Signed-off-by: Flora Feng <4florafeng@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>

tests/entrypoints/cohere/test_api_router.py

@@ -32,8 +32,14 @@ from vllm.entrypoints.cohere.protocol import (
     AssistantMessageResponse,
     CohereChatV2Response,
 )
-from vllm.entrypoints.cohere.serving import CohereServingChatV2
-from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
+from vllm.entrypoints.cohere.serving import (
+    _API_SERVER_ONLY_TEMPLATE_KWARGS,
+    CohereServingChatV2,
+)
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
+    GenerateRequest,
+    ReasoningParserKwargs,
+)
 from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
 from vllm.entrypoints.serve.exception_handling.handlers.http import (
     http_exception_handler,
@@ -97,6 +103,12 @@ class _RenderChatHandler:
     def to_chat_completion_request(self, request):
         return CohereServingChatV2._convert_v2_to_chat_completion(request)
 
+    def _engine_chat_template_kwargs(self, chat_template_kwargs):
+        return CohereServingChatV2._engine_chat_template_kwargs(
+            self,  # type: ignore[arg-type]
+            chat_template_kwargs,
+        )
+
 
 class _RenderHandler:
     """Minimal stand-in for :class:`ServingRender`.
@@ -459,6 +471,24 @@ class TestRenderEndpoint:
         # The conversion lower-cases safety_mode for the renderer.
         assert kwargs["safety_mode"] == "strict"
 
+    def test_api_server_only_kwargs_not_forwarded(self):
+        """As on ``/cohere/v2/chat``, the engine's reasoning parser must not
+        receive the template kwargs that only the API server reads."""
+        result = _generate_request()
+        result.reasoning_parser_kwargs = ReasoningParserKwargs(
+            chat_template_kwargs={
+                "safety_mode": "strict",
+                **dict.fromkeys(_API_SERVER_ONLY_TEMPLATE_KWARGS, "x"),
+            }
+        )
+        app = _build_render_app(_RenderChatHandler(), _RenderHandler(result))
+        with TestClient(app) as client:
+            r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
+        assert r.status_code == HTTPStatus.OK
+        assert r.json()["reasoning_parser_kwargs"] == {
+            "chat_template_kwargs": {"safety_mode": "strict"}
+        }
+
     def test_501_when_chat_handler_missing(self):
         app = _build_render_app(None, _RenderHandler(_generate_request()))
         with TestClient(app) as client:

tests/entrypoints/scale_out/render/test_render.py

@@ -38,6 +38,7 @@ def _build_responses_serving_render() -> ServingRender:
         OnlineRenderer.validate_chat_template.__get__(serving.online_renderer)
     )
     serving.online_renderer.trust_request_chat_template = True
+    serving.online_renderer.parser = None
     serving._check_model = AsyncMock(return_value=None)
     return serving
 
@@ -135,6 +136,37 @@ async def test_render_responses_rejects_empty_token_ids():
     assert response.error.message == "No token_ids rendered"
 
 
+@pytest.mark.asyncio
+@pytest.mark.skip_global_cleanup
+async def test_render_responses_carries_reasoning_parser_kwargs():
+    """/v1/responses gives the engine's reasoning parser the effective template
+    kwargs, so the rendered request must carry them to generate."""
+    serving = _build_responses_serving_render()
+    online = serving.online_renderer
+    online.parser = MagicMock(reasoning_parser_cls=object)
+    online.chat_template = None
+    online.chat_template_content_format = "auto"
+    online.default_chat_template_kwargs = {"enable_thinking": True}
+    online.effective_chat_template_kwargs = (
+        OnlineRenderer.effective_chat_template_kwargs.__get__(online)
+    )
+    online.render_responses = AsyncMock(
+        return_value=MagicMock(messages=[], engine_input={"prompt_token_ids": [7]})
+    )
+    request = ResponsesRequest(
+        model=MODEL_NAME,
+        input="Test prompt",
+        chat_template_kwargs={"enable_thinking": False},
+    )
+
+    response = await serving.render_responses_request(request)
+
+    assert not isinstance(response, ErrorResponse)
+    assert response.reasoning_parser_kwargs is not None
+    kwargs = response.reasoning_parser_kwargs.chat_template_kwargs
+    assert kwargs["enable_thinking"] is False
+
+
 @pytest.fixture(scope="module")
 def server():
     args: list[str] = ["--trust-request-chat-template"]

tests/entrypoints/scale_out/token_in_token_out/test_protocol.py

@@ -124,6 +124,14 @@ def test_generate_request_rejects_placeholder_outside_prompt():
         )
 
 
+def test_reasoning_parser_kwargs_rejects_unknown_keys():
+    """The engine splats these into its reasoning parser's constructor, so a
+    client must not be able to pass arbitrary keyword arguments."""
+    payload = {**_base_payload(), "reasoning_parser_kwargs": {"tokenizer": None}}
+    with pytest.raises(ValidationError, match="tokenizer"):
+        GenerateRequest.model_validate(payload)
+
+
 def test_output_mode_defaults_to_tokens():
     assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens"
 

tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py

@@ -589,6 +589,83 @@ async def test_generate_with_lora_adapter(client, tokenizer, messages):
     assert generate_res == completions_res
 
 
+def _structured_chat_body(messages, **overrides) -> dict:
+    return {
+        "model": MODEL_NAME,
+        "messages": messages,
+        "max_tokens": 64,
+        "temperature": 0.0,
+        "chat_template_kwargs": {"enable_thinking": True},
+        "response_format": {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "answer",
+                "schema": {
+                    "type": "object",
+                    "properties": {"count": {"type": "integer"}},
+                    "required": ["count"],
+                },
+            },
+        },
+        **overrides,
+    }
+
+
+async def _render_then_generate(client, body) -> tuple[dict, list[int]]:
+    render_resp = await client.post("/v1/chat/completions/render", json=body)
+    render_resp.raise_for_status()
+    generate_request = render_resp.json()
+    generate_resp = await client.post(GEN_ENDPOINT, json=generate_request)
+    generate_resp.raise_for_status()
+    return generate_request, generate_resp.json()["choices"][0]["token_ids"]
+
+
+# deepseek_v3 reads `enable_thinking` from the template kwargs and uses Qwen3's
+# <think> tokens, so the engine gates structured outputs differently when
+# either reasoning field is dropped between render and generate.
+@pytest.mark.asyncio
+@pytest.mark.skipif(
+    envs.VLLM_USE_RUST_FRONTEND,
+    reason="the Rust frontend does not forward reasoning fields to the engine",
+)
+@pytest.mark.parametrize(
+    "server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
+)
+async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, messages):
+    """With thinking on, structured outputs must wait for the reasoning to
+    end on generate, the same as on chat."""
+    body = _structured_chat_body(messages, return_token_ids=True)
+    chat_resp = await client.post("/v1/chat/completions", json=body)
+    chat_resp.raise_for_status()
+
+    generate_request, token_ids = await _render_then_generate(client, body)
+
+    assert generate_request["reasoning_ended"] is False
+    assert token_ids == chat_resp.json()["choices"][0]["token_ids"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(
+    envs.VLLM_USE_RUST_FRONTEND,
+    reason="the Rust frontend does not forward reasoning fields to the engine",
+)
+@pytest.mark.parametrize(
+    "server", [["--reasoning-parser", "deepseek_v3"]], indirect=True
+)
+async def test_render_then_generate_forwards_reasoning_ended(
+    client, tokenizer, messages
+):
+    """`include_reasoning=false` constrains from the first token on chat, so
+    generate must too, even though the engine's parser expects thinking."""
+    body = _structured_chat_body(messages, include_reasoning=False)
+
+    generate_request, token_ids = await _render_then_generate(client, body)
+
+    assert generate_request["reasoning_ended"] is True
+    output = json.loads(tokenizer.decode(token_ids, skip_special_tokens=True))
+    assert isinstance(output["count"], int)
+
+
 def _chat_prompt_token_ids(tokenizer, messages) -> list[int]:
     return tokenizer.apply_chat_template(
         messages,

vllm/entrypoints/cohere/api_router.py

@@ -207,6 +207,11 @@ if _SDK_AVAILABLE:
         if isinstance(result, ErrorResponse):
             return _error_response(result, raw_request)
 
+        if (kwargs := result.reasoning_parser_kwargs) is not None:
+            kwargs.chat_template_kwargs = handler._engine_chat_template_kwargs(
+                kwargs.chat_template_kwargs
+            )
+
         return JSONResponse(content=result.model_dump())
 
     class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):

vllm/entrypoints/openai/chat_completion/protocol.py

@@ -4,7 +4,7 @@
 # Adapted from
 # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
 import time
-from typing import Annotated, Any, ClassVar, Literal
+from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal
 
 from openai.types.chat.chat_completion_audio import (
     ChatCompletionAudio as OpenAIChatCompletionAudio,
@@ -53,6 +53,9 @@ from vllm.sampling_params import (
 )
 from vllm.utils import random_uuid
 
+if TYPE_CHECKING:
+    from vllm.parser.abstract_parser import Parser
+
 logger = init_logger(__name__)
 
 
@@ -566,6 +569,25 @@ class ChatCompletionRequest(OpenAIBaseModel):
     _grammar_from_parser: bool = PrivateAttr(default=False)
     """CAUTION: Should only be set by the parser-engine adapter's adjust_request."""
 
+    def resolve_reasoning_ended(
+        self, parser: "Parser | None", prompt_token_ids: list[int]
+    ) -> bool | None:
+        """Resolve the engine's `reasoning_ended` for structured outputs.
+
+        Call after `adjust_request`, which may set `_grammar_from_parser`.
+        `True` constrains from the first generated token; `None` lets the
+        engine check the prompt with its own reasoning parser.
+        """
+        if not self.include_reasoning:
+            return True
+        if self._grammar_from_parser:
+            # The Mistral grammar already includes an optional `think?`
+            # rule that handles both reasoning and non-reasoning outputs.
+            return True
+        if parser is not None and parser.reasoning_parser is not None:
+            return parser.is_reasoning_end(prompt_token_ids)
+        return None
+
     def build_chat_params(
         self,
         default_template: str | None,

vllm/entrypoints/openai/chat_completion/serving.py

@@ -364,17 +364,9 @@ class OpenAIServingChat(GenerateBaseServing):
                     session_id=session_id,
                 )
             else:
-                if not request.include_reasoning:
-                    reasoning_ended = True
-                elif request._grammar_from_parser:
-                    # The Mistral grammar already includes an optional
-                    # `think?` rule that handles both reasoning and
-                    # non-reasoning outputs.
-                    reasoning_ended = True
-                elif parser is not None and parser.reasoning_parser is not None:
-                    reasoning_ended = parser.is_reasoning_end(prompt_token_ids or [])
-                else:
-                    reasoning_ended = None
+                reasoning_ended = request.resolve_reasoning_ended(
+                    parser, prompt_token_ids or []
+                )
 
                 generator = self.engine_client.generate(
                     engine_input,

vllm/entrypoints/scale_out/render/serving.py

@@ -17,6 +17,7 @@ from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
 from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
     GenerateRequest,
     MultiModalFeatures,
+    ReasoningParserKwargs,
 )
 from vllm.entrypoints.serve.engine.protocol import ErrorResponse
 from vllm.entrypoints.serve.engine.serving import BaseServing
@@ -124,6 +125,20 @@ class ServingRender(BaseServing):
         )
         params = request.to_sampling_params(max_tokens, self.default_sampling_params)
 
+        # Resolve here what OpenAIServingChat passes to the engine:
+        # `_grammar_from_parser` (set by `adjust_request`) is not serialized.
+        reasoning_parser_kwargs = self._reasoning_parser_kwargs(request)
+        parser = None
+        if reasoning_parser_kwargs is not None:
+            assert self.online_renderer.parser is not None
+            parser = self.online_renderer.parser(
+                self.online_renderer.renderer.get_tokenizer(),
+                request.tools,
+                chat_template_kwargs=reasoning_parser_kwargs.chat_template_kwargs,
+                model_config=self.model_config,
+            )
+        reasoning_ended = request.resolve_reasoning_ended(parser, token_ids)
+
         request_id = f"chatcmpl-{random_uuid()}"
 
         return GenerateRequest(
@@ -132,6 +147,8 @@ class ServingRender(BaseServing):
             features=self._extract_mm_features(engine_input),
             sampling_params=params,
             model=request.model,
+            reasoning_ended=reasoning_ended,
+            reasoning_parser_kwargs=reasoning_parser_kwargs,
             stream=bool(request.stream),
             stream_options=(request.stream_options if request.stream else None),
             cache_salt=request.cache_salt,
@@ -263,6 +280,7 @@ class ServingRender(BaseServing):
             features=self._extract_mm_features(engine_input),
             sampling_params=params,
             model=request.model,
+            reasoning_parser_kwargs=self._reasoning_parser_kwargs(request),
             stream=bool(request.stream),
             cache_salt=request.cache_salt,
             priority=request.priority,
@@ -271,6 +289,18 @@ class ServingRender(BaseServing):
             token_offsets=engine_input.get("prompt_token_offsets"),
         )
 
+    def _reasoning_parser_kwargs(
+        self, request: ChatCompletionRequest | ResponsesRequest
+    ) -> ReasoningParserKwargs | None:
+        parser_cls = self.online_renderer.parser
+        if parser_cls is None or parser_cls.reasoning_parser_cls is None:
+            return None
+        return ReasoningParserKwargs(
+            chat_template_kwargs=self.online_renderer.effective_chat_template_kwargs(
+                request
+            )
+        )
+
     def _placeholder_metadata_fields(self, modality: str) -> set[str]:
         """Fields the EC consumer still requires after embeddings transfer."""
         if self._placeholder_metadata_parser_failed:

vllm/entrypoints/scale_out/token_in_token_out/protocol.py

@@ -4,6 +4,7 @@ from typing import Annotated, Any, Literal, TypeAlias
 
 from pydantic import (
     BaseModel,
+    ConfigDict,
     Discriminator,
     Field,
     NonNegativeInt,
@@ -147,6 +148,16 @@ class MultiModalFeatures(BaseModel):
         return self
 
 
+class ReasoningParserKwargs(BaseModel):
+    """Kwargs for the engine-side reasoning parser that gates structured
+    outputs. Typed so clients cannot pass arbitrary constructor kwargs."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    chat_template_kwargs: dict[str, Any] = Field(default_factory=dict)
+    """The effective chat template kwargs the prompt was rendered with."""
+
+
 class GenerateRequest(BaseModel):
     request_id: str = Field(
         default_factory=lambda: f"{random_uuid()}",
@@ -212,6 +223,15 @@ class GenerateRequest(BaseModel):
 
     model: str | None = None
 
+    reasoning_ended: bool | None = None
+    """Whether reasoning has ended before the first generated token, as
+    resolved by /render. `True` applies structured outputs from the first
+    token; `None` lets the engine check the prompt with its reasoning parser."""
+
+    reasoning_parser_kwargs: ReasoningParserKwargs | None = None
+    """Set by /render when it has a reasoning parser, so the engine-side
+    parser agrees with the frontend on flags such as `enable_thinking`."""
+
     return_token_ids: bool | None = Field(
         default=None,
         description=(

vllm/entrypoints/scale_out/token_in_token_out/serving.py

@@ -349,6 +349,12 @@ class ServingTokens(GenerateBaseServing):
             priority=request.priority,
             data_parallel_rank=data_parallel_rank,
             session_id=session_id,
+            reasoning_ended=request.reasoning_ended,
+            reasoning_parser_kwargs=(
+                request.reasoning_parser_kwargs.model_dump()
+                if request.reasoning_parser_kwargs is not None
+                else None
+            ),
         )
 
         assert result_generator is not None

vllm/renderers/online_renderer.py

@@ -209,6 +209,18 @@ class OnlineRenderer:
             )
         )
 
+    def effective_chat_template_kwargs(
+        self, request: ChatCompletionRequest | ResponsesRequest
+    ) -> dict[str, Any]:
+        return (
+            request.build_chat_params(
+                self.chat_template,
+                self.chat_template_content_format,
+            )
+            .with_defaults(self.default_chat_template_kwargs)
+            .chat_template_kwargs
+        )
+
     async def render_chat(
         self,
         request: ChatCompletionRequest,
@@ -356,14 +368,7 @@ class OnlineRenderer:
                 else None
             ),
         )
-        chat_template_kwargs = (
-            request.build_chat_params(
-                self.chat_template,
-                self.chat_template_content_format,
-            )
-            .with_defaults(self.default_chat_template_kwargs)
-            .chat_template_kwargs
-        )
+        chat_template_kwargs = self.effective_chat_template_kwargs(request)
         _, engine_inputs = await self.preprocess_chat(
             request,
             messages,