diff --git a/tests/entrypoints/cohere/test_api_router.py b/tests/entrypoints/cohere/test_api_router.py index 94324dbed7..1bb2151621 100644 --- a/tests/entrypoints/cohere/test_api_router.py +++ b/tests/entrypoints/cohere/test_api_router.py @@ -32,8 +32,14 @@ from vllm.entrypoints.cohere.protocol import ( AssistantMessageResponse, CohereChatV2Response, ) -from vllm.entrypoints.cohere.serving import CohereServingChatV2 -from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest +from vllm.entrypoints.cohere.serving import ( + _API_SERVER_ONLY_TEMPLATE_KWARGS, + CohereServingChatV2, +) +from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + GenerateRequest, + ReasoningParserKwargs, +) from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse from vllm.entrypoints.serve.exception_handling.handlers.http import ( http_exception_handler, @@ -97,6 +103,12 @@ class _RenderChatHandler: def to_chat_completion_request(self, request): return CohereServingChatV2._convert_v2_to_chat_completion(request) + def _engine_chat_template_kwargs(self, chat_template_kwargs): + return CohereServingChatV2._engine_chat_template_kwargs( + self, # type: ignore[arg-type] + chat_template_kwargs, + ) + class _RenderHandler: """Minimal stand-in for :class:`ServingRender`. @@ -459,6 +471,24 @@ class TestRenderEndpoint: # The conversion lower-cases safety_mode for the renderer. assert kwargs["safety_mode"] == "strict" + def test_api_server_only_kwargs_not_forwarded(self): + """As on ``/cohere/v2/chat``, the engine's reasoning parser must not + receive the template kwargs that only the API server reads.""" + result = _generate_request() + result.reasoning_parser_kwargs = ReasoningParserKwargs( + chat_template_kwargs={ + "safety_mode": "strict", + **dict.fromkeys(_API_SERVER_ONLY_TEMPLATE_KWARGS, "x"), + } + ) + app = _build_render_app(_RenderChatHandler(), _RenderHandler(result)) + with TestClient(app) as client: + r = client.post("/cohere/v2/chat/render", json=_minimal_request_body()) + assert r.status_code == HTTPStatus.OK + assert r.json()["reasoning_parser_kwargs"] == { + "chat_template_kwargs": {"safety_mode": "strict"} + } + def test_501_when_chat_handler_missing(self): app = _build_render_app(None, _RenderHandler(_generate_request())) with TestClient(app) as client: diff --git a/tests/entrypoints/scale_out/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py index 0daf4d405e..6c11a518d7 100644 --- a/tests/entrypoints/scale_out/render/test_render.py +++ b/tests/entrypoints/scale_out/render/test_render.py @@ -38,6 +38,7 @@ def _build_responses_serving_render() -> ServingRender: OnlineRenderer.validate_chat_template.__get__(serving.online_renderer) ) serving.online_renderer.trust_request_chat_template = True + serving.online_renderer.parser = None serving._check_model = AsyncMock(return_value=None) return serving @@ -135,6 +136,37 @@ async def test_render_responses_rejects_empty_token_ids(): assert response.error.message == "No token_ids rendered" +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_render_responses_carries_reasoning_parser_kwargs(): + """/v1/responses gives the engine's reasoning parser the effective template + kwargs, so the rendered request must carry them to generate.""" + serving = _build_responses_serving_render() + online = serving.online_renderer + online.parser = MagicMock(reasoning_parser_cls=object) + online.chat_template = None + online.chat_template_content_format = "auto" + online.default_chat_template_kwargs = {"enable_thinking": True} + online.effective_chat_template_kwargs = ( + OnlineRenderer.effective_chat_template_kwargs.__get__(online) + ) + online.render_responses = AsyncMock( + return_value=MagicMock(messages=[], engine_input={"prompt_token_ids": [7]}) + ) + request = ResponsesRequest( + model=MODEL_NAME, + input="Test prompt", + chat_template_kwargs={"enable_thinking": False}, + ) + + response = await serving.render_responses_request(request) + + assert not isinstance(response, ErrorResponse) + assert response.reasoning_parser_kwargs is not None + kwargs = response.reasoning_parser_kwargs.chat_template_kwargs + assert kwargs["enable_thinking"] is False + + @pytest.fixture(scope="module") def server(): args: list[str] = ["--trust-request-chat-template"] diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py index 9894d09180..6a57e3b2b4 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py @@ -124,6 +124,14 @@ def test_generate_request_rejects_placeholder_outside_prompt(): ) +def test_reasoning_parser_kwargs_rejects_unknown_keys(): + """The engine splats these into its reasoning parser's constructor, so a + client must not be able to pass arbitrary keyword arguments.""" + payload = {**_base_payload(), "reasoning_parser_kwargs": {"tokenizer": None}} + with pytest.raises(ValidationError, match="tokenizer"): + GenerateRequest.model_validate(payload) + + def test_output_mode_defaults_to_tokens(): assert GenerateRequest.model_validate(_base_payload()).output_mode == "tokens" diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index 2ee8f76e0f..1027dcef03 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -589,6 +589,83 @@ async def test_generate_with_lora_adapter(client, tokenizer, messages): assert generate_res == completions_res +def _structured_chat_body(messages, **overrides) -> dict: + return { + "model": MODEL_NAME, + "messages": messages, + "max_tokens": 64, + "temperature": 0.0, + "chat_template_kwargs": {"enable_thinking": True}, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "answer", + "schema": { + "type": "object", + "properties": {"count": {"type": "integer"}}, + "required": ["count"], + }, + }, + }, + **overrides, + } + + +async def _render_then_generate(client, body) -> tuple[dict, list[int]]: + render_resp = await client.post("/v1/chat/completions/render", json=body) + render_resp.raise_for_status() + generate_request = render_resp.json() + generate_resp = await client.post(GEN_ENDPOINT, json=generate_request) + generate_resp.raise_for_status() + return generate_request, generate_resp.json()["choices"][0]["token_ids"] + + +# deepseek_v3 reads `enable_thinking` from the template kwargs and uses Qwen3's +# tokens, so the engine gates structured outputs differently when +# either reasoning field is dropped between render and generate. +@pytest.mark.asyncio +@pytest.mark.skipif( + envs.VLLM_USE_RUST_FRONTEND, + reason="the Rust frontend does not forward reasoning fields to the engine", +) +@pytest.mark.parametrize( + "server", [["--reasoning-parser", "deepseek_v3"]], indirect=True +) +async def test_render_then_generate_forwards_reasoning_parser_kwargs(client, messages): + """With thinking on, structured outputs must wait for the reasoning to + end on generate, the same as on chat.""" + body = _structured_chat_body(messages, return_token_ids=True) + chat_resp = await client.post("/v1/chat/completions", json=body) + chat_resp.raise_for_status() + + generate_request, token_ids = await _render_then_generate(client, body) + + assert generate_request["reasoning_ended"] is False + assert token_ids == chat_resp.json()["choices"][0]["token_ids"] + + +@pytest.mark.asyncio +@pytest.mark.skipif( + envs.VLLM_USE_RUST_FRONTEND, + reason="the Rust frontend does not forward reasoning fields to the engine", +) +@pytest.mark.parametrize( + "server", [["--reasoning-parser", "deepseek_v3"]], indirect=True +) +async def test_render_then_generate_forwards_reasoning_ended( + client, tokenizer, messages +): + """`include_reasoning=false` constrains from the first token on chat, so + generate must too, even though the engine's parser expects thinking.""" + body = _structured_chat_body(messages, include_reasoning=False) + + generate_request, token_ids = await _render_then_generate(client, body) + + assert generate_request["reasoning_ended"] is True + output = json.loads(tokenizer.decode(token_ids, skip_special_tokens=True)) + assert isinstance(output["count"], int) + + def _chat_prompt_token_ids(tokenizer, messages) -> list[int]: return tokenizer.apply_chat_template( messages, diff --git a/vllm/entrypoints/cohere/api_router.py b/vllm/entrypoints/cohere/api_router.py index eaac36f133..db6366df71 100644 --- a/vllm/entrypoints/cohere/api_router.py +++ b/vllm/entrypoints/cohere/api_router.py @@ -207,6 +207,11 @@ if _SDK_AVAILABLE: if isinstance(result, ErrorResponse): return _error_response(result, raw_request) + if (kwargs := result.reasoning_parser_kwargs) is not None: + kwargs.chat_template_kwargs = handler._engine_chat_template_kwargs( + kwargs.chat_template_kwargs + ) + return JSONResponse(content=result.model_dump()) class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware): diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 6a70624be4..d2d1cbab93 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -4,7 +4,7 @@ # Adapted from # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py import time -from typing import Annotated, Any, ClassVar, Literal +from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal from openai.types.chat.chat_completion_audio import ( ChatCompletionAudio as OpenAIChatCompletionAudio, @@ -53,6 +53,9 @@ from vllm.sampling_params import ( ) from vllm.utils import random_uuid +if TYPE_CHECKING: + from vllm.parser.abstract_parser import Parser + logger = init_logger(__name__) @@ -566,6 +569,25 @@ class ChatCompletionRequest(OpenAIBaseModel): _grammar_from_parser: bool = PrivateAttr(default=False) """CAUTION: Should only be set by the parser-engine adapter's adjust_request.""" + def resolve_reasoning_ended( + self, parser: "Parser | None", prompt_token_ids: list[int] + ) -> bool | None: + """Resolve the engine's `reasoning_ended` for structured outputs. + + Call after `adjust_request`, which may set `_grammar_from_parser`. + `True` constrains from the first generated token; `None` lets the + engine check the prompt with its own reasoning parser. + """ + if not self.include_reasoning: + return True + if self._grammar_from_parser: + # The Mistral grammar already includes an optional `think?` + # rule that handles both reasoning and non-reasoning outputs. + return True + if parser is not None and parser.reasoning_parser is not None: + return parser.is_reasoning_end(prompt_token_ids) + return None + def build_chat_params( self, default_template: str | None, diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 400089c963..361cc52f72 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -364,17 +364,9 @@ class OpenAIServingChat(GenerateBaseServing): session_id=session_id, ) else: - if not request.include_reasoning: - reasoning_ended = True - elif request._grammar_from_parser: - # The Mistral grammar already includes an optional - # `think?` rule that handles both reasoning and - # non-reasoning outputs. - reasoning_ended = True - elif parser is not None and parser.reasoning_parser is not None: - reasoning_ended = parser.is_reasoning_end(prompt_token_ids or []) - else: - reasoning_ended = None + reasoning_ended = request.resolve_reasoning_ended( + parser, prompt_token_ids or [] + ) generator = self.engine_client.generate( engine_input, diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index 117ede0f9b..bddbe90f28 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -17,6 +17,7 @@ from vllm.entrypoints.scale_out.token_in_token_out.mm_features import ( from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( GenerateRequest, MultiModalFeatures, + ReasoningParserKwargs, ) from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.engine.serving import BaseServing @@ -124,6 +125,20 @@ class ServingRender(BaseServing): ) params = request.to_sampling_params(max_tokens, self.default_sampling_params) + # Resolve here what OpenAIServingChat passes to the engine: + # `_grammar_from_parser` (set by `adjust_request`) is not serialized. + reasoning_parser_kwargs = self._reasoning_parser_kwargs(request) + parser = None + if reasoning_parser_kwargs is not None: + assert self.online_renderer.parser is not None + parser = self.online_renderer.parser( + self.online_renderer.renderer.get_tokenizer(), + request.tools, + chat_template_kwargs=reasoning_parser_kwargs.chat_template_kwargs, + model_config=self.model_config, + ) + reasoning_ended = request.resolve_reasoning_ended(parser, token_ids) + request_id = f"chatcmpl-{random_uuid()}" return GenerateRequest( @@ -132,6 +147,8 @@ class ServingRender(BaseServing): features=self._extract_mm_features(engine_input), sampling_params=params, model=request.model, + reasoning_ended=reasoning_ended, + reasoning_parser_kwargs=reasoning_parser_kwargs, stream=bool(request.stream), stream_options=(request.stream_options if request.stream else None), cache_salt=request.cache_salt, @@ -263,6 +280,7 @@ class ServingRender(BaseServing): features=self._extract_mm_features(engine_input), sampling_params=params, model=request.model, + reasoning_parser_kwargs=self._reasoning_parser_kwargs(request), stream=bool(request.stream), cache_salt=request.cache_salt, priority=request.priority, @@ -271,6 +289,18 @@ class ServingRender(BaseServing): token_offsets=engine_input.get("prompt_token_offsets"), ) + def _reasoning_parser_kwargs( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> ReasoningParserKwargs | None: + parser_cls = self.online_renderer.parser + if parser_cls is None or parser_cls.reasoning_parser_cls is None: + return None + return ReasoningParserKwargs( + chat_template_kwargs=self.online_renderer.effective_chat_template_kwargs( + request + ) + ) + def _placeholder_metadata_fields(self, modality: str) -> set[str]: """Fields the EC consumer still requires after embeddings transfer.""" if self._placeholder_metadata_parser_failed: diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index 31bd7e2eae..4e29f7e59d 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -4,6 +4,7 @@ from typing import Annotated, Any, Literal, TypeAlias from pydantic import ( BaseModel, + ConfigDict, Discriminator, Field, NonNegativeInt, @@ -147,6 +148,16 @@ class MultiModalFeatures(BaseModel): return self +class ReasoningParserKwargs(BaseModel): + """Kwargs for the engine-side reasoning parser that gates structured + outputs. Typed so clients cannot pass arbitrary constructor kwargs.""" + + model_config = ConfigDict(extra="forbid") + + chat_template_kwargs: dict[str, Any] = Field(default_factory=dict) + """The effective chat template kwargs the prompt was rendered with.""" + + class GenerateRequest(BaseModel): request_id: str = Field( default_factory=lambda: f"{random_uuid()}", @@ -212,6 +223,15 @@ class GenerateRequest(BaseModel): model: str | None = None + reasoning_ended: bool | None = None + """Whether reasoning has ended before the first generated token, as + resolved by /render. `True` applies structured outputs from the first + token; `None` lets the engine check the prompt with its reasoning parser.""" + + reasoning_parser_kwargs: ReasoningParserKwargs | None = None + """Set by /render when it has a reasoning parser, so the engine-side + parser agrees with the frontend on flags such as `enable_thinking`.""" + return_token_ids: bool | None = Field( default=None, description=( diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 6b0195b104..e7afa0401b 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -349,6 +349,12 @@ class ServingTokens(GenerateBaseServing): priority=request.priority, data_parallel_rank=data_parallel_rank, session_id=session_id, + reasoning_ended=request.reasoning_ended, + reasoning_parser_kwargs=( + request.reasoning_parser_kwargs.model_dump() + if request.reasoning_parser_kwargs is not None + else None + ), ) assert result_generator is not None diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 23352ef1cf..e8220938b2 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -209,6 +209,18 @@ class OnlineRenderer: ) ) + def effective_chat_template_kwargs( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> dict[str, Any]: + return ( + request.build_chat_params( + self.chat_template, + self.chat_template_content_format, + ) + .with_defaults(self.default_chat_template_kwargs) + .chat_template_kwargs + ) + async def render_chat( self, request: ChatCompletionRequest, @@ -356,14 +368,7 @@ class OnlineRenderer: else None ), ) - chat_template_kwargs = ( - request.build_chat_params( - self.chat_template, - self.chat_template_content_format, - ) - .with_defaults(self.default_chat_template_kwargs) - .chat_template_kwargs - ) + chat_template_kwargs = self.effective_chat_template_kwargs(request) _, engine_inputs = await self.preprocess_chat( request, messages,