From bab0ac61f8dae8020df6725c90474bdd647f1c40 Mon Sep 17 00:00:00 2001 From: dingdangmao <2756823972@qq.com> Date: Fri, 9 Oct 2026 03:32:46 +0800 Subject: [PATCH] [Bugfix][Frontend] Forward the reasoning wiring to the engine for batch chat completions (#59263) Signed-off-by: apex-mochen <2756823972@qq.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../test_batched_chat_completions.py | 102 ++++++++++++++++++ .../openai/chat_completion/batch_serving.py | 13 ++- .../openai/chat_completion/serving.py | 37 ++++--- 3 files changed, 139 insertions(+), 13 deletions(-) diff --git a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py index 946615f971..41c74bb5ce 100644 --- a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py +++ b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py @@ -10,6 +10,7 @@ import pytest from tests.entrypoints.openai.chat_completion.test_serving_chat import ( BASE_MODEL_PATHS, + CHAT_TEMPLATE, MockHFConfig, MockModelConfig, _build_renderer, @@ -24,10 +25,13 @@ from vllm.entrypoints.openai.chat_completion.batch_serving import ( ) from vllm.entrypoints.openai.chat_completion.protocol import ( BatchChatCompletionRequest, + ChatCompletionRequest, ) +from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.exceptions import VLLMValidationError from vllm.outputs import CompletionOutput, RequestOutput +from vllm.reasoning import ReasoningParserManager from vllm.renderers.online_renderer import OnlineRenderer from vllm.v1.engine.async_llm import AsyncLLM @@ -454,3 +458,101 @@ async def test_batched_parser_receives_each_prompt_prefix() -> None: ) assert prefixes == [[1], [2]] + + +REASONING_PARSER = "kimi_k3" + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_batched_reasoning_wiring_matches_single_chat() -> None: + """`/v1/chat/completions/batch` must hand the engine the same reasoning + wiring as `/v1/chat/completions`. + + Both `reasoning_ended` and `reasoning_parser_kwargs` are carried on the + request into `EngineCoreRequest`, where + `StructuredOutputManager._get_reasoner()` rebuilds the request-local + reasoning parser from the forwarded `chat_template_kwargs` -- "so the + structured-output gate observes the same template kwargs used by the + frontend". A divergence makes a template-kwarg-driven parser fall back to + its default (`thinking` on) for a request that disabled thinking, so + `is_reasoning_end()` keeps reporting False, the structured-output gate + never opens, and the grammar is silently never applied. + + Both frontends run over one and the same model and renderer, so the only + thing that can differ between the two calls is the serving code itself. + """ + mock_model_config = MockModelConfig() + shared_renderer = _build_renderer(mock_model_config) + + def _build_serving(cls): + mock_engine = MagicMock(spec=AsyncLLM) + mock_engine.model_config = mock_model_config + mock_engine.errored = False + mock_engine.input_processor = MagicMock() + mock_engine.renderer = shared_renderer + mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok") + + models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS) + online_renderer = OnlineRenderer( + model_config=mock_model_config, + renderer=mock_engine.renderer, + request_logger=None, + chat_template=CHAT_TEMPLATE, + chat_template_content_format="auto", + reasoning_parser=REASONING_PARSER, + ) + return mock_engine, cls( + mock_engine, + models, + response_role="assistant", + online_renderer=online_renderer, + chat_template=CHAT_TEMPLATE, + chat_template_content_format="auto", + request_logger=None, + reasoning_parser=REASONING_PARSER, + ) + + conversation = [{"role": "user", "content": "What is the capital of France?"}] + chat_template_kwargs = {"enable_thinking": False} + + single_engine, single_serving = _build_serving(OpenAIServingChat) + await single_serving.create_chat_completion( + ChatCompletionRequest( + model=MOCK_MODEL_NAME, + messages=conversation, + chat_template_kwargs=chat_template_kwargs, + ) + ) + single_call = single_engine.generate.call_args_list[-1] + + batch_engine, batch_serving = _build_serving(OpenAIServingChatBatch) + await batch_serving.create_batch_chat_completion( + BatchChatCompletionRequest( + model=MOCK_MODEL_NAME, + messages=[conversation], + chat_template_kwargs=chat_template_kwargs, + ) + ) + batch_call = batch_engine.generate.call_args_list[-1] + + # Guard against a vacuous comparison: the reference path must be wired up. + assert single_call.kwargs["reasoning_ended"] is not None + assert single_call.kwargs["reasoning_parser_kwargs"] is not None + + assert batch_call.kwargs["reasoning_ended"] == single_call.kwargs["reasoning_ended"] + assert ( + batch_call.kwargs["reasoning_parser_kwargs"] + == single_call.kwargs["reasoning_parser_kwargs"] + ) + + # ...and the forwarded kwargs must actually turn thinking off for the + # engine-side parser, i.e. the structured-output gate opens right away. + engine_chat_template_kwargs = batch_call.kwargs["reasoning_parser_kwargs"][ + "chat_template_kwargs" + ] + reasoner = ReasoningParserManager.get_reasoning_parser(REASONING_PARSER)( + tokenizer=batch_engine.renderer.tokenizer, + chat_template_kwargs=engine_chat_template_kwargs, + ) + assert reasoner.is_reasoning_end([]) is True diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py index e630b97f18..d3ea1a1ac3 100644 --- a/vllm/entrypoints/openai/chat_completion/batch_serving.py +++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py @@ -169,6 +169,12 @@ class OpenAIServingChatBatch(OpenAIServingChat): else await self._get_trace_headers(raw_request.headers) ) session_id = self._get_session_id(single_request, raw_request) + chat_template_kwargs = self._effective_chat_template_kwargs(single_request) + parser = ( + self._make_parser(single_request, tokenizer, chat_template_kwargs) + if self.parser_cls is not None + else None + ) generators.append( self.engine_client.generate( engine_prompt, @@ -179,7 +185,12 @@ class OpenAIServingChatBatch(OpenAIServingChat): priority=request.priority, data_parallel_rank=data_parallel_rank, session_id=session_id, - reasoning_ended=None, + **self._engine_reasoning_kwargs( + single_request, + parser, + chat_template_kwargs, + self._extract_prompt_components(engine_prompt).token_ids, + ), ) ) diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 9b0737c25d..69b46ba67f 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -233,6 +233,28 @@ class OpenAIServingChat(GenerateBaseServing): """ return chat_template_kwargs + def _engine_reasoning_kwargs( + self, + request: ChatCompletionRequest, + parser: Parser | None, + chat_template_kwargs: dict[str, Any], + prompt_token_ids: list[int] | None, + ) -> dict[str, Any]: + """`reasoning_ended` and `reasoning_parser_kwargs` for `generate()`.""" + reasoning_parser_kwargs = None + if parser is not None and parser.reasoning_parser is not None: + reasoning_parser_kwargs = { + "chat_template_kwargs": self._engine_chat_template_kwargs( + chat_template_kwargs + ), + } + return { + "reasoning_ended": request.resolve_reasoning_ended( + parser, prompt_token_ids or [] + ), + "reasoning_parser_kwargs": reasoning_parser_kwargs, + } + async def render_chat_request( self, request: ChatCompletionRequest, @@ -364,10 +386,6 @@ class OpenAIServingChat(GenerateBaseServing): session_id=session_id, ) else: - reasoning_ended = request.resolve_reasoning_ended( - parser, prompt_token_ids or [] - ) - generator = self.engine_client.generate( engine_input, sampling_params, @@ -377,14 +395,9 @@ class OpenAIServingChat(GenerateBaseServing): priority=self._get_priority(request, raw_request), data_parallel_rank=data_parallel_rank, session_id=session_id, - reasoning_ended=reasoning_ended, - reasoning_parser_kwargs={ - "chat_template_kwargs": self._engine_chat_template_kwargs( - chat_template_kwargs - ), - } - if parser is not None and parser.reasoning_parser is not None - else None, + **self._engine_reasoning_kwargs( + request, parser, chat_template_kwargs, prompt_token_ids + ), ) generators.append(generator)