From a07642667db1284ad2128d7f9ef089e6b0d24a4c Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 1 May 2026 14:38:02 +0800 Subject: [PATCH] [Bugfix] Pass reasoning parser kwargs to structured output (#41199) Signed-off-by: Bugen Zhao Co-authored-by: wang.yuqi --- .../test_reasoning_structured_output.py | 116 +++++++++++------- vllm/engine/protocol.py | 1 + .../openai/chat_completion/serving.py | 5 + vllm/entrypoints/openai/responses/serving.py | 11 +- vllm/v1/engine/__init__.py | 1 + vllm/v1/engine/async_llm.py | 7 +- vllm/v1/request.py | 5 + vllm/v1/structured_output/__init__.py | 33 +++-- vllm/v1/structured_output/request.py | 9 +- 9 files changed, 132 insertions(+), 56 deletions(-) diff --git a/tests/v1/structured_output/test_reasoning_structured_output.py b/tests/v1/structured_output/test_reasoning_structured_output.py index 98a25e41df..a12b36b8a4 100644 --- a/tests/v1/structured_output/test_reasoning_structured_output.py +++ b/tests/v1/structured_output/test_reasoning_structured_output.py @@ -8,11 +8,16 @@ from unittest.mock import Mock import pytest from vllm.config import ModelConfig, SchedulerConfig, VllmConfig -from vllm.reasoning import ReasoningParser from vllm.v1.request import Request from vllm.v1.structured_output import StructuredOutputManager +class MockReasoner: + def __init__(self, tokenizer): + self.is_reasoning_end = Mock(return_value=False) + self.is_reasoning_end_streaming = Mock(return_value=False) + + class TestReasoningStructuredOutput: """Test reasoning-aware structured output functionality.""" @@ -50,13 +55,6 @@ class TestReasoningStructuredOutput: config.speculative_config = None return config - @pytest.fixture - def mock_reasoning_parser(self): - """Create a mock ReasoningParser.""" - parser = Mock(spec=ReasoningParser) - parser.is_reasoning_end = Mock(return_value=False) - return parser - @pytest.fixture def mock_request_with_structured_output(self): """Create a mock request with structured output.""" @@ -64,6 +62,8 @@ class TestReasoningStructuredOutput: request.structured_output_request = Mock() request.structured_output_request.reasoning_ended = None request.structured_output_request.grammar = Mock() + request.structured_output_request.reasoning_parser_kwargs = None + request.structured_output_request.reasoner = None request.structured_output_request.grammar.is_terminated = Mock( return_value=False ) @@ -74,6 +74,13 @@ class TestReasoningStructuredOutput: request.num_output_placeholders = 0 return request + @pytest.fixture + def manager_with_reasoner(self, mock_vllm_config): + manager = StructuredOutputManager(mock_vllm_config) + manager.reasoner_cls = MockReasoner + manager.tokenizer = Mock() + return manager + def test_should_fill_bitmask_with_enable_in_reasoning( self, mock_vllm_config, mock_request_with_structured_output ): @@ -89,22 +96,17 @@ class TestReasoningStructuredOutput: def test_should_fill_bitmask_without_enable_in_reasoning( self, - mock_vllm_config, + manager_with_reasoner, mock_request_with_structured_output, - mock_reasoning_parser, ): """Test should_fill_bitmask when enable_in_reasoning is False.""" # Keep enable_in_reasoning as False (default) - config = mock_vllm_config.structured_outputs_config + config = manager_with_reasoner.vllm_config.structured_outputs_config assert config.enable_in_reasoning is False - manager = StructuredOutputManager(mock_vllm_config) - manager.reasoner = mock_reasoning_parser - - # Mock reasoning not ended - mock_reasoning_parser.is_reasoning_end.return_value = False - - result = manager.should_fill_bitmask(mock_request_with_structured_output) + result = manager_with_reasoner.should_fill_bitmask( + mock_request_with_structured_output + ) # Should set reasoning_ended and return its value assert ( @@ -118,68 +120,92 @@ class TestReasoningStructuredOutput: ): """Test should_fill_bitmask when no reasoner is configured.""" manager = StructuredOutputManager(mock_vllm_config) - manager.reasoner = None result = manager.should_fill_bitmask(mock_request_with_structured_output) # Should default to True when no reasoner assert result is True + def test_should_fill_bitmask_uses_request_reasoning_parser_kwargs( + self, mock_vllm_config, mock_request_with_structured_output + ): + """Test request-level parser kwargs override the default reasoner.""" + + class KwargReasoner: + def __init__(self, tokenizer, chat_template_kwargs=None): + self.chat_template_kwargs = chat_template_kwargs or {} + + def is_reasoning_end(self, input_ids): + return not self.chat_template_kwargs.get("enable_thinking", False) + + manager = StructuredOutputManager(mock_vllm_config) + manager.reasoner_cls = KwargReasoner + manager.tokenizer = Mock() + + structured_req = mock_request_with_structured_output.structured_output_request + structured_req.reasoning_parser_kwargs = { + "chat_template_kwargs": {"enable_thinking": True} + } + + result = manager.should_fill_bitmask(mock_request_with_structured_output) + + assert result is False + assert ( + mock_request_with_structured_output.structured_output_request.reasoner + is not None + ) + def test_should_advance_with_enable_in_reasoning( self, - mock_vllm_config, + manager_with_reasoner, mock_request_with_structured_output, - mock_reasoning_parser, ): """Test should_advance when enable_in_reasoning is True.""" # Enable enable_in_reasoning - mock_vllm_config.structured_outputs_config.enable_in_reasoning = True - - manager = StructuredOutputManager(mock_vllm_config) - manager.reasoner = mock_reasoning_parser + manager_with_reasoner.enable_in_reasoning = True # Should always return True when enable_in_reasoning is enabled - result = manager.should_advance(mock_request_with_structured_output) + result = manager_with_reasoner.should_advance( + mock_request_with_structured_output + ) assert result is True def test_should_advance_reasoning_not_ended( self, - mock_vllm_config, + manager_with_reasoner, mock_request_with_structured_output, - mock_reasoning_parser, ): """Test should_advance when reasoning has not ended.""" - manager = StructuredOutputManager(mock_vllm_config) - manager.reasoner = mock_reasoning_parser - # Set reasoning as not ended ( mock_request_with_structured_output.structured_output_request ).reasoning_ended = False - mock_reasoning_parser.is_reasoning_end.return_value = False - result = manager.should_advance(mock_request_with_structured_output) + result = manager_with_reasoner.should_advance( + mock_request_with_structured_output + ) # Should return False since reasoning hasn't ended assert result is False def test_should_advance_reasoning_just_ended( self, - mock_vllm_config, + manager_with_reasoner, mock_request_with_structured_output, - mock_reasoning_parser, ): """Test should_advance when reasoning ends in current step.""" - manager = StructuredOutputManager(mock_vllm_config) - manager.reasoner = mock_reasoning_parser - # Set reasoning as not ended initially, but ends in this step ( mock_request_with_structured_output.structured_output_request ).reasoning_ended = False - mock_reasoning_parser.is_reasoning_end.return_value = True + reasoner = MockReasoner(tokenizer=Mock()) + reasoner.is_reasoning_end_streaming.return_value = True + structured_req = mock_request_with_structured_output.structured_output_request + structured_req.reasoner = reasoner - result = manager.should_advance(mock_request_with_structured_output) + result = manager_with_reasoner.should_advance( + mock_request_with_structured_output + ) # Should set reasoning_ended to True but return False for this step assert ( @@ -190,20 +216,18 @@ class TestReasoningStructuredOutput: def test_should_advance_reasoning_already_ended( self, - mock_vllm_config, + manager_with_reasoner, mock_request_with_structured_output, - mock_reasoning_parser, ): """Test should_advance when reasoning has already ended.""" - manager = StructuredOutputManager(mock_vllm_config) - manager.reasoner = mock_reasoning_parser - # Set reasoning as already ended ( mock_request_with_structured_output.structured_output_request ).reasoning_ended = True - result = manager.should_advance(mock_request_with_structured_output) + result = manager_with_reasoner.should_advance( + mock_request_with_structured_output + ) # Should return True since reasoning has ended assert result is True diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 50013a060a..6058d8ed86 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -78,6 +78,7 @@ class EngineClient(ABC): priority: int = 0, data_parallel_rank: int | None = None, reasoning_ended: bool | None = None, + reasoning_parser_kwargs: dict[str, Any] | None = None, ) -> AsyncGenerator[RequestOutput, None]: """Generate outputs for a request.""" ... diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index b8ad54adb5..694ff80047 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -347,6 +347,11 @@ class OpenAIServingChat(OpenAIServing): priority=request.priority, data_parallel_rank=data_parallel_rank, reasoning_ended=reasoning_ended, + reasoning_parser_kwargs={ + "chat_template_kwargs": chat_template_kwargs, + } + if reasoning_parser + else None, ) generators.append(generator) diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index ed9204ccaf..a6a9355aea 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -456,9 +456,13 @@ class OpenAIServingResponses(OpenAIServing): context = SimpleContext() if self.parser and self.parser.reasoning_parser_cls is not None: + chat_template_kwargs = self._effective_chat_template_kwargs(request) + reasoning_parser_kwargs = { + "chat_template_kwargs": chat_template_kwargs, + } reasoning_parser = self.parser.reasoning_parser_cls( tokenizer, - chat_template_kwargs=self._effective_chat_template_kwargs(request), + chat_template_kwargs=chat_template_kwargs, ) if ( isinstance( @@ -481,6 +485,9 @@ class OpenAIServingResponses(OpenAIServing): lora_request=lora_request, priority=request.priority, trace_headers=trace_headers, + reasoning_parser_kwargs=reasoning_parser_kwargs + if self.parser and self.parser.reasoning_parser_cls is not None + else None, ) generators.append(generator) @@ -627,6 +634,7 @@ class OpenAIServingResponses(OpenAIServing): lora_request: LoRARequest | None = None, priority: int = 0, trace_headers: Mapping[str, str] | None = None, + reasoning_parser_kwargs: dict[str, Any] | None = None, ): max_model_len = self.model_config.max_model_len @@ -650,6 +658,7 @@ class OpenAIServingResponses(OpenAIServing): lora_request=lora_request, trace_headers=trace_headers, priority=priority, + reasoning_parser_kwargs=reasoning_parser_kwargs, ) async for res in generator: diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index c9946238e5..8172ead083 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -120,6 +120,7 @@ class EngineCoreRequest( external_req_id: str | None = None reasoning_ended: bool | None = None + reasoning_parser_kwargs: dict[str, Any] | None = None @property def params(self) -> SamplingParams | PoolingParams: diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 45ae416529..0e55a685bd 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -293,6 +293,7 @@ class AsyncLLM(EngineClient): data_parallel_rank: int | None = None, prompt_text: str | None = None, reasoning_ended: bool | None = None, + reasoning_parser_kwargs: dict[str, Any] | None = None, ) -> RequestOutputCollector: """Add new request to the AsyncLLM.""" @@ -313,7 +314,7 @@ class AsyncLLM(EngineClient): ) if isinstance(prompt, AsyncGenerator): - if reasoning_ended is not None: + if reasoning_ended is not None or reasoning_parser_kwargs is not None: raise NotImplementedError # Streaming input case. @@ -361,6 +362,8 @@ class AsyncLLM(EngineClient): if reasoning_ended is not None: request.reasoning_ended = reasoning_ended + if reasoning_parser_kwargs is not None: + request.reasoning_parser_kwargs = reasoning_parser_kwargs self.input_processor.assign_request_id(request) @@ -534,6 +537,7 @@ class AsyncLLM(EngineClient): priority: int = 0, data_parallel_rank: int | None = None, reasoning_ended: bool | None = None, + reasoning_parser_kwargs: dict[str, Any] | None = None, ) -> AsyncGenerator[RequestOutput, None]: """ Main function called by the API server to kick off a request @@ -563,6 +567,7 @@ class AsyncLLM(EngineClient): data_parallel_rank=data_parallel_rank, prompt_text=prompt_text, reasoning_ended=reasoning_ended, + reasoning_parser_kwargs=reasoning_parser_kwargs, ) # The output_handler task pushes items into the queue. diff --git a/vllm/v1/request.py b/vllm/v1/request.py index 0ae07c29d4..738a68c836 100644 --- a/vllm/v1/request.py +++ b/vllm/v1/request.py @@ -75,6 +75,7 @@ class Request: block_hasher: Callable[["Request"], list["BlockHash"]] | None = None, resumable: bool = False, reasoning_ended: bool | None = None, + reasoning_parser_kwargs: dict[str, Any] | None = None, ) -> None: self.request_id = request_id self.client_index = client_index @@ -87,6 +88,9 @@ class Request: ) if self.structured_output_request is not None: self.structured_output_request.reasoning_ended = reasoning_ended + self.structured_output_request.reasoning_parser_kwargs = ( + reasoning_parser_kwargs + ) self.arrival_time = arrival_time if arrival_time is not None else time.time() self.status = RequestStatus.WAITING @@ -201,6 +205,7 @@ class Request: block_hasher=block_hasher, resumable=request.resumable, reasoning_ended=request.reasoning_ended, + reasoning_parser_kwargs=request.reasoning_parser_kwargs, ) def append_output_token_ids( diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 213b49f28d..3080fe30b0 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -37,7 +37,10 @@ class StructuredOutputManager: def __init__(self, vllm_config: VllmConfig): self.backend: StructuredOutputBackend | None = None - self.reasoner: ReasoningParser | None = None + # We only store the class of the reasoner in the manager. + # The parser instance is request-scoped because some reasoning parsers + # depend on per-request chat-template kwargs. + self.reasoner_cls: type[ReasoningParser] | None = None self.vllm_config = vllm_config # When in external_launcher mode, async grammar compilation causes deadlocks @@ -85,15 +88,29 @@ class StructuredOutputManager: self.vllm_config.structured_outputs_config.reasoning_parser ) if reasoning_parser: - reasoner_cls = ReasoningParserManager.get_reasoning_parser( + self.reasoner_cls = ReasoningParserManager.get_reasoning_parser( reasoning_parser ) - self.reasoner = reasoner_cls(tokenizer=self.tokenizer) self.enable_in_reasoning = ( self.vllm_config.structured_outputs_config.enable_in_reasoning ) + def _get_reasoner(self, request: "Request") -> "ReasoningParser | None": + structured_req = request.structured_output_request + if structured_req is None or self.reasoner_cls is None: + return None + + if structured_req.reasoner is None: + # Lazily build the request-local parser so the structured-output + # gate observes the same template kwargs used by the frontend. + parser_kwargs = structured_req.reasoning_parser_kwargs or {} + structured_req.reasoner = self.reasoner_cls( + tokenizer=self.tokenizer, + **parser_kwargs, + ) + return structured_req.reasoner + def grammar_init(self, request: "Request") -> None: if request.structured_output_request is None: return @@ -285,7 +302,8 @@ class StructuredOutputManager: # NOTE (Hanchen) if enable_in_reasoning is True, it means that # the model needs to be constrained in reasoning. So we should always # enable the bitmask filling. - if self.reasoner is not None: + reasoner = self._get_reasoner(request) + if reasoner is not None: if self.enable_in_reasoning: return True assert request.structured_output_request is not None @@ -295,7 +313,7 @@ class StructuredOutputManager: # After unifying the `openai_gptoss` and non-`openai_gptoss` styles, # it can be removed. request.structured_output_request.reasoning_ended = ( - self.reasoner.is_reasoning_end(request.prompt_token_ids or []) + reasoner.is_reasoning_end(request.prompt_token_ids or []) ) return request.structured_output_request.reasoning_ended return True @@ -311,7 +329,8 @@ class StructuredOutputManager: assert request.structured_output_request.grammar is not None # by default, we should always advance # for cases that don't use thinking mode. - if self.reasoner is None: + reasoner = self._get_reasoner(request) + if reasoner is None: return True # if the model needs structured in reasoning, we should advance @@ -328,7 +347,7 @@ class StructuredOutputManager: start = ( delta_from if delta_from >= 0 else max(len(all_token_ids) + delta_from, 0) ) - if self.reasoner.is_reasoning_end_streaming( + if reasoner.is_reasoning_end_streaming( all_token_ids, itertools.islice(all_token_ids, start, None) ): # Reasoning just ended, so we shouldn't advance til diff --git a/vllm/v1/structured_output/request.py b/vllm/v1/structured_output/request.py index b921a71b3c..dfa8c7efca 100644 --- a/vllm/v1/structured_output/request.py +++ b/vllm/v1/structured_output/request.py @@ -5,7 +5,7 @@ import functools import json from concurrent.futures import Future from concurrent.futures._base import TimeoutError -from typing import cast +from typing import TYPE_CHECKING, Any, cast from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.v1.structured_output.backend_types import ( @@ -14,12 +14,19 @@ from vllm.v1.structured_output.backend_types import ( StructuredOutputOptions, ) +if TYPE_CHECKING: + from vllm.reasoning import ReasoningParser + @dataclasses.dataclass class StructuredOutputRequest: params: StructuredOutputsParams _grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None reasoning_ended: bool | None = None + reasoning_parser_kwargs: dict[str, Any] | None = None + # Cached per request; do not share reasoning parsers across requests because + # their behavior can depend on reasoning_parser_kwargs. + reasoner: "ReasoningParser | None" = None @staticmethod def from_sampling_params(