[Bugfix] Pass reasoning parser kwargs to structured output (#41199)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
Bugen Zhao
2026-04-30 23:38:02 -07:00
committed by GitHub
co-authored by wang.yuqi
parent c3868bbbe4
commit a07642667d
9 changed files with 132 additions and 56 deletions
@@ -8,11 +8,16 @@ from unittest.mock import Mock
import pytest
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
from vllm.reasoning import ReasoningParser
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
class MockReasoner:
def __init__(self, tokenizer):
self.is_reasoning_end = Mock(return_value=False)
self.is_reasoning_end_streaming = Mock(return_value=False)
class TestReasoningStructuredOutput:
"""Test reasoning-aware structured output functionality."""
@@ -50,13 +55,6 @@ class TestReasoningStructuredOutput:
config.speculative_config = None
return config
@pytest.fixture
def mock_reasoning_parser(self):
"""Create a mock ReasoningParser."""
parser = Mock(spec=ReasoningParser)
parser.is_reasoning_end = Mock(return_value=False)
return parser
@pytest.fixture
def mock_request_with_structured_output(self):
"""Create a mock request with structured output."""
@@ -64,6 +62,8 @@ class TestReasoningStructuredOutput:
request.structured_output_request = Mock()
request.structured_output_request.reasoning_ended = None
request.structured_output_request.grammar = Mock()
request.structured_output_request.reasoning_parser_kwargs = None
request.structured_output_request.reasoner = None
request.structured_output_request.grammar.is_terminated = Mock(
return_value=False
)
@@ -74,6 +74,13 @@ class TestReasoningStructuredOutput:
request.num_output_placeholders = 0
return request
@pytest.fixture
def manager_with_reasoner(self, mock_vllm_config):
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner_cls = MockReasoner
manager.tokenizer = Mock()
return manager
def test_should_fill_bitmask_with_enable_in_reasoning(
self, mock_vllm_config, mock_request_with_structured_output
):
@@ -89,22 +96,17 @@ class TestReasoningStructuredOutput:
def test_should_fill_bitmask_without_enable_in_reasoning(
self,
mock_vllm_config,
manager_with_reasoner,
mock_request_with_structured_output,
mock_reasoning_parser,
):
"""Test should_fill_bitmask when enable_in_reasoning is False."""
# Keep enable_in_reasoning as False (default)
config = mock_vllm_config.structured_outputs_config
config = manager_with_reasoner.vllm_config.structured_outputs_config
assert config.enable_in_reasoning is False
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner = mock_reasoning_parser
# Mock reasoning not ended
mock_reasoning_parser.is_reasoning_end.return_value = False
result = manager.should_fill_bitmask(mock_request_with_structured_output)
result = manager_with_reasoner.should_fill_bitmask(
mock_request_with_structured_output
)
# Should set reasoning_ended and return its value
assert (
@@ -118,68 +120,92 @@ class TestReasoningStructuredOutput:
):
"""Test should_fill_bitmask when no reasoner is configured."""
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner = None
result = manager.should_fill_bitmask(mock_request_with_structured_output)
# Should default to True when no reasoner
assert result is True
def test_should_fill_bitmask_uses_request_reasoning_parser_kwargs(
self, mock_vllm_config, mock_request_with_structured_output
):
"""Test request-level parser kwargs override the default reasoner."""
class KwargReasoner:
def __init__(self, tokenizer, chat_template_kwargs=None):
self.chat_template_kwargs = chat_template_kwargs or {}
def is_reasoning_end(self, input_ids):
return not self.chat_template_kwargs.get("enable_thinking", False)
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner_cls = KwargReasoner
manager.tokenizer = Mock()
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_parser_kwargs = {
"chat_template_kwargs": {"enable_thinking": True}
}
result = manager.should_fill_bitmask(mock_request_with_structured_output)
assert result is False
assert (
mock_request_with_structured_output.structured_output_request.reasoner
is not None
)
def test_should_advance_with_enable_in_reasoning(
self,
mock_vllm_config,
manager_with_reasoner,
mock_request_with_structured_output,
mock_reasoning_parser,
):
"""Test should_advance when enable_in_reasoning is True."""
# Enable enable_in_reasoning
mock_vllm_config.structured_outputs_config.enable_in_reasoning = True
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner = mock_reasoning_parser
manager_with_reasoner.enable_in_reasoning = True
# Should always return True when enable_in_reasoning is enabled
result = manager.should_advance(mock_request_with_structured_output)
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
assert result is True
def test_should_advance_reasoning_not_ended(
self,
mock_vllm_config,
manager_with_reasoner,
mock_request_with_structured_output,
mock_reasoning_parser,
):
"""Test should_advance when reasoning has not ended."""
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner = mock_reasoning_parser
# Set reasoning as not ended
(
mock_request_with_structured_output.structured_output_request
).reasoning_ended = False
mock_reasoning_parser.is_reasoning_end.return_value = False
result = manager.should_advance(mock_request_with_structured_output)
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# Should return False since reasoning hasn't ended
assert result is False
def test_should_advance_reasoning_just_ended(
self,
mock_vllm_config,
manager_with_reasoner,
mock_request_with_structured_output,
mock_reasoning_parser,
):
"""Test should_advance when reasoning ends in current step."""
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner = mock_reasoning_parser
# Set reasoning as not ended initially, but ends in this step
(
mock_request_with_structured_output.structured_output_request
).reasoning_ended = False
mock_reasoning_parser.is_reasoning_end.return_value = True
reasoner = MockReasoner(tokenizer=Mock())
reasoner.is_reasoning_end_streaming.return_value = True
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoner = reasoner
result = manager.should_advance(mock_request_with_structured_output)
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# Should set reasoning_ended to True but return False for this step
assert (
@@ -190,20 +216,18 @@ class TestReasoningStructuredOutput:
def test_should_advance_reasoning_already_ended(
self,
mock_vllm_config,
manager_with_reasoner,
mock_request_with_structured_output,
mock_reasoning_parser,
):
"""Test should_advance when reasoning has already ended."""
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner = mock_reasoning_parser
# Set reasoning as already ended
(
mock_request_with_structured_output.structured_output_request
).reasoning_ended = True
result = manager.should_advance(mock_request_with_structured_output)
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# Should return True since reasoning has ended
assert result is True
+1
View File
@@ -78,6 +78,7 @@ class EngineClient(ABC):
priority: int = 0,
data_parallel_rank: int | None = None,
reasoning_ended: bool | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> AsyncGenerator[RequestOutput, None]:
"""Generate outputs for a request."""
...
@@ -347,6 +347,11 @@ class OpenAIServingChat(OpenAIServing):
priority=request.priority,
data_parallel_rank=data_parallel_rank,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs={
"chat_template_kwargs": chat_template_kwargs,
}
if reasoning_parser
else None,
)
generators.append(generator)
+10 -1
View File
@@ -456,9 +456,13 @@ class OpenAIServingResponses(OpenAIServing):
context = SimpleContext()
if self.parser and self.parser.reasoning_parser_cls is not None:
chat_template_kwargs = self._effective_chat_template_kwargs(request)
reasoning_parser_kwargs = {
"chat_template_kwargs": chat_template_kwargs,
}
reasoning_parser = self.parser.reasoning_parser_cls(
tokenizer,
chat_template_kwargs=self._effective_chat_template_kwargs(request),
chat_template_kwargs=chat_template_kwargs,
)
if (
isinstance(
@@ -481,6 +485,9 @@ class OpenAIServingResponses(OpenAIServing):
lora_request=lora_request,
priority=request.priority,
trace_headers=trace_headers,
reasoning_parser_kwargs=reasoning_parser_kwargs
if self.parser and self.parser.reasoning_parser_cls is not None
else None,
)
generators.append(generator)
@@ -627,6 +634,7 @@ class OpenAIServingResponses(OpenAIServing):
lora_request: LoRARequest | None = None,
priority: int = 0,
trace_headers: Mapping[str, str] | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
):
max_model_len = self.model_config.max_model_len
@@ -650,6 +658,7 @@ class OpenAIServingResponses(OpenAIServing):
lora_request=lora_request,
trace_headers=trace_headers,
priority=priority,
reasoning_parser_kwargs=reasoning_parser_kwargs,
)
async for res in generator:
+1
View File
@@ -120,6 +120,7 @@ class EngineCoreRequest(
external_req_id: str | None = None
reasoning_ended: bool | None = None
reasoning_parser_kwargs: dict[str, Any] | None = None
@property
def params(self) -> SamplingParams | PoolingParams:
+6 -1
View File
@@ -293,6 +293,7 @@ class AsyncLLM(EngineClient):
data_parallel_rank: int | None = None,
prompt_text: str | None = None,
reasoning_ended: bool | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> RequestOutputCollector:
"""Add new request to the AsyncLLM."""
@@ -313,7 +314,7 @@ class AsyncLLM(EngineClient):
)
if isinstance(prompt, AsyncGenerator):
if reasoning_ended is not None:
if reasoning_ended is not None or reasoning_parser_kwargs is not None:
raise NotImplementedError
# Streaming input case.
@@ -361,6 +362,8 @@ class AsyncLLM(EngineClient):
if reasoning_ended is not None:
request.reasoning_ended = reasoning_ended
if reasoning_parser_kwargs is not None:
request.reasoning_parser_kwargs = reasoning_parser_kwargs
self.input_processor.assign_request_id(request)
@@ -534,6 +537,7 @@ class AsyncLLM(EngineClient):
priority: int = 0,
data_parallel_rank: int | None = None,
reasoning_ended: bool | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> AsyncGenerator[RequestOutput, None]:
"""
Main function called by the API server to kick off a request
@@ -563,6 +567,7 @@ class AsyncLLM(EngineClient):
data_parallel_rank=data_parallel_rank,
prompt_text=prompt_text,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs=reasoning_parser_kwargs,
)
# The output_handler task pushes items into the queue.
+5
View File
@@ -75,6 +75,7 @@ class Request:
block_hasher: Callable[["Request"], list["BlockHash"]] | None = None,
resumable: bool = False,
reasoning_ended: bool | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> None:
self.request_id = request_id
self.client_index = client_index
@@ -87,6 +88,9 @@ class Request:
)
if self.structured_output_request is not None:
self.structured_output_request.reasoning_ended = reasoning_ended
self.structured_output_request.reasoning_parser_kwargs = (
reasoning_parser_kwargs
)
self.arrival_time = arrival_time if arrival_time is not None else time.time()
self.status = RequestStatus.WAITING
@@ -201,6 +205,7 @@ class Request:
block_hasher=block_hasher,
resumable=request.resumable,
reasoning_ended=request.reasoning_ended,
reasoning_parser_kwargs=request.reasoning_parser_kwargs,
)
def append_output_token_ids(
+26 -7
View File
@@ -37,7 +37,10 @@ class StructuredOutputManager:
def __init__(self, vllm_config: VllmConfig):
self.backend: StructuredOutputBackend | None = None
self.reasoner: ReasoningParser | None = None
# We only store the class of the reasoner in the manager.
# The parser instance is request-scoped because some reasoning parsers
# depend on per-request chat-template kwargs.
self.reasoner_cls: type[ReasoningParser] | None = None
self.vllm_config = vllm_config
# When in external_launcher mode, async grammar compilation causes deadlocks
@@ -85,15 +88,29 @@ class StructuredOutputManager:
self.vllm_config.structured_outputs_config.reasoning_parser
)
if reasoning_parser:
reasoner_cls = ReasoningParserManager.get_reasoning_parser(
self.reasoner_cls = ReasoningParserManager.get_reasoning_parser(
reasoning_parser
)
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
self.enable_in_reasoning = (
self.vllm_config.structured_outputs_config.enable_in_reasoning
)
def _get_reasoner(self, request: "Request") -> "ReasoningParser | None":
structured_req = request.structured_output_request
if structured_req is None or self.reasoner_cls is None:
return None
if structured_req.reasoner is None:
# Lazily build the request-local parser so the structured-output
# gate observes the same template kwargs used by the frontend.
parser_kwargs = structured_req.reasoning_parser_kwargs or {}
structured_req.reasoner = self.reasoner_cls(
tokenizer=self.tokenizer,
**parser_kwargs,
)
return structured_req.reasoner
def grammar_init(self, request: "Request") -> None:
if request.structured_output_request is None:
return
@@ -285,7 +302,8 @@ class StructuredOutputManager:
# NOTE (Hanchen) if enable_in_reasoning is True, it means that
# the model needs to be constrained in reasoning. So we should always
# enable the bitmask filling.
if self.reasoner is not None:
reasoner = self._get_reasoner(request)
if reasoner is not None:
if self.enable_in_reasoning:
return True
assert request.structured_output_request is not None
@@ -295,7 +313,7 @@ class StructuredOutputManager:
# After unifying the `openai_gptoss` and non-`openai_gptoss` styles,
# it can be removed.
request.structured_output_request.reasoning_ended = (
self.reasoner.is_reasoning_end(request.prompt_token_ids or [])
reasoner.is_reasoning_end(request.prompt_token_ids or [])
)
return request.structured_output_request.reasoning_ended
return True
@@ -311,7 +329,8 @@ class StructuredOutputManager:
assert request.structured_output_request.grammar is not None
# by default, we should always advance
# for cases that don't use thinking mode.
if self.reasoner is None:
reasoner = self._get_reasoner(request)
if reasoner is None:
return True
# if the model needs structured in reasoning, we should advance
@@ -328,7 +347,7 @@ class StructuredOutputManager:
start = (
delta_from if delta_from >= 0 else max(len(all_token_ids) + delta_from, 0)
)
if self.reasoner.is_reasoning_end_streaming(
if reasoner.is_reasoning_end_streaming(
all_token_ids, itertools.islice(all_token_ids, start, None)
):
# Reasoning just ended, so we shouldn't advance til
+8 -1
View File
@@ -5,7 +5,7 @@ import functools
import json
from concurrent.futures import Future
from concurrent.futures._base import TimeoutError
from typing import cast
from typing import TYPE_CHECKING, Any, cast
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.structured_output.backend_types import (
@@ -14,12 +14,19 @@ from vllm.v1.structured_output.backend_types import (
StructuredOutputOptions,
)
if TYPE_CHECKING:
from vllm.reasoning import ReasoningParser
@dataclasses.dataclass
class StructuredOutputRequest:
params: StructuredOutputsParams
_grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None
reasoning_ended: bool | None = None
reasoning_parser_kwargs: dict[str, Any] | None = None
# Cached per request; do not share reasoning parsers across requests because
# their behavior can depend on reasoning_parser_kwargs.
reasoner: "ReasoningParser | None" = None
@staticmethod
def from_sampling_params(