[Bugfix] Pass reasoning parser kwargs to structured output (#41199)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
Bugen Zhao
2026-04-30 23:38:02 -07:00
committed by GitHub
co-authored by wang.yuqi
parent c3868bbbe4
commit a07642667d
9 changed files with 132 additions and 56 deletions
+1
View File
@@ -78,6 +78,7 @@ class EngineClient(ABC):
priority: int = 0,
data_parallel_rank: int | None = None,
reasoning_ended: bool | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> AsyncGenerator[RequestOutput, None]:
"""Generate outputs for a request."""
...
@@ -347,6 +347,11 @@ class OpenAIServingChat(OpenAIServing):
priority=request.priority,
data_parallel_rank=data_parallel_rank,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs={
"chat_template_kwargs": chat_template_kwargs,
}
if reasoning_parser
else None,
)
generators.append(generator)
+10 -1
View File
@@ -456,9 +456,13 @@ class OpenAIServingResponses(OpenAIServing):
context = SimpleContext()
if self.parser and self.parser.reasoning_parser_cls is not None:
chat_template_kwargs = self._effective_chat_template_kwargs(request)
reasoning_parser_kwargs = {
"chat_template_kwargs": chat_template_kwargs,
}
reasoning_parser = self.parser.reasoning_parser_cls(
tokenizer,
chat_template_kwargs=self._effective_chat_template_kwargs(request),
chat_template_kwargs=chat_template_kwargs,
)
if (
isinstance(
@@ -481,6 +485,9 @@ class OpenAIServingResponses(OpenAIServing):
lora_request=lora_request,
priority=request.priority,
trace_headers=trace_headers,
reasoning_parser_kwargs=reasoning_parser_kwargs
if self.parser and self.parser.reasoning_parser_cls is not None
else None,
)
generators.append(generator)
@@ -627,6 +634,7 @@ class OpenAIServingResponses(OpenAIServing):
lora_request: LoRARequest | None = None,
priority: int = 0,
trace_headers: Mapping[str, str] | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
):
max_model_len = self.model_config.max_model_len
@@ -650,6 +658,7 @@ class OpenAIServingResponses(OpenAIServing):
lora_request=lora_request,
trace_headers=trace_headers,
priority=priority,
reasoning_parser_kwargs=reasoning_parser_kwargs,
)
async for res in generator:
+1
View File
@@ -120,6 +120,7 @@ class EngineCoreRequest(
external_req_id: str | None = None
reasoning_ended: bool | None = None
reasoning_parser_kwargs: dict[str, Any] | None = None
@property
def params(self) -> SamplingParams | PoolingParams:
+6 -1
View File
@@ -293,6 +293,7 @@ class AsyncLLM(EngineClient):
data_parallel_rank: int | None = None,
prompt_text: str | None = None,
reasoning_ended: bool | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> RequestOutputCollector:
"""Add new request to the AsyncLLM."""
@@ -313,7 +314,7 @@ class AsyncLLM(EngineClient):
)
if isinstance(prompt, AsyncGenerator):
if reasoning_ended is not None:
if reasoning_ended is not None or reasoning_parser_kwargs is not None:
raise NotImplementedError
# Streaming input case.
@@ -361,6 +362,8 @@ class AsyncLLM(EngineClient):
if reasoning_ended is not None:
request.reasoning_ended = reasoning_ended
if reasoning_parser_kwargs is not None:
request.reasoning_parser_kwargs = reasoning_parser_kwargs
self.input_processor.assign_request_id(request)
@@ -534,6 +537,7 @@ class AsyncLLM(EngineClient):
priority: int = 0,
data_parallel_rank: int | None = None,
reasoning_ended: bool | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> AsyncGenerator[RequestOutput, None]:
"""
Main function called by the API server to kick off a request
@@ -563,6 +567,7 @@ class AsyncLLM(EngineClient):
data_parallel_rank=data_parallel_rank,
prompt_text=prompt_text,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs=reasoning_parser_kwargs,
)
# The output_handler task pushes items into the queue.
+5
View File
@@ -75,6 +75,7 @@ class Request:
block_hasher: Callable[["Request"], list["BlockHash"]] | None = None,
resumable: bool = False,
reasoning_ended: bool | None = None,
reasoning_parser_kwargs: dict[str, Any] | None = None,
) -> None:
self.request_id = request_id
self.client_index = client_index
@@ -87,6 +88,9 @@ class Request:
)
if self.structured_output_request is not None:
self.structured_output_request.reasoning_ended = reasoning_ended
self.structured_output_request.reasoning_parser_kwargs = (
reasoning_parser_kwargs
)
self.arrival_time = arrival_time if arrival_time is not None else time.time()
self.status = RequestStatus.WAITING
@@ -201,6 +205,7 @@ class Request:
block_hasher=block_hasher,
resumable=request.resumable,
reasoning_ended=request.reasoning_ended,
reasoning_parser_kwargs=request.reasoning_parser_kwargs,
)
def append_output_token_ids(
+26 -7
View File
@@ -37,7 +37,10 @@ class StructuredOutputManager:
def __init__(self, vllm_config: VllmConfig):
self.backend: StructuredOutputBackend | None = None
self.reasoner: ReasoningParser | None = None
# We only store the class of the reasoner in the manager.
# The parser instance is request-scoped because some reasoning parsers
# depend on per-request chat-template kwargs.
self.reasoner_cls: type[ReasoningParser] | None = None
self.vllm_config = vllm_config
# When in external_launcher mode, async grammar compilation causes deadlocks
@@ -85,15 +88,29 @@ class StructuredOutputManager:
self.vllm_config.structured_outputs_config.reasoning_parser
)
if reasoning_parser:
reasoner_cls = ReasoningParserManager.get_reasoning_parser(
self.reasoner_cls = ReasoningParserManager.get_reasoning_parser(
reasoning_parser
)
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
self.enable_in_reasoning = (
self.vllm_config.structured_outputs_config.enable_in_reasoning
)
def _get_reasoner(self, request: "Request") -> "ReasoningParser | None":
structured_req = request.structured_output_request
if structured_req is None or self.reasoner_cls is None:
return None
if structured_req.reasoner is None:
# Lazily build the request-local parser so the structured-output
# gate observes the same template kwargs used by the frontend.
parser_kwargs = structured_req.reasoning_parser_kwargs or {}
structured_req.reasoner = self.reasoner_cls(
tokenizer=self.tokenizer,
**parser_kwargs,
)
return structured_req.reasoner
def grammar_init(self, request: "Request") -> None:
if request.structured_output_request is None:
return
@@ -285,7 +302,8 @@ class StructuredOutputManager:
# NOTE (Hanchen) if enable_in_reasoning is True, it means that
# the model needs to be constrained in reasoning. So we should always
# enable the bitmask filling.
if self.reasoner is not None:
reasoner = self._get_reasoner(request)
if reasoner is not None:
if self.enable_in_reasoning:
return True
assert request.structured_output_request is not None
@@ -295,7 +313,7 @@ class StructuredOutputManager:
# After unifying the `openai_gptoss` and non-`openai_gptoss` styles,
# it can be removed.
request.structured_output_request.reasoning_ended = (
self.reasoner.is_reasoning_end(request.prompt_token_ids or [])
reasoner.is_reasoning_end(request.prompt_token_ids or [])
)
return request.structured_output_request.reasoning_ended
return True
@@ -311,7 +329,8 @@ class StructuredOutputManager:
assert request.structured_output_request.grammar is not None
# by default, we should always advance
# for cases that don't use thinking mode.
if self.reasoner is None:
reasoner = self._get_reasoner(request)
if reasoner is None:
return True
# if the model needs structured in reasoning, we should advance
@@ -328,7 +347,7 @@ class StructuredOutputManager:
start = (
delta_from if delta_from >= 0 else max(len(all_token_ids) + delta_from, 0)
)
if self.reasoner.is_reasoning_end_streaming(
if reasoner.is_reasoning_end_streaming(
all_token_ids, itertools.islice(all_token_ids, start, None)
):
# Reasoning just ended, so we shouldn't advance til
+8 -1
View File
@@ -5,7 +5,7 @@ import functools
import json
from concurrent.futures import Future
from concurrent.futures._base import TimeoutError
from typing import cast
from typing import TYPE_CHECKING, Any, cast
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.structured_output.backend_types import (
@@ -14,12 +14,19 @@ from vllm.v1.structured_output.backend_types import (
StructuredOutputOptions,
)
if TYPE_CHECKING:
from vllm.reasoning import ReasoningParser
@dataclasses.dataclass
class StructuredOutputRequest:
params: StructuredOutputsParams
_grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None
reasoning_ended: bool | None = None
reasoning_parser_kwargs: dict[str, Any] | None = None
# Cached per request; do not share reasoning parsers across requests because
# their behavior can depend on reasoning_parser_kwargs.
reasoner: "ReasoningParser | None" = None
@staticmethod
def from_sampling_params(