[Bugfix] Pass reasoning parser kwargs to structured output (#41199)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -78,6 +78,7 @@ class EngineClient(ABC):
|
||||
priority: int = 0,
|
||||
data_parallel_rank: int | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
"""Generate outputs for a request."""
|
||||
...
|
||||
|
||||
@@ -347,6 +347,11 @@ class OpenAIServingChat(OpenAIServing):
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs={
|
||||
"chat_template_kwargs": chat_template_kwargs,
|
||||
}
|
||||
if reasoning_parser
|
||||
else None,
|
||||
)
|
||||
|
||||
generators.append(generator)
|
||||
|
||||
@@ -456,9 +456,13 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
context = SimpleContext()
|
||||
|
||||
if self.parser and self.parser.reasoning_parser_cls is not None:
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
||||
reasoning_parser_kwargs = {
|
||||
"chat_template_kwargs": chat_template_kwargs,
|
||||
}
|
||||
reasoning_parser = self.parser.reasoning_parser_cls(
|
||||
tokenizer,
|
||||
chat_template_kwargs=self._effective_chat_template_kwargs(request),
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
if (
|
||||
isinstance(
|
||||
@@ -481,6 +485,9 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
trace_headers=trace_headers,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs
|
||||
if self.parser and self.parser.reasoning_parser_cls is not None
|
||||
else None,
|
||||
)
|
||||
generators.append(generator)
|
||||
|
||||
@@ -627,6 +634,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
max_model_len = self.model_config.max_model_len
|
||||
|
||||
@@ -650,6 +658,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
)
|
||||
|
||||
async for res in generator:
|
||||
|
||||
@@ -120,6 +120,7 @@ class EngineCoreRequest(
|
||||
external_req_id: str | None = None
|
||||
|
||||
reasoning_ended: bool | None = None
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None
|
||||
|
||||
@property
|
||||
def params(self) -> SamplingParams | PoolingParams:
|
||||
|
||||
@@ -293,6 +293,7 @@ class AsyncLLM(EngineClient):
|
||||
data_parallel_rank: int | None = None,
|
||||
prompt_text: str | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> RequestOutputCollector:
|
||||
"""Add new request to the AsyncLLM."""
|
||||
|
||||
@@ -313,7 +314,7 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
|
||||
if isinstance(prompt, AsyncGenerator):
|
||||
if reasoning_ended is not None:
|
||||
if reasoning_ended is not None or reasoning_parser_kwargs is not None:
|
||||
raise NotImplementedError
|
||||
|
||||
# Streaming input case.
|
||||
@@ -361,6 +362,8 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
if reasoning_ended is not None:
|
||||
request.reasoning_ended = reasoning_ended
|
||||
if reasoning_parser_kwargs is not None:
|
||||
request.reasoning_parser_kwargs = reasoning_parser_kwargs
|
||||
|
||||
self.input_processor.assign_request_id(request)
|
||||
|
||||
@@ -534,6 +537,7 @@ class AsyncLLM(EngineClient):
|
||||
priority: int = 0,
|
||||
data_parallel_rank: int | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
"""
|
||||
Main function called by the API server to kick off a request
|
||||
@@ -563,6 +567,7 @@ class AsyncLLM(EngineClient):
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
prompt_text=prompt_text,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
)
|
||||
|
||||
# The output_handler task pushes items into the queue.
|
||||
|
||||
@@ -75,6 +75,7 @@ class Request:
|
||||
block_hasher: Callable[["Request"], list["BlockHash"]] | None = None,
|
||||
resumable: bool = False,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
self.request_id = request_id
|
||||
self.client_index = client_index
|
||||
@@ -87,6 +88,9 @@ class Request:
|
||||
)
|
||||
if self.structured_output_request is not None:
|
||||
self.structured_output_request.reasoning_ended = reasoning_ended
|
||||
self.structured_output_request.reasoning_parser_kwargs = (
|
||||
reasoning_parser_kwargs
|
||||
)
|
||||
self.arrival_time = arrival_time if arrival_time is not None else time.time()
|
||||
|
||||
self.status = RequestStatus.WAITING
|
||||
@@ -201,6 +205,7 @@ class Request:
|
||||
block_hasher=block_hasher,
|
||||
resumable=request.resumable,
|
||||
reasoning_ended=request.reasoning_ended,
|
||||
reasoning_parser_kwargs=request.reasoning_parser_kwargs,
|
||||
)
|
||||
|
||||
def append_output_token_ids(
|
||||
|
||||
@@ -37,7 +37,10 @@ class StructuredOutputManager:
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig):
|
||||
self.backend: StructuredOutputBackend | None = None
|
||||
self.reasoner: ReasoningParser | None = None
|
||||
# We only store the class of the reasoner in the manager.
|
||||
# The parser instance is request-scoped because some reasoning parsers
|
||||
# depend on per-request chat-template kwargs.
|
||||
self.reasoner_cls: type[ReasoningParser] | None = None
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
# When in external_launcher mode, async grammar compilation causes deadlocks
|
||||
@@ -85,15 +88,29 @@ class StructuredOutputManager:
|
||||
self.vllm_config.structured_outputs_config.reasoning_parser
|
||||
)
|
||||
if reasoning_parser:
|
||||
reasoner_cls = ReasoningParserManager.get_reasoning_parser(
|
||||
self.reasoner_cls = ReasoningParserManager.get_reasoning_parser(
|
||||
reasoning_parser
|
||||
)
|
||||
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
|
||||
|
||||
self.enable_in_reasoning = (
|
||||
self.vllm_config.structured_outputs_config.enable_in_reasoning
|
||||
)
|
||||
|
||||
def _get_reasoner(self, request: "Request") -> "ReasoningParser | None":
|
||||
structured_req = request.structured_output_request
|
||||
if structured_req is None or self.reasoner_cls is None:
|
||||
return None
|
||||
|
||||
if structured_req.reasoner is None:
|
||||
# Lazily build the request-local parser so the structured-output
|
||||
# gate observes the same template kwargs used by the frontend.
|
||||
parser_kwargs = structured_req.reasoning_parser_kwargs or {}
|
||||
structured_req.reasoner = self.reasoner_cls(
|
||||
tokenizer=self.tokenizer,
|
||||
**parser_kwargs,
|
||||
)
|
||||
return structured_req.reasoner
|
||||
|
||||
def grammar_init(self, request: "Request") -> None:
|
||||
if request.structured_output_request is None:
|
||||
return
|
||||
@@ -285,7 +302,8 @@ class StructuredOutputManager:
|
||||
# NOTE (Hanchen) if enable_in_reasoning is True, it means that
|
||||
# the model needs to be constrained in reasoning. So we should always
|
||||
# enable the bitmask filling.
|
||||
if self.reasoner is not None:
|
||||
reasoner = self._get_reasoner(request)
|
||||
if reasoner is not None:
|
||||
if self.enable_in_reasoning:
|
||||
return True
|
||||
assert request.structured_output_request is not None
|
||||
@@ -295,7 +313,7 @@ class StructuredOutputManager:
|
||||
# After unifying the `openai_gptoss` and non-`openai_gptoss` styles,
|
||||
# it can be removed.
|
||||
request.structured_output_request.reasoning_ended = (
|
||||
self.reasoner.is_reasoning_end(request.prompt_token_ids or [])
|
||||
reasoner.is_reasoning_end(request.prompt_token_ids or [])
|
||||
)
|
||||
return request.structured_output_request.reasoning_ended
|
||||
return True
|
||||
@@ -311,7 +329,8 @@ class StructuredOutputManager:
|
||||
assert request.structured_output_request.grammar is not None
|
||||
# by default, we should always advance
|
||||
# for cases that don't use thinking mode.
|
||||
if self.reasoner is None:
|
||||
reasoner = self._get_reasoner(request)
|
||||
if reasoner is None:
|
||||
return True
|
||||
|
||||
# if the model needs structured in reasoning, we should advance
|
||||
@@ -328,7 +347,7 @@ class StructuredOutputManager:
|
||||
start = (
|
||||
delta_from if delta_from >= 0 else max(len(all_token_ids) + delta_from, 0)
|
||||
)
|
||||
if self.reasoner.is_reasoning_end_streaming(
|
||||
if reasoner.is_reasoning_end_streaming(
|
||||
all_token_ids, itertools.islice(all_token_ids, start, None)
|
||||
):
|
||||
# Reasoning just ended, so we shouldn't advance til
|
||||
|
||||
@@ -5,7 +5,7 @@ import functools
|
||||
import json
|
||||
from concurrent.futures import Future
|
||||
from concurrent.futures._base import TimeoutError
|
||||
from typing import cast
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
@@ -14,12 +14,19 @@ from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputOptions,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class StructuredOutputRequest:
|
||||
params: StructuredOutputsParams
|
||||
_grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None
|
||||
reasoning_ended: bool | None = None
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None
|
||||
# Cached per request; do not share reasoning parsers across requests because
|
||||
# their behavior can depend on reasoning_parser_kwargs.
|
||||
reasoner: "ReasoningParser | None" = None
|
||||
|
||||
@staticmethod
|
||||
def from_sampling_params(
|
||||
|
||||
Reference in New Issue
Block a user