[Bugfix] Pass reasoning parser kwargs to structured output (#41199)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -8,11 +8,16 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.reasoning import ReasoningParser
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
|
||||
class MockReasoner:
|
||||
def __init__(self, tokenizer):
|
||||
self.is_reasoning_end = Mock(return_value=False)
|
||||
self.is_reasoning_end_streaming = Mock(return_value=False)
|
||||
|
||||
|
||||
class TestReasoningStructuredOutput:
|
||||
"""Test reasoning-aware structured output functionality."""
|
||||
|
||||
@@ -50,13 +55,6 @@ class TestReasoningStructuredOutput:
|
||||
config.speculative_config = None
|
||||
return config
|
||||
|
||||
@pytest.fixture
|
||||
def mock_reasoning_parser(self):
|
||||
"""Create a mock ReasoningParser."""
|
||||
parser = Mock(spec=ReasoningParser)
|
||||
parser.is_reasoning_end = Mock(return_value=False)
|
||||
return parser
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request_with_structured_output(self):
|
||||
"""Create a mock request with structured output."""
|
||||
@@ -64,6 +62,8 @@ class TestReasoningStructuredOutput:
|
||||
request.structured_output_request = Mock()
|
||||
request.structured_output_request.reasoning_ended = None
|
||||
request.structured_output_request.grammar = Mock()
|
||||
request.structured_output_request.reasoning_parser_kwargs = None
|
||||
request.structured_output_request.reasoner = None
|
||||
request.structured_output_request.grammar.is_terminated = Mock(
|
||||
return_value=False
|
||||
)
|
||||
@@ -74,6 +74,13 @@ class TestReasoningStructuredOutput:
|
||||
request.num_output_placeholders = 0
|
||||
return request
|
||||
|
||||
@pytest.fixture
|
||||
def manager_with_reasoner(self, mock_vllm_config):
|
||||
manager = StructuredOutputManager(mock_vllm_config)
|
||||
manager.reasoner_cls = MockReasoner
|
||||
manager.tokenizer = Mock()
|
||||
return manager
|
||||
|
||||
def test_should_fill_bitmask_with_enable_in_reasoning(
|
||||
self, mock_vllm_config, mock_request_with_structured_output
|
||||
):
|
||||
@@ -89,22 +96,17 @@ class TestReasoningStructuredOutput:
|
||||
|
||||
def test_should_fill_bitmask_without_enable_in_reasoning(
|
||||
self,
|
||||
mock_vllm_config,
|
||||
manager_with_reasoner,
|
||||
mock_request_with_structured_output,
|
||||
mock_reasoning_parser,
|
||||
):
|
||||
"""Test should_fill_bitmask when enable_in_reasoning is False."""
|
||||
# Keep enable_in_reasoning as False (default)
|
||||
config = mock_vllm_config.structured_outputs_config
|
||||
config = manager_with_reasoner.vllm_config.structured_outputs_config
|
||||
assert config.enable_in_reasoning is False
|
||||
|
||||
manager = StructuredOutputManager(mock_vllm_config)
|
||||
manager.reasoner = mock_reasoning_parser
|
||||
|
||||
# Mock reasoning not ended
|
||||
mock_reasoning_parser.is_reasoning_end.return_value = False
|
||||
|
||||
result = manager.should_fill_bitmask(mock_request_with_structured_output)
|
||||
result = manager_with_reasoner.should_fill_bitmask(
|
||||
mock_request_with_structured_output
|
||||
)
|
||||
|
||||
# Should set reasoning_ended and return its value
|
||||
assert (
|
||||
@@ -118,68 +120,92 @@ class TestReasoningStructuredOutput:
|
||||
):
|
||||
"""Test should_fill_bitmask when no reasoner is configured."""
|
||||
manager = StructuredOutputManager(mock_vllm_config)
|
||||
manager.reasoner = None
|
||||
|
||||
result = manager.should_fill_bitmask(mock_request_with_structured_output)
|
||||
|
||||
# Should default to True when no reasoner
|
||||
assert result is True
|
||||
|
||||
def test_should_fill_bitmask_uses_request_reasoning_parser_kwargs(
|
||||
self, mock_vllm_config, mock_request_with_structured_output
|
||||
):
|
||||
"""Test request-level parser kwargs override the default reasoner."""
|
||||
|
||||
class KwargReasoner:
|
||||
def __init__(self, tokenizer, chat_template_kwargs=None):
|
||||
self.chat_template_kwargs = chat_template_kwargs or {}
|
||||
|
||||
def is_reasoning_end(self, input_ids):
|
||||
return not self.chat_template_kwargs.get("enable_thinking", False)
|
||||
|
||||
manager = StructuredOutputManager(mock_vllm_config)
|
||||
manager.reasoner_cls = KwargReasoner
|
||||
manager.tokenizer = Mock()
|
||||
|
||||
structured_req = mock_request_with_structured_output.structured_output_request
|
||||
structured_req.reasoning_parser_kwargs = {
|
||||
"chat_template_kwargs": {"enable_thinking": True}
|
||||
}
|
||||
|
||||
result = manager.should_fill_bitmask(mock_request_with_structured_output)
|
||||
|
||||
assert result is False
|
||||
assert (
|
||||
mock_request_with_structured_output.structured_output_request.reasoner
|
||||
is not None
|
||||
)
|
||||
|
||||
def test_should_advance_with_enable_in_reasoning(
|
||||
self,
|
||||
mock_vllm_config,
|
||||
manager_with_reasoner,
|
||||
mock_request_with_structured_output,
|
||||
mock_reasoning_parser,
|
||||
):
|
||||
"""Test should_advance when enable_in_reasoning is True."""
|
||||
# Enable enable_in_reasoning
|
||||
mock_vllm_config.structured_outputs_config.enable_in_reasoning = True
|
||||
|
||||
manager = StructuredOutputManager(mock_vllm_config)
|
||||
manager.reasoner = mock_reasoning_parser
|
||||
manager_with_reasoner.enable_in_reasoning = True
|
||||
|
||||
# Should always return True when enable_in_reasoning is enabled
|
||||
result = manager.should_advance(mock_request_with_structured_output)
|
||||
result = manager_with_reasoner.should_advance(
|
||||
mock_request_with_structured_output
|
||||
)
|
||||
assert result is True
|
||||
|
||||
def test_should_advance_reasoning_not_ended(
|
||||
self,
|
||||
mock_vllm_config,
|
||||
manager_with_reasoner,
|
||||
mock_request_with_structured_output,
|
||||
mock_reasoning_parser,
|
||||
):
|
||||
"""Test should_advance when reasoning has not ended."""
|
||||
manager = StructuredOutputManager(mock_vllm_config)
|
||||
manager.reasoner = mock_reasoning_parser
|
||||
|
||||
# Set reasoning as not ended
|
||||
(
|
||||
mock_request_with_structured_output.structured_output_request
|
||||
).reasoning_ended = False
|
||||
mock_reasoning_parser.is_reasoning_end.return_value = False
|
||||
|
||||
result = manager.should_advance(mock_request_with_structured_output)
|
||||
result = manager_with_reasoner.should_advance(
|
||||
mock_request_with_structured_output
|
||||
)
|
||||
|
||||
# Should return False since reasoning hasn't ended
|
||||
assert result is False
|
||||
|
||||
def test_should_advance_reasoning_just_ended(
|
||||
self,
|
||||
mock_vllm_config,
|
||||
manager_with_reasoner,
|
||||
mock_request_with_structured_output,
|
||||
mock_reasoning_parser,
|
||||
):
|
||||
"""Test should_advance when reasoning ends in current step."""
|
||||
manager = StructuredOutputManager(mock_vllm_config)
|
||||
manager.reasoner = mock_reasoning_parser
|
||||
|
||||
# Set reasoning as not ended initially, but ends in this step
|
||||
(
|
||||
mock_request_with_structured_output.structured_output_request
|
||||
).reasoning_ended = False
|
||||
mock_reasoning_parser.is_reasoning_end.return_value = True
|
||||
reasoner = MockReasoner(tokenizer=Mock())
|
||||
reasoner.is_reasoning_end_streaming.return_value = True
|
||||
structured_req = mock_request_with_structured_output.structured_output_request
|
||||
structured_req.reasoner = reasoner
|
||||
|
||||
result = manager.should_advance(mock_request_with_structured_output)
|
||||
result = manager_with_reasoner.should_advance(
|
||||
mock_request_with_structured_output
|
||||
)
|
||||
|
||||
# Should set reasoning_ended to True but return False for this step
|
||||
assert (
|
||||
@@ -190,20 +216,18 @@ class TestReasoningStructuredOutput:
|
||||
|
||||
def test_should_advance_reasoning_already_ended(
|
||||
self,
|
||||
mock_vllm_config,
|
||||
manager_with_reasoner,
|
||||
mock_request_with_structured_output,
|
||||
mock_reasoning_parser,
|
||||
):
|
||||
"""Test should_advance when reasoning has already ended."""
|
||||
manager = StructuredOutputManager(mock_vllm_config)
|
||||
manager.reasoner = mock_reasoning_parser
|
||||
|
||||
# Set reasoning as already ended
|
||||
(
|
||||
mock_request_with_structured_output.structured_output_request
|
||||
).reasoning_ended = True
|
||||
|
||||
result = manager.should_advance(mock_request_with_structured_output)
|
||||
result = manager_with_reasoner.should_advance(
|
||||
mock_request_with_structured_output
|
||||
)
|
||||
|
||||
# Should return True since reasoning has ended
|
||||
assert result is True
|
||||
|
||||
@@ -78,6 +78,7 @@ class EngineClient(ABC):
|
||||
priority: int = 0,
|
||||
data_parallel_rank: int | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
"""Generate outputs for a request."""
|
||||
...
|
||||
|
||||
@@ -347,6 +347,11 @@ class OpenAIServingChat(OpenAIServing):
|
||||
priority=request.priority,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs={
|
||||
"chat_template_kwargs": chat_template_kwargs,
|
||||
}
|
||||
if reasoning_parser
|
||||
else None,
|
||||
)
|
||||
|
||||
generators.append(generator)
|
||||
|
||||
@@ -456,9 +456,13 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
context = SimpleContext()
|
||||
|
||||
if self.parser and self.parser.reasoning_parser_cls is not None:
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
||||
reasoning_parser_kwargs = {
|
||||
"chat_template_kwargs": chat_template_kwargs,
|
||||
}
|
||||
reasoning_parser = self.parser.reasoning_parser_cls(
|
||||
tokenizer,
|
||||
chat_template_kwargs=self._effective_chat_template_kwargs(request),
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
if (
|
||||
isinstance(
|
||||
@@ -481,6 +485,9 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
trace_headers=trace_headers,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs
|
||||
if self.parser and self.parser.reasoning_parser_cls is not None
|
||||
else None,
|
||||
)
|
||||
generators.append(generator)
|
||||
|
||||
@@ -627,6 +634,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
max_model_len = self.model_config.max_model_len
|
||||
|
||||
@@ -650,6 +658,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
)
|
||||
|
||||
async for res in generator:
|
||||
|
||||
@@ -120,6 +120,7 @@ class EngineCoreRequest(
|
||||
external_req_id: str | None = None
|
||||
|
||||
reasoning_ended: bool | None = None
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None
|
||||
|
||||
@property
|
||||
def params(self) -> SamplingParams | PoolingParams:
|
||||
|
||||
@@ -293,6 +293,7 @@ class AsyncLLM(EngineClient):
|
||||
data_parallel_rank: int | None = None,
|
||||
prompt_text: str | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> RequestOutputCollector:
|
||||
"""Add new request to the AsyncLLM."""
|
||||
|
||||
@@ -313,7 +314,7 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
|
||||
if isinstance(prompt, AsyncGenerator):
|
||||
if reasoning_ended is not None:
|
||||
if reasoning_ended is not None or reasoning_parser_kwargs is not None:
|
||||
raise NotImplementedError
|
||||
|
||||
# Streaming input case.
|
||||
@@ -361,6 +362,8 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
if reasoning_ended is not None:
|
||||
request.reasoning_ended = reasoning_ended
|
||||
if reasoning_parser_kwargs is not None:
|
||||
request.reasoning_parser_kwargs = reasoning_parser_kwargs
|
||||
|
||||
self.input_processor.assign_request_id(request)
|
||||
|
||||
@@ -534,6 +537,7 @@ class AsyncLLM(EngineClient):
|
||||
priority: int = 0,
|
||||
data_parallel_rank: int | None = None,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> AsyncGenerator[RequestOutput, None]:
|
||||
"""
|
||||
Main function called by the API server to kick off a request
|
||||
@@ -563,6 +567,7 @@ class AsyncLLM(EngineClient):
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
prompt_text=prompt_text,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs=reasoning_parser_kwargs,
|
||||
)
|
||||
|
||||
# The output_handler task pushes items into the queue.
|
||||
|
||||
@@ -75,6 +75,7 @@ class Request:
|
||||
block_hasher: Callable[["Request"], list["BlockHash"]] | None = None,
|
||||
resumable: bool = False,
|
||||
reasoning_ended: bool | None = None,
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
self.request_id = request_id
|
||||
self.client_index = client_index
|
||||
@@ -87,6 +88,9 @@ class Request:
|
||||
)
|
||||
if self.structured_output_request is not None:
|
||||
self.structured_output_request.reasoning_ended = reasoning_ended
|
||||
self.structured_output_request.reasoning_parser_kwargs = (
|
||||
reasoning_parser_kwargs
|
||||
)
|
||||
self.arrival_time = arrival_time if arrival_time is not None else time.time()
|
||||
|
||||
self.status = RequestStatus.WAITING
|
||||
@@ -201,6 +205,7 @@ class Request:
|
||||
block_hasher=block_hasher,
|
||||
resumable=request.resumable,
|
||||
reasoning_ended=request.reasoning_ended,
|
||||
reasoning_parser_kwargs=request.reasoning_parser_kwargs,
|
||||
)
|
||||
|
||||
def append_output_token_ids(
|
||||
|
||||
@@ -37,7 +37,10 @@ class StructuredOutputManager:
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig):
|
||||
self.backend: StructuredOutputBackend | None = None
|
||||
self.reasoner: ReasoningParser | None = None
|
||||
# We only store the class of the reasoner in the manager.
|
||||
# The parser instance is request-scoped because some reasoning parsers
|
||||
# depend on per-request chat-template kwargs.
|
||||
self.reasoner_cls: type[ReasoningParser] | None = None
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
# When in external_launcher mode, async grammar compilation causes deadlocks
|
||||
@@ -85,15 +88,29 @@ class StructuredOutputManager:
|
||||
self.vllm_config.structured_outputs_config.reasoning_parser
|
||||
)
|
||||
if reasoning_parser:
|
||||
reasoner_cls = ReasoningParserManager.get_reasoning_parser(
|
||||
self.reasoner_cls = ReasoningParserManager.get_reasoning_parser(
|
||||
reasoning_parser
|
||||
)
|
||||
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
|
||||
|
||||
self.enable_in_reasoning = (
|
||||
self.vllm_config.structured_outputs_config.enable_in_reasoning
|
||||
)
|
||||
|
||||
def _get_reasoner(self, request: "Request") -> "ReasoningParser | None":
|
||||
structured_req = request.structured_output_request
|
||||
if structured_req is None or self.reasoner_cls is None:
|
||||
return None
|
||||
|
||||
if structured_req.reasoner is None:
|
||||
# Lazily build the request-local parser so the structured-output
|
||||
# gate observes the same template kwargs used by the frontend.
|
||||
parser_kwargs = structured_req.reasoning_parser_kwargs or {}
|
||||
structured_req.reasoner = self.reasoner_cls(
|
||||
tokenizer=self.tokenizer,
|
||||
**parser_kwargs,
|
||||
)
|
||||
return structured_req.reasoner
|
||||
|
||||
def grammar_init(self, request: "Request") -> None:
|
||||
if request.structured_output_request is None:
|
||||
return
|
||||
@@ -285,7 +302,8 @@ class StructuredOutputManager:
|
||||
# NOTE (Hanchen) if enable_in_reasoning is True, it means that
|
||||
# the model needs to be constrained in reasoning. So we should always
|
||||
# enable the bitmask filling.
|
||||
if self.reasoner is not None:
|
||||
reasoner = self._get_reasoner(request)
|
||||
if reasoner is not None:
|
||||
if self.enable_in_reasoning:
|
||||
return True
|
||||
assert request.structured_output_request is not None
|
||||
@@ -295,7 +313,7 @@ class StructuredOutputManager:
|
||||
# After unifying the `openai_gptoss` and non-`openai_gptoss` styles,
|
||||
# it can be removed.
|
||||
request.structured_output_request.reasoning_ended = (
|
||||
self.reasoner.is_reasoning_end(request.prompt_token_ids or [])
|
||||
reasoner.is_reasoning_end(request.prompt_token_ids or [])
|
||||
)
|
||||
return request.structured_output_request.reasoning_ended
|
||||
return True
|
||||
@@ -311,7 +329,8 @@ class StructuredOutputManager:
|
||||
assert request.structured_output_request.grammar is not None
|
||||
# by default, we should always advance
|
||||
# for cases that don't use thinking mode.
|
||||
if self.reasoner is None:
|
||||
reasoner = self._get_reasoner(request)
|
||||
if reasoner is None:
|
||||
return True
|
||||
|
||||
# if the model needs structured in reasoning, we should advance
|
||||
@@ -328,7 +347,7 @@ class StructuredOutputManager:
|
||||
start = (
|
||||
delta_from if delta_from >= 0 else max(len(all_token_ids) + delta_from, 0)
|
||||
)
|
||||
if self.reasoner.is_reasoning_end_streaming(
|
||||
if reasoner.is_reasoning_end_streaming(
|
||||
all_token_ids, itertools.islice(all_token_ids, start, None)
|
||||
):
|
||||
# Reasoning just ended, so we shouldn't advance til
|
||||
|
||||
@@ -5,7 +5,7 @@ import functools
|
||||
import json
|
||||
from concurrent.futures import Future
|
||||
from concurrent.futures._base import TimeoutError
|
||||
from typing import cast
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
@@ -14,12 +14,19 @@ from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputOptions,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.reasoning import ReasoningParser
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class StructuredOutputRequest:
|
||||
params: StructuredOutputsParams
|
||||
_grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None
|
||||
reasoning_ended: bool | None = None
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None
|
||||
# Cached per request; do not share reasoning parsers across requests because
|
||||
# their behavior can depend on reasoning_parser_kwargs.
|
||||
reasoner: "ReasoningParser | None" = None
|
||||
|
||||
@staticmethod
|
||||
def from_sampling_params(
|
||||
|
||||
Reference in New Issue
Block a user