[Bugfix][Structured Output][Spec Decode] Advance grammar across reasoning boundary (#44993)
Signed-off-by: Allen.Yu <yuyue0225sc@163.com>
This commit is contained in:
@@ -73,6 +73,7 @@ class TestReasoningStructuredOutput:
|
||||
request.all_token_ids = [1, 2, 3, 4, 5, 6, 7, 8]
|
||||
request.num_computed_tokens = 5
|
||||
request.num_output_placeholders = 0
|
||||
request.request_id = "mock_req"
|
||||
return request
|
||||
|
||||
@pytest.fixture
|
||||
@@ -208,37 +209,11 @@ class TestReasoningStructuredOutput:
|
||||
mock_request_with_structured_output
|
||||
)
|
||||
|
||||
# Should set reasoning_ended to True but return False for this step
|
||||
# The scheduler trims the reasoning prefix before advancing the grammar.
|
||||
assert (
|
||||
mock_request_with_structured_output.structured_output_request.reasoning_ended
|
||||
is True
|
||||
)
|
||||
assert result is False
|
||||
|
||||
def test_should_advance_reasoning_just_ended_with_spec_decode_structural_tag(
|
||||
self,
|
||||
manager_with_reasoner,
|
||||
mock_request_with_structured_output,
|
||||
):
|
||||
"""When reasoning ends this step, advance immediately for structural
|
||||
tags with speculative decoding."""
|
||||
structured_req = mock_request_with_structured_output.structured_output_request
|
||||
structured_req.reasoning_ended = False
|
||||
structured_req.structured_output_key = (
|
||||
StructuredOutputOptions.STRUCTURAL_TAG,
|
||||
"{}",
|
||||
)
|
||||
reasoner = MockReasoner(tokenizer=Mock())
|
||||
reasoner.is_reasoning_end_streaming.return_value = True
|
||||
structured_req.reasoner = reasoner
|
||||
|
||||
manager_with_reasoner.vllm_config.speculative_config = Mock()
|
||||
|
||||
result = manager_with_reasoner.should_advance(
|
||||
mock_request_with_structured_output
|
||||
)
|
||||
|
||||
assert structured_req.reasoning_ended is True
|
||||
assert result is True
|
||||
|
||||
def test_should_advance_reasoning_already_ended(
|
||||
@@ -258,3 +233,120 @@ class TestReasoningStructuredOutput:
|
||||
|
||||
# Should return True since reasoning has ended
|
||||
assert result is True
|
||||
|
||||
def test_should_advance_uses_new_token_ids_when_provided(
|
||||
self,
|
||||
manager_with_reasoner,
|
||||
mock_request_with_structured_output,
|
||||
):
|
||||
"""Regression for #43388: when caller passes new_token_ids, the
|
||||
reasoner sees the exact multi-token delta rather than the
|
||||
placeholder-derived window.
|
||||
"""
|
||||
structured_req = mock_request_with_structured_output.structured_output_request
|
||||
structured_req.reasoning_ended = False
|
||||
|
||||
end_token_id = 248069
|
||||
|
||||
reasoner = MockReasoner(tokenizer=Mock())
|
||||
# Detection mirrors the real Qwen3 parser: end token in the delta.
|
||||
reasoner.is_reasoning_end_streaming = Mock(
|
||||
side_effect=lambda input_ids, delta_ids: end_token_id in list(delta_ids)
|
||||
)
|
||||
structured_req.reasoner = reasoner
|
||||
|
||||
# Scenario from #43388: async + spec decode K=4, 4 tokens accepted
|
||||
# but only 1 placeholder remains (some drafts were rejected).
|
||||
# The placeholder math would yield delta=[271] and miss </think>.
|
||||
# Passing new_token_ids must override that.
|
||||
new_token_ids = [9, 198, end_token_id, 271]
|
||||
mock_request_with_structured_output.all_token_ids = [
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
] + new_token_ids
|
||||
mock_request_with_structured_output.num_computed_tokens = 9
|
||||
mock_request_with_structured_output.num_output_placeholders = 1
|
||||
|
||||
result = manager_with_reasoner.should_advance(
|
||||
mock_request_with_structured_output,
|
||||
new_token_ids=new_token_ids,
|
||||
)
|
||||
|
||||
# First call to is_reasoning_end_streaming was with the full
|
||||
# new_token_ids (not the truncated placeholder window).
|
||||
first_call = reasoner.is_reasoning_end_streaming.call_args_list[0]
|
||||
_, called_delta = first_call.args
|
||||
assert list(called_delta) == new_token_ids
|
||||
|
||||
assert structured_req.reasoning_ended is True
|
||||
assert result is True
|
||||
|
||||
def test_should_advance_without_new_token_ids_falls_back(
|
||||
self,
|
||||
manager_with_reasoner,
|
||||
mock_request_with_structured_output,
|
||||
):
|
||||
"""Backward compat: callers that don't pass new_token_ids keep
|
||||
the original placeholder-derived delta window.
|
||||
"""
|
||||
structured_req = mock_request_with_structured_output.structured_output_request
|
||||
structured_req.reasoning_ended = False
|
||||
reasoner = MockReasoner(tokenizer=Mock())
|
||||
reasoner.is_reasoning_end_streaming.return_value = False
|
||||
structured_req.reasoner = reasoner
|
||||
|
||||
mock_request_with_structured_output.all_token_ids = [1, 2, 3, 4, 5]
|
||||
mock_request_with_structured_output.num_computed_tokens = 5
|
||||
mock_request_with_structured_output.num_output_placeholders = 2
|
||||
|
||||
result = manager_with_reasoner.should_advance(
|
||||
mock_request_with_structured_output
|
||||
)
|
||||
|
||||
# placeholder window: start = 5 - 2 = 3, delta = [4, 5]
|
||||
_, called_delta = reasoner.is_reasoning_end_streaming.call_args[0]
|
||||
assert list(called_delta) == [4, 5]
|
||||
assert result is False
|
||||
|
||||
def test_should_advance_trims_reasoning_prefix_for_json(
|
||||
self,
|
||||
manager_with_reasoner,
|
||||
mock_request_with_structured_output,
|
||||
):
|
||||
"""JSON uses the common trim-then-advance path at the boundary."""
|
||||
structured_req = mock_request_with_structured_output.structured_output_request
|
||||
structured_req.reasoning_ended = False
|
||||
structured_req.structured_output_key = (
|
||||
StructuredOutputOptions.JSON_OBJECT,
|
||||
"{}",
|
||||
)
|
||||
|
||||
marker = 248069
|
||||
|
||||
class MarkerReasoner:
|
||||
def __init__(self, *_, **__):
|
||||
pass
|
||||
|
||||
def is_reasoning_end_streaming(self, input_ids, delta_ids):
|
||||
return marker in list(delta_ids)
|
||||
|
||||
structured_req.reasoner = MarkerReasoner()
|
||||
|
||||
new_token_ids = [9, 198, marker, 271, 5005]
|
||||
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
|
||||
|
||||
result = manager_with_reasoner.should_advance(
|
||||
mock_request_with_structured_output,
|
||||
new_token_ids=new_token_ids,
|
||||
)
|
||||
|
||||
structured_req.grammar.accept_tokens.assert_not_called()
|
||||
assert structured_req.reasoning_ended is True
|
||||
assert result is True
|
||||
assert structured_req.reasoning_end_token_index == 5
|
||||
assert manager_with_reasoner.trim_reasoning_for_advance(
|
||||
mock_request_with_structured_output, new_token_ids
|
||||
) == [271, 5005]
|
||||
|
||||
@@ -1716,7 +1716,9 @@ class Scheduler(SchedulerInterface):
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
stopped = True
|
||||
|
||||
if new_token_ids and self.structured_output_manager.should_advance(request):
|
||||
if new_token_ids and self.structured_output_manager.should_advance(
|
||||
request, new_token_ids=new_token_ids
|
||||
):
|
||||
struct_output_request = request.structured_output_request
|
||||
assert struct_output_request is not None
|
||||
grammar = struct_output_request.grammar
|
||||
|
||||
@@ -7,7 +7,6 @@ from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
@@ -15,7 +14,6 @@ from vllm.v1.structured_output.backend_guidance import GuidanceBackend
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
StructuredOutputOptions,
|
||||
)
|
||||
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
|
||||
|
||||
@@ -30,9 +28,6 @@ else:
|
||||
torch = LazyLoader("torch", globals(), "torch")
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class StructuredOutputManager:
|
||||
"""Engine-level manager for structured output requests."""
|
||||
|
||||
@@ -379,7 +374,11 @@ class StructuredOutputManager:
|
||||
return request.structured_output_request.reasoning_ended
|
||||
return True
|
||||
|
||||
def should_advance(self, request: "Request") -> bool:
|
||||
def should_advance(
|
||||
self,
|
||||
request: "Request",
|
||||
new_token_ids: list[int] | None = None,
|
||||
) -> bool:
|
||||
if not request.use_structured_output:
|
||||
return False
|
||||
|
||||
@@ -402,37 +401,36 @@ class StructuredOutputManager:
|
||||
if structured_req.reasoning_ended:
|
||||
return True
|
||||
|
||||
# Check if reasoning ends in *this* step
|
||||
delta_from = request.num_computed_tokens - request.num_output_placeholders
|
||||
# Check if reasoning ends in *this* step.
|
||||
# When the caller passes new_token_ids (the tokens that were just
|
||||
# appended this step), use it directly as the delta window. The
|
||||
# placeholder-derived fallback assumes num_output_placeholders ==
|
||||
# len(new_token_ids), which breaks under async scheduling + spec
|
||||
# decode when some drafts are rejected (#43388): the placeholder
|
||||
# count remains > 0 after the step and the computed delta window
|
||||
# starts past the reasoning-end marker.
|
||||
all_token_ids = request.all_token_ids
|
||||
start = (
|
||||
delta_from if delta_from >= 0 else max(len(all_token_ids) + delta_from, 0)
|
||||
)
|
||||
if reasoner.is_reasoning_end_streaming(
|
||||
all_token_ids, itertools.islice(all_token_ids, start, None)
|
||||
):
|
||||
if new_token_ids:
|
||||
# The tokens were already appended this step, so the step window
|
||||
# starts exactly len(new_token_ids) from the end.
|
||||
start = len(all_token_ids) - len(new_token_ids)
|
||||
delta_ids: Iterable[int] = new_token_ids
|
||||
else:
|
||||
delta_from = request.num_computed_tokens - request.num_output_placeholders
|
||||
start = (
|
||||
delta_from
|
||||
if delta_from >= 0
|
||||
else max(len(all_token_ids) + delta_from, 0)
|
||||
)
|
||||
delta_ids = itertools.islice(all_token_ids, start, None)
|
||||
if reasoner.is_reasoning_end_streaming(all_token_ids, delta_ids):
|
||||
structured_req.reasoning_ended = True
|
||||
|
||||
# Reasoning just ended this step. Defer FSM advance until the next
|
||||
# pass (see reasoning_ended check above) for JSON/regex/choice/grammar:
|
||||
# advancing on the closing boundary token can accept tokens that still
|
||||
# belong to the reasoning stream. Structural tags are the only safe
|
||||
# same-step exception: they model phased output (e.g. thinking tag ->
|
||||
# answer tag), and speculative decoding must run grammar.validate_tokens
|
||||
# on draft tokens produced immediately after that transition.
|
||||
if (
|
||||
self.vllm_config.speculative_config is not None
|
||||
and structured_req.structured_output_key[0]
|
||||
== StructuredOutputOptions.STRUCTURAL_TAG
|
||||
):
|
||||
# The scheduler will advance the grammar with this step's
|
||||
# tokens right away, but the step still contains reasoning
|
||||
# content up to and including the end marker. Record where
|
||||
# it ends so trim_reasoning_for_advance() can drop it.
|
||||
structured_req.reasoning_end_token_index = (
|
||||
self._find_reasoning_end_index(reasoner, all_token_ids, start)
|
||||
)
|
||||
return True
|
||||
# Record the boundary so the scheduler can exclude reasoning tokens.
|
||||
end_index = self._find_reasoning_end_index(reasoner, all_token_ids, start)
|
||||
|
||||
structured_req.reasoning_end_token_index = end_index
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
Reference in New Issue
Block a user