diff --git a/tests/v1/structured_output/test_reasoning_structured_output.py b/tests/v1/structured_output/test_reasoning_structured_output.py index 861e919c10..ad5f1d5d79 100644 --- a/tests/v1/structured_output/test_reasoning_structured_output.py +++ b/tests/v1/structured_output/test_reasoning_structured_output.py @@ -73,6 +73,7 @@ class TestReasoningStructuredOutput: request.all_token_ids = [1, 2, 3, 4, 5, 6, 7, 8] request.num_computed_tokens = 5 request.num_output_placeholders = 0 + request.request_id = "mock_req" return request @pytest.fixture @@ -208,37 +209,11 @@ class TestReasoningStructuredOutput: mock_request_with_structured_output ) - # Should set reasoning_ended to True but return False for this step + # The scheduler trims the reasoning prefix before advancing the grammar. assert ( mock_request_with_structured_output.structured_output_request.reasoning_ended is True ) - assert result is False - - def test_should_advance_reasoning_just_ended_with_spec_decode_structural_tag( - self, - manager_with_reasoner, - mock_request_with_structured_output, - ): - """When reasoning ends this step, advance immediately for structural - tags with speculative decoding.""" - structured_req = mock_request_with_structured_output.structured_output_request - structured_req.reasoning_ended = False - structured_req.structured_output_key = ( - StructuredOutputOptions.STRUCTURAL_TAG, - "{}", - ) - reasoner = MockReasoner(tokenizer=Mock()) - reasoner.is_reasoning_end_streaming.return_value = True - structured_req.reasoner = reasoner - - manager_with_reasoner.vllm_config.speculative_config = Mock() - - result = manager_with_reasoner.should_advance( - mock_request_with_structured_output - ) - - assert structured_req.reasoning_ended is True assert result is True def test_should_advance_reasoning_already_ended( @@ -258,3 +233,120 @@ class TestReasoningStructuredOutput: # Should return True since reasoning has ended assert result is True + + def test_should_advance_uses_new_token_ids_when_provided( + self, + manager_with_reasoner, + mock_request_with_structured_output, + ): + """Regression for #43388: when caller passes new_token_ids, the + reasoner sees the exact multi-token delta rather than the + placeholder-derived window. + """ + structured_req = mock_request_with_structured_output.structured_output_request + structured_req.reasoning_ended = False + + end_token_id = 248069 + + reasoner = MockReasoner(tokenizer=Mock()) + # Detection mirrors the real Qwen3 parser: end token in the delta. + reasoner.is_reasoning_end_streaming = Mock( + side_effect=lambda input_ids, delta_ids: end_token_id in list(delta_ids) + ) + structured_req.reasoner = reasoner + + # Scenario from #43388: async + spec decode K=4, 4 tokens accepted + # but only 1 placeholder remains (some drafts were rejected). + # The placeholder math would yield delta=[271] and miss . + # Passing new_token_ids must override that. + new_token_ids = [9, 198, end_token_id, 271] + mock_request_with_structured_output.all_token_ids = [ + 1, + 2, + 3, + 4, + 5, + ] + new_token_ids + mock_request_with_structured_output.num_computed_tokens = 9 + mock_request_with_structured_output.num_output_placeholders = 1 + + result = manager_with_reasoner.should_advance( + mock_request_with_structured_output, + new_token_ids=new_token_ids, + ) + + # First call to is_reasoning_end_streaming was with the full + # new_token_ids (not the truncated placeholder window). + first_call = reasoner.is_reasoning_end_streaming.call_args_list[0] + _, called_delta = first_call.args + assert list(called_delta) == new_token_ids + + assert structured_req.reasoning_ended is True + assert result is True + + def test_should_advance_without_new_token_ids_falls_back( + self, + manager_with_reasoner, + mock_request_with_structured_output, + ): + """Backward compat: callers that don't pass new_token_ids keep + the original placeholder-derived delta window. + """ + structured_req = mock_request_with_structured_output.structured_output_request + structured_req.reasoning_ended = False + reasoner = MockReasoner(tokenizer=Mock()) + reasoner.is_reasoning_end_streaming.return_value = False + structured_req.reasoner = reasoner + + mock_request_with_structured_output.all_token_ids = [1, 2, 3, 4, 5] + mock_request_with_structured_output.num_computed_tokens = 5 + mock_request_with_structured_output.num_output_placeholders = 2 + + result = manager_with_reasoner.should_advance( + mock_request_with_structured_output + ) + + # placeholder window: start = 5 - 2 = 3, delta = [4, 5] + _, called_delta = reasoner.is_reasoning_end_streaming.call_args[0] + assert list(called_delta) == [4, 5] + assert result is False + + def test_should_advance_trims_reasoning_prefix_for_json( + self, + manager_with_reasoner, + mock_request_with_structured_output, + ): + """JSON uses the common trim-then-advance path at the boundary.""" + structured_req = mock_request_with_structured_output.structured_output_request + structured_req.reasoning_ended = False + structured_req.structured_output_key = ( + StructuredOutputOptions.JSON_OBJECT, + "{}", + ) + + marker = 248069 + + class MarkerReasoner: + def __init__(self, *_, **__): + pass + + def is_reasoning_end_streaming(self, input_ids, delta_ids): + return marker in list(delta_ids) + + structured_req.reasoner = MarkerReasoner() + + new_token_ids = [9, 198, marker, 271, 5005] + mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids + + result = manager_with_reasoner.should_advance( + mock_request_with_structured_output, + new_token_ids=new_token_ids, + ) + + structured_req.grammar.accept_tokens.assert_not_called() + assert structured_req.reasoning_ended is True + assert result is True + assert structured_req.reasoning_end_token_index == 5 + assert manager_with_reasoner.trim_reasoning_for_advance( + mock_request_with_structured_output, new_token_ids + ) == [271, 5005] diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index aca76d9cca..1190b93cc8 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1716,7 +1716,9 @@ class Scheduler(SchedulerInterface): request.status = RequestStatus.FINISHED_STOPPED stopped = True - if new_token_ids and self.structured_output_manager.should_advance(request): + if new_token_ids and self.structured_output_manager.should_advance( + request, new_token_ids=new_token_ids + ): struct_output_request = request.structured_output_request assert struct_output_request is not None grammar = struct_output_request.grammar diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 939a387ee9..7619a7fc25 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -7,7 +7,6 @@ from concurrent.futures import Future, ThreadPoolExecutor from typing import TYPE_CHECKING from vllm.config import VllmConfig -from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager from vllm.tokenizers import cached_tokenizer_from_config from vllm.utils.import_utils import LazyLoader @@ -15,7 +14,6 @@ from vllm.v1.structured_output.backend_guidance import GuidanceBackend from vllm.v1.structured_output.backend_types import ( StructuredOutputBackend, StructuredOutputGrammar, - StructuredOutputOptions, ) from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend @@ -30,9 +28,6 @@ else: torch = LazyLoader("torch", globals(), "torch") -logger = init_logger(__name__) - - class StructuredOutputManager: """Engine-level manager for structured output requests.""" @@ -379,7 +374,11 @@ class StructuredOutputManager: return request.structured_output_request.reasoning_ended return True - def should_advance(self, request: "Request") -> bool: + def should_advance( + self, + request: "Request", + new_token_ids: list[int] | None = None, + ) -> bool: if not request.use_structured_output: return False @@ -402,37 +401,36 @@ class StructuredOutputManager: if structured_req.reasoning_ended: return True - # Check if reasoning ends in *this* step - delta_from = request.num_computed_tokens - request.num_output_placeholders + # Check if reasoning ends in *this* step. + # When the caller passes new_token_ids (the tokens that were just + # appended this step), use it directly as the delta window. The + # placeholder-derived fallback assumes num_output_placeholders == + # len(new_token_ids), which breaks under async scheduling + spec + # decode when some drafts are rejected (#43388): the placeholder + # count remains > 0 after the step and the computed delta window + # starts past the reasoning-end marker. all_token_ids = request.all_token_ids - start = ( - delta_from if delta_from >= 0 else max(len(all_token_ids) + delta_from, 0) - ) - if reasoner.is_reasoning_end_streaming( - all_token_ids, itertools.islice(all_token_ids, start, None) - ): + if new_token_ids: + # The tokens were already appended this step, so the step window + # starts exactly len(new_token_ids) from the end. + start = len(all_token_ids) - len(new_token_ids) + delta_ids: Iterable[int] = new_token_ids + else: + delta_from = request.num_computed_tokens - request.num_output_placeholders + start = ( + delta_from + if delta_from >= 0 + else max(len(all_token_ids) + delta_from, 0) + ) + delta_ids = itertools.islice(all_token_ids, start, None) + if reasoner.is_reasoning_end_streaming(all_token_ids, delta_ids): structured_req.reasoning_ended = True - # Reasoning just ended this step. Defer FSM advance until the next - # pass (see reasoning_ended check above) for JSON/regex/choice/grammar: - # advancing on the closing boundary token can accept tokens that still - # belong to the reasoning stream. Structural tags are the only safe - # same-step exception: they model phased output (e.g. thinking tag -> - # answer tag), and speculative decoding must run grammar.validate_tokens - # on draft tokens produced immediately after that transition. - if ( - self.vllm_config.speculative_config is not None - and structured_req.structured_output_key[0] - == StructuredOutputOptions.STRUCTURAL_TAG - ): - # The scheduler will advance the grammar with this step's - # tokens right away, but the step still contains reasoning - # content up to and including the end marker. Record where - # it ends so trim_reasoning_for_advance() can drop it. - structured_req.reasoning_end_token_index = ( - self._find_reasoning_end_index(reasoner, all_token_ids, start) - ) - return True + # Record the boundary so the scheduler can exclude reasoning tokens. + end_index = self._find_reasoning_end_index(reasoner, all_token_ids, start) + + structured_req.reasoning_end_token_index = end_index + return True return False