[Refactor] StructuredOutputManager x Speculative Decoding Refactor (#48200)

This commit is contained in:
yzong-rh
2026-09-15 09:48:49 -04:00
committed by GitHub
parent 64856080b0
commit e6960af33b
12 changed files with 596 additions and 1192 deletions
+1 -1
View File
@@ -574,7 +574,7 @@ class TestImplicitReasoningEnd:
# the text lexer.
assert thinking_parser.reasoning_end_token_ids == {_THINK_END_ID}
assert thinking_parser.find_reasoning_end_offset([7, _THINK_END_ID, 8]) == 1
assert thinking_parser.find_reasoning_end_offset([7, 8]) is None
assert thinking_parser.find_reasoning_end_offset([7, 8]) == 2
def test_streaming_reasoning_implicit_end(self, thinking_parser):
chunks = [
+2 -2
View File
@@ -185,8 +185,8 @@ class TestReasoningEndTokenIds:
def test_find_reasoning_end_offset_returns_first_match(self):
engine = _make_engine()
assert engine.find_reasoning_end_offset([5, 201, 6, 201]) == 1
assert engine.find_reasoning_end_offset([5, 6]) is None
assert engine.find_reasoning_end_offset([]) is None
assert engine.find_reasoning_end_offset([5, 6]) == 2
assert engine.find_reasoning_end_offset([]) == 0
# Rejected-draft placeholders never match.
assert engine.find_reasoning_end_offset([-1, 201]) == 1
+1 -5
View File
@@ -334,7 +334,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
request = create_requests(num_requests=1, num_tokens=1)[0]
request.structured_output_request = Mock()
request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar)
request.structured_output_request.grammar.accept_tokens.return_value = False
request.status = RequestStatus.RUNNING
request.num_computed_tokens = request.num_tokens
request.num_output_placeholders = 1
@@ -343,10 +342,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
scheduler.connector = None
scheduler.ec_connector = None
scheduler.structured_output_manager = Mock()
scheduler.structured_output_manager.should_advance.return_value = True
scheduler.structured_output_manager.trim_reasoning_for_advance.side_effect = (
lambda request, new_token_ids: new_token_ids
)
scheduler.structured_output_manager.accept_tokens.return_value = False
scheduler.requests = {request.request_id: request}
scheduler.running = [request]
scheduler.waiting = Mock()
+11 -17
View File
@@ -3674,7 +3674,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
)
request.structured_output_request = Mock()
request.structured_output_request.grammar = Mock(spec=StructuredOutputGrammar)
request.structured_output_request.grammar.accept_tokens.return_value = False
request.status = RequestStatus.RUNNING
request.num_computed_tokens = request.num_tokens
@@ -3682,10 +3681,7 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
scheduler.connector = None
scheduler.ec_connector = None
scheduler.structured_output_manager = Mock()
scheduler.structured_output_manager.should_advance.return_value = True
scheduler.structured_output_manager.trim_reasoning_for_advance.side_effect = (
lambda request, new_token_ids: new_token_ids
)
scheduler.structured_output_manager.accept_tokens.return_value = False
scheduler.requests = {request.request_id: request}
scheduler.running = [request]
scheduler.waiting = Mock()
@@ -3734,8 +3730,8 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
)
engine_core_outputs = scheduler.update_from_output(output, model_runner_output)
request.structured_output_request.grammar.accept_tokens.assert_called_once_with(
request.request_id, [123]
scheduler.structured_output_manager.accept_tokens.assert_called_once_with(
request, [123]
)
assert request.resumable is False
assert request.status == RequestStatus.FINISHED_ERROR
@@ -6573,16 +6569,15 @@ def _decode_ready_request(scheduler):
return request
def test_update_draft_token_ids_strips_ngram_padding(monkeypatch):
"""ngram_gpu pads unfilled draft slots with -1; update_draft_token_ids must
def test_update_draft_token_ids_strips_ngram_padding():
"""ngram_gpu pads unfilled draft slots with -1; manager validation must
strip them before grammar.validate_tokens (which otherwise raises)."""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
grammar = _RecordingGrammar()
request.structured_output_request = SimpleNamespace(grammar=grammar)
monkeypatch.setattr(
scheduler.structured_output_manager, "should_advance", lambda req: True
request.structured_output_request = SimpleNamespace(
grammar=grammar, reasoning_ended=True
)
scheduler.update_draft_token_ids(
@@ -6593,16 +6588,15 @@ def test_update_draft_token_ids_strips_ngram_padding(monkeypatch):
assert request.spec_token_ids == [10, 11]
def test_update_draft_token_ids_in_output_strips_padding(monkeypatch):
def test_update_draft_token_ids_in_output_strips_padding():
"""Same guard on the output path; the -1 pad-back for the rejected count
is preserved (only the input to validate_tokens is stripped)."""
is preserved (only the input to manager validation is stripped)."""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
grammar = _RecordingGrammar()
request.structured_output_request = SimpleNamespace(grammar=grammar)
monkeypatch.setattr(
scheduler.structured_output_manager, "should_advance", lambda req: True
request.structured_output_request = SimpleNamespace(
grammar=grammar, reasoning_ended=True
)
scheduler_output = SimpleNamespace(
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""grammar_bitmask under spec-decode draft padding (#44006)."""
"""Backend and engine-reasoner checks that are not manager-flow tests."""
import pytest
from transformers import AutoTokenizer
@@ -49,221 +49,6 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'):
return tokenizer, manager, request, prompt
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bitmask_with_padded_invalid_drafts(backend):
"""Bitmask handles -1 padded drafts and returns N+1 rows."""
tokenizer, manager, request, prompt = _make_manager_and_request(
backend, prompt_str='{"a"'
)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
valid_drafts = [tokenizer.encode(":")[0], tokenizer.encode(' "')[0]]
padded = valid_drafts + [-1, -1]
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: padded},
)
assert bitmask is not None
assert bitmask.shape[0] == len(padded) + 1
assert not grammar.is_terminated()
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bitmask_when_grammar_terminates_mid_window(backend):
"""Drafts following an EOS that terminates the grammar are a no-op."""
tokenizer, manager, request, prompt = _make_manager_and_request(backend)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
eos = tokenizer.eos_token_id
drafts = [eos] + [tokenizer.encode(" ")[0]] * (NUM_SPEC_TOKENS - 1)
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert bitmask.shape[0] == NUM_SPEC_TOKENS + 1
assert not grammar.is_terminated()
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bitmask_idempotent_across_calls(backend):
"""Repeated calls with the same input return the same bitmask."""
tokenizer, manager, request, prompt = _make_manager_and_request(
backend, prompt_str='{"a"'
)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
drafts = [tokenizer.encode(":")[0], -1, -1, -1]
first = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
second = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert first is not None and second is not None
assert (first == second).all()
assert not grammar.is_terminated()
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bonus_position_constrained_after_invalid_drafts(backend):
"""Regression for #44006: bonus row stays constrained after -1 padding."""
tokenizer, manager, request, prompt = _make_manager_and_request(
backend, prompt_str='{"a"'
)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
valid = tokenizer.encode(":")[0]
drafts = [valid, -1, -1, -1]
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert bitmask.shape[0] == len(drafts) + 1
assert not (bitmask[-1] == -1).all()
assert not grammar.is_terminated()
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bitmask_constrained_when_reasoning_ends_midwindow(backend):
"""Drafts after a mid-window reasoning-end marker stay constrained."""
tokenizer, manager, request, prompt = _make_manager_and_request(backend)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
marker = tokenizer.encode("\n")[0]
class StubReasoner:
def __init__(self, *_, **__):
self.end_token_id = marker
def is_reasoning_end(self, input_ids):
return marker in list(input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids):
return marker in list(delta_ids)
manager.reasoner_cls = StubReasoner
request.structured_output_request.reasoner = StubReasoner()
request.structured_output_request.reasoning_ended = False
pre = tokenizer.encode(" ")[0]
post = tokenizer.encode(",")[0]
drafts = [pre, marker, post]
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert bitmask.shape[0] == len(drafts) + 1
assert (bitmask[0] == -1).all()
assert (bitmask[1] == -1).all()
assert not (bitmask[2] == -1).all()
assert not (bitmask[-1] == -1).all()
assert not grammar.is_terminated()
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bitmask_post_reasoning_end_drafts_skip_grammar_advance(backend, caplog):
"""Post-marker drafts predate the bitmask and may be grammar-invalid;
grammar_bitmask must skip the grammar advance instead of asserting.
"""
tokenizer, manager, request, prompt = _make_manager_and_request(
backend, prompt_str="{"
)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
assert not grammar.is_terminated()
marker = tokenizer.encode("\n")[0]
class StubReasoner:
def __init__(self, *_, **__):
self.end_token_id = marker
def is_reasoning_end(self, input_ids):
return marker in list(input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids):
return marker in list(delta_ids)
manager.reasoner_cls = StubReasoner
request.structured_output_request.reasoner = StubReasoner()
request.structured_output_request.reasoning_ended = False
pre = tokenizer.encode(" ")[0]
# A token that the JSON grammar would reject as the first post-marker
# token; without the fix grammar.accept_tokens fires the assertion.
invalid_post = tokenizer.encode("z")[0]
drafts = [pre, marker, invalid_post]
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert bitmask.shape[0] == len(drafts) + 1
# Post-marker position is still bitmask-constrained.
assert not (bitmask[2] == -1).all()
# Grammar must not have advanced through the unvalidated draft.
assert not grammar.is_terminated()
assert "Failed to advance FSM" not in caplog.text
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_validate_tokens_then_bitmask_round_trip(backend):
"""validate_tokens -> pad with -1 -> grammar_bitmask must not assert."""
tokenizer, manager, request, prompt = _make_manager_and_request(backend)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
raw_drafts = [tokenizer.encode(",")[0], 99999, 12345, 67890]
accepted = grammar.validate_tokens(raw_drafts)
assert len(accepted) <= len(raw_drafts)
padded = accepted + [-1] * (len(raw_drafts) - len(accepted))
assert len(padded) == len(raw_drafts)
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: padded},
)
assert bitmask is not None
assert bitmask.shape[0] == len(padded) + 1
assert not grammar.is_terminated()
def test_xgrammar_accept_tokens_stops_at_termination(capfd):
"""Tokens after a terminating EOS do not reach the matcher."""
tokenizer, _, request, prompt = _make_manager_and_request("xgrammar")
@@ -302,7 +87,6 @@ def test_xgrammar_validate_tokens_stops_at_termination(capfd):
assert grammar.validate_tokens([eos, trailing]) == [eos]
assert "trying to accept new token" not in capfd.readouterr().err
# Check matcher state directly to verify validation rolled it back.
assert not grammar.matcher.is_terminated()
assert grammar.accept_tokens(request.request_id, [eos])
@@ -312,87 +96,6 @@ def test_xgrammar_validate_tokens_stops_at_termination(capfd):
assert "trying to accept new token" not in capfd.readouterr().err
class _MarkerReasoner:
"""Stub reasoner whose reasoning-end marker is a single fixed token."""
def __init__(self, marker: int):
self.marker = marker
def is_reasoning_end(self, input_ids):
return self.marker in list(input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids):
return self.marker in list(delta_ids)
def _setup_boundary_request(backend: str):
"""Request with a structural-tag key and reasoning not yet ended."""
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
tokenizer, manager, request, prompt = _make_manager_and_request(backend)
marker = tokenizer.encode("\n")[0]
structured_req = request.structured_output_request
# The grammar itself is JSON (cheap to build); only the key kind matters
# for the should_advance structural-tag branch, so pre-seed the cached
# property.
structured_req.__dict__["structured_output_key"] = (
StructuredOutputOptions.STRUCTURAL_TAG,
"",
)
manager.reasoner_cls = _MarkerReasoner
structured_req.reasoner = _MarkerReasoner(marker)
structured_req.reasoning_ended = False
return tokenizer, manager, request, prompt, marker
def test_should_advance_records_reasoning_end_index():
"""Regression for #44006 on post-#42452 main: the boundary step must
record where reasoning ends so the scheduler can trim before advancing.
"""
tokenizer, manager, request, prompt, marker = _setup_boundary_request("xgrammar")
structured_req = request.structured_output_request
pre = tokenizer.encode(" ")[0]
post = tokenizer.encode("{")[0]
request.append_output_token_ids([pre, marker, post])
assert manager.should_advance(request)
assert structured_req.reasoning_ended
# Marker sits at absolute index len(prompt) + 1.
assert structured_req.reasoning_end_token_index == len(prompt) + 1
def test_trim_reasoning_for_advance():
"""trim drops the marker and everything before it; later steps and
requests without a recorded boundary pass through unchanged.
"""
tokenizer, manager, request, prompt, marker = _setup_boundary_request("xgrammar")
structured_req = request.structured_output_request
pre = tokenizer.encode(" ")[0]
post = tokenizer.encode("{")[0]
# No boundary recorded yet: pass-through.
assert manager.trim_reasoning_for_advance(request, [pre]) == [pre]
# Boundary step: marker mid-step keeps only the suffix.
step_tokens = [pre, marker, post]
request.append_output_token_ids(step_tokens)
assert manager.should_advance(request)
assert manager.trim_reasoning_for_advance(request, step_tokens) == [post]
# Boundary step variant: marker last (the #44006 crash shape
# [198, </think>]) trims to empty -> scheduler skips accept_tokens.
structured_req.reasoning_end_token_index = len(request.all_token_ids) - 1
assert manager.trim_reasoning_for_advance(request, step_tokens) == []
# Later steps: tokens are past the boundary, returned unchanged.
structured_req.reasoning_end_token_index = len(prompt) + 1
next_step = [post, post]
request.append_output_token_ids(next_step)
assert manager.trim_reasoning_for_advance(request, next_step) == next_step
class _EngineReasonerStub(ParserEngineReasoningAdapter):
"""Adapter-typed reasoner with a fixed end-token set and no real engine."""
@@ -409,7 +112,7 @@ class _EngineReasonerStub(ParserEngineReasoningAdapter):
for offset, token in enumerate(token_ids):
if token in self._end_token_ids:
return offset
return None
return len(token_ids)
def is_reasoning_end(self, input_ids):
return any(token in self._end_token_ids for token in input_ids)
@@ -449,61 +152,3 @@ def test_bitmask_engine_reasoner_ends_midwindow_with_padding(backend):
assert not (bitmask[2] == -1).all()
assert reasoner.windows == [[pre, marker, post]]
assert not grammar.is_terminated()
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bitmask_legacy_order_sensitive_predicate_probed_per_token(backend):
"""Legacy predicates are probed one draft at a time, never on the whole
window. A predicate that reports which marker is *newest* (KimiK3) would
otherwise answer False for a window that closes and then reopens
reasoning, leaving the post-marker drafts and the bonus row
unconstrained.
"""
tokenizer, manager, request, prompt = _make_manager_and_request(backend)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
close = tokenizer.encode("\n")[0]
reopen = tokenizer.encode("#")[0]
assert close != reopen
class NewestMarkerReasoner:
def __init__(self, *_, **__):
pass
def is_reasoning_end(self, input_ids):
return self.is_reasoning_end_streaming(input_ids, input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids):
for token in reversed(list(delta_ids)):
if token == close:
return True
if token == reopen:
return False
return False
manager.reasoner_cls = NewestMarkerReasoner
request.structured_output_request.reasoner = NewestMarkerReasoner()
request.structured_output_request.reasoning_ended = False
pre = tokenizer.encode(" ")[0]
post = tokenizer.encode(",")[0]
drafts = [pre, close, post, reopen]
# Sanity: the whole window says "reopened", so only per-token probing
# can find the close marker.
assert not NewestMarkerReasoner().is_reasoning_end_streaming([], drafts)
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert bitmask.shape[0] == len(drafts) + 1
assert (bitmask[0] == -1).all()
assert (bitmask[1] == -1).all()
assert not (bitmask[2] == -1).all()
assert not (bitmask[-1] == -1).all()
assert not grammar.is_terminated()
@@ -1,557 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for reasoning-aware structured output functionality (PR #25515)."""
from collections.abc import Sequence
from unittest.mock import Mock
import pytest
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
class UntouchableTokenIds(Sequence[int]):
"""A token history that fails the test if anything reads it."""
def __init__(self, length: int) -> None:
self._length = length
def __len__(self) -> int:
return self._length
def __getitem__(self, item):
raise AssertionError("token history must not be read")
def __iter__(self):
raise AssertionError("token history must not be iterated")
def make_qwen3_reasoner():
"""A real engine-based reasoning adapter over a tiny mock vocabulary."""
from tests.parser.engine.conftest import make_mock_tokenizer
from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter
tokenizer = make_mock_tokenizer(
{"<think>": 200, "</think>": 201, "<tool_call>": 202, "</tool_call>": 203}
)
return Qwen3ParserReasoningAdapter(tokenizer)
class MockReasoner:
def __init__(self, tokenizer):
self.is_reasoning_end = Mock(return_value=False)
self.is_reasoning_end_streaming = Mock(return_value=False)
class TestReasoningStructuredOutput:
"""Test reasoning-aware structured output functionality."""
@pytest.fixture
def mock_model_config(self):
"""Create a mock ModelConfig."""
config = Mock(spec=ModelConfig)
config.skip_tokenizer_init = True # Skip tokenizer init to avoid network calls
config.get_vocab_size = Mock(return_value=50000)
# Add missing runner_type attribute that tokenizer initialization expects
config.runner_type = "generate"
# Add other attributes that tokenizer initialization might need
config.tokenizer = "test-tokenizer"
config.tokenizer_mode = "auto"
config.trust_remote_code = False
config.tokenizer_revision = None
return config
@pytest.fixture
def mock_scheduler_config(self):
"""Create a mock SchedulerConfig."""
config = Mock(spec=SchedulerConfig)
config.max_num_seqs = 128
return config
@pytest.fixture
def mock_vllm_config(self, mock_model_config, mock_scheduler_config):
"""Create a mock VllmConfig."""
config = Mock(spec=VllmConfig)
config.model_config = mock_model_config
config.scheduler_config = mock_scheduler_config
config.structured_outputs_config = Mock()
config.structured_outputs_config.reasoning_parser = None
config.structured_outputs_config.enable_in_reasoning = False
config.speculative_config = None
return config
@pytest.fixture
def mock_request_with_structured_output(self):
"""Create a mock request with structured output."""
request = Mock(spec=Request)
request.structured_output_request = Mock()
request.structured_output_request.reasoning_ended = None
request.structured_output_request.grammar = Mock()
request.structured_output_request.reasoning_parser_kwargs = None
request.structured_output_request.reasoner = None
request.structured_output_request.grammar.is_terminated = Mock(
return_value=False
)
request.use_structured_output = True
request.prompt_token_ids = [1, 2, 3, 4, 5]
request.all_token_ids = [1, 2, 3, 4, 5, 6, 7, 8]
request.num_computed_tokens = 5
request.num_output_placeholders = 0
request.request_id = "mock_req"
return request
@pytest.fixture
def manager_with_reasoner(self, mock_vllm_config):
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner_cls = MockReasoner
manager.tokenizer = Mock()
return manager
def test_should_fill_bitmask_with_enable_in_reasoning(
self, manager_with_reasoner, mock_request_with_structured_output
):
"""Test should_fill_bitmask when enable_in_reasoning is True."""
manager_with_reasoner.enable_in_reasoning = True
# Should always return True when enable_in_reasoning is enabled
result = manager_with_reasoner.should_fill_bitmask(
mock_request_with_structured_output
)
assert result is True
assert (
mock_request_with_structured_output.structured_output_request.reasoner
is None
)
def test_should_fill_bitmask_reasoning_already_ended(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""An active grammar does not need a request-local reasoner."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = True
result = manager_with_reasoner.should_fill_bitmask(
mock_request_with_structured_output
)
assert result is True
assert structured_req.reasoner is None
def test_grammar_bitmask_skips_reasoner_when_already_active(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Bitmask generation skips reasoning-boundary detection."""
manager_with_reasoner.vllm_config.num_speculative_tokens = 1
manager_with_reasoner._grammar_bitmask = Mock()
manager_with_reasoner._grammar_bitmask.shape = (1, 1)
expected_bitmask = Mock()
manager_with_reasoner._grammar_bitmask.numpy.return_value = expected_bitmask
manager_with_reasoner._fill_bitmasks = Mock()
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = True
request_id = mock_request_with_structured_output.request_id
bitmask = manager_with_reasoner.grammar_bitmask(
requests={request_id: mock_request_with_structured_output},
structured_output_request_ids=[request_id],
scheduled_spec_decode_tokens={},
)
assert bitmask is expected_bitmask
assert structured_req.reasoner is None
def test_should_fill_bitmask_without_enable_in_reasoning(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Test should_fill_bitmask when enable_in_reasoning is False."""
# Keep enable_in_reasoning as False (default)
config = manager_with_reasoner.vllm_config.structured_outputs_config
assert config.enable_in_reasoning is False
result = manager_with_reasoner.should_fill_bitmask(
mock_request_with_structured_output
)
# Should set reasoning_ended and return its value
assert (
mock_request_with_structured_output.structured_output_request.reasoning_ended
is False
)
assert result is False
def test_should_fill_bitmask_no_reasoner(
self, mock_vllm_config, mock_request_with_structured_output
):
"""Test should_fill_bitmask when no reasoner is configured."""
manager = StructuredOutputManager(mock_vllm_config)
result = manager.should_fill_bitmask(mock_request_with_structured_output)
# Should default to True when no reasoner
assert result is True
def test_should_fill_bitmask_uses_request_reasoning_parser_kwargs(
self, mock_vllm_config, mock_request_with_structured_output
):
"""Test request-level parser kwargs override the default reasoner."""
class KwargReasoner:
def __init__(self, tokenizer, chat_template_kwargs=None):
self.chat_template_kwargs = chat_template_kwargs or {}
def is_reasoning_end(self, input_ids):
return not self.chat_template_kwargs.get("enable_thinking", False)
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner_cls = KwargReasoner
manager.tokenizer = Mock()
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_parser_kwargs = {
"chat_template_kwargs": {"enable_thinking": True}
}
result = manager.should_fill_bitmask(mock_request_with_structured_output)
assert result is False
assert (
mock_request_with_structured_output.structured_output_request.reasoner
is not None
)
def test_should_advance_with_enable_in_reasoning(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Test should_advance when enable_in_reasoning is True."""
# Enable enable_in_reasoning
manager_with_reasoner.enable_in_reasoning = True
# Should always return True when enable_in_reasoning is enabled
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
assert result is True
assert (
mock_request_with_structured_output.structured_output_request.reasoner
is None
)
def test_should_advance_reasoning_not_ended(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Test should_advance when reasoning has not ended."""
# Set reasoning as not ended
(
mock_request_with_structured_output.structured_output_request
).reasoning_ended = False
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# Should return False since reasoning hasn't ended
assert result is False
def test_should_advance_reasoning_just_ended(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Test should_advance when reasoning ends in current step."""
# Set reasoning as not ended initially, but ends in this step
(
mock_request_with_structured_output.structured_output_request
).reasoning_ended = False
reasoner = MockReasoner(tokenizer=Mock())
reasoner.is_reasoning_end_streaming.return_value = True
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoner = reasoner
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# The scheduler trims the reasoning prefix before advancing the grammar.
assert (
mock_request_with_structured_output.structured_output_request.reasoning_ended
is True
)
assert result is True
def test_should_advance_reasoning_already_ended(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Test should_advance when reasoning has already ended."""
# Set reasoning as already ended
(
mock_request_with_structured_output.structured_output_request
).reasoning_ended = True
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# Should return True since reasoning has ended
assert result is True
assert (
mock_request_with_structured_output.structured_output_request.reasoner
is None
)
def test_should_advance_uses_new_token_ids_when_provided(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Regression for #43388: when caller passes new_token_ids, the
reasoner sees the exact multi-token delta rather than the
placeholder-derived window.
"""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
end_token_id = 248069
reasoner = MockReasoner(tokenizer=Mock())
# Detection mirrors the real Qwen3 parser: end token in the delta.
reasoner.is_reasoning_end_streaming = Mock(
side_effect=lambda input_ids, delta_ids: end_token_id in list(delta_ids)
)
structured_req.reasoner = reasoner
# Scenario from #43388: async + spec decode K=4, 4 tokens accepted
# but only 1 placeholder remains (some drafts were rejected).
# The placeholder math would yield delta=[271] and miss </think>.
# Passing new_token_ids must override that.
new_token_ids = [9, 198, end_token_id, 271]
mock_request_with_structured_output.all_token_ids = [
1,
2,
3,
4,
5,
] + new_token_ids
mock_request_with_structured_output.num_computed_tokens = 9
mock_request_with_structured_output.num_output_placeholders = 1
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output,
new_token_ids=new_token_ids,
)
# First call to is_reasoning_end_streaming was with the full
# new_token_ids (not the truncated placeholder window).
first_call = reasoner.is_reasoning_end_streaming.call_args_list[0]
_, called_delta = first_call.args
assert list(called_delta) == new_token_ids
assert structured_req.reasoning_ended is True
assert result is True
def test_should_advance_without_new_token_ids_falls_back(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Backward compat: callers that don't pass new_token_ids keep
the original placeholder-derived delta window.
"""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
reasoner = MockReasoner(tokenizer=Mock())
reasoner.is_reasoning_end_streaming.return_value = False
structured_req.reasoner = reasoner
mock_request_with_structured_output.all_token_ids = [1, 2, 3, 4, 5]
mock_request_with_structured_output.num_computed_tokens = 5
mock_request_with_structured_output.num_output_placeholders = 2
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# placeholder window: start = 5 - 2 = 3, delta = [4, 5]
_, called_delta = reasoner.is_reasoning_end_streaming.call_args[0]
assert list(called_delta) == [4, 5]
assert result is False
def test_should_advance_whole_delta_fallback_pins_last_sequence_index(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""When only the whole-delta predicate fires, the boundary is the
last index of the sequence, even if the placeholder window
overshoots it and the delta is empty.
"""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
reasoner = MockReasoner(tokenizer=Mock())
# Base-default shape: rescans the whole sequence, ignores the delta.
reasoner.is_reasoning_end_streaming.return_value = True
structured_req.reasoner = reasoner
mock_request_with_structured_output.all_token_ids = [1, 2, 3, 4, 5]
# placeholder window: start = 9 - 1 = 8, past the end -> delta = []
mock_request_with_structured_output.num_computed_tokens = 9
mock_request_with_structured_output.num_output_placeholders = 1
assert manager_with_reasoner.should_advance(mock_request_with_structured_output)
assert structured_req.reasoning_ended is True
assert structured_req.reasoning_end_token_index == 4
def test_should_advance_trims_reasoning_prefix_for_json(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""JSON uses the common trim-then-advance path at the boundary."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.structured_output_key = (
StructuredOutputOptions.JSON_OBJECT,
"{}",
)
marker = 248069
class MarkerReasoner:
def __init__(self, *_, **__):
pass
def is_reasoning_end_streaming(self, input_ids, delta_ids):
return marker in list(delta_ids)
structured_req.reasoner = MarkerReasoner()
new_token_ids = [9, 198, marker, 271, 5005]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output,
new_token_ids=new_token_ids,
)
structured_req.grammar.accept_tokens.assert_not_called()
assert structured_req.reasoning_ended is True
assert result is True
assert structured_req.reasoning_end_token_index == 5
assert manager_with_reasoner.trim_reasoning_for_advance(
mock_request_with_structured_output, new_token_ids
) == [271, 5005]
def test_should_advance_engine_adapter_scans_only_the_delta(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Engine-based parsers locate the boundary from the delta alone."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
reasoner = make_qwen3_reasoner()
assert reasoner.reasoning_end_token_ids == {201, 202}
reasoner.is_reasoning_end_streaming = Mock(
side_effect=AssertionError("engine path must not rescan the prefix")
)
structured_req.reasoner = reasoner
new_token_ids = [9, 198, 201, 271]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
assert manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_ended is True
assert structured_req.reasoning_end_token_index == 5
assert manager_with_reasoner.trim_reasoning_for_advance(
mock_request_with_structured_output, new_token_ids
) == [271]
def test_should_advance_engine_adapter_tool_start_ends_reasoning(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
new_token_ids = [9, 202, 271]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
assert manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_end_token_index == 4
def test_should_advance_engine_adapter_no_boundary(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
new_token_ids = [9, 198, 271]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
assert not manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_ended is False
def test_should_advance_engine_adapter_never_reads_history(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Per-step cost must not depend on how much has been generated."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
new_token_ids = [9, 201]
total = (1 << 20) + len(new_token_ids)
mock_request_with_structured_output.all_token_ids = UntouchableTokenIds(total)
assert manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_end_token_index == total - 1
def test_should_advance_engine_adapter_without_new_tokens_persists(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Draft-validation callers persist the boundary like the legacy path."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
mock_request_with_structured_output.all_token_ids = [1, 2, 3, 4, 5, 201, 7]
# placeholder window: start = 7 - 2 = 5, delta = [201, 7]
mock_request_with_structured_output.num_computed_tokens = 7
mock_request_with_structured_output.num_output_placeholders = 2
assert manager_with_reasoner.should_advance(mock_request_with_structured_output)
assert structured_req.reasoning_ended is True
assert structured_req.reasoning_end_token_index == 5
@@ -0,0 +1,414 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from dataclasses import dataclass
import pytest
from transformers import AutoTokenizer
from vllm.config import DeviceConfig, StructuredOutputsConfig, VllmConfig
from vllm.config.model import ModelConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
TOKENIZER = "gpt2"
THINK_END = "\n" # reasoning-end marker (single GPT-2 token)
EOS = "<|eos|>" # resolved to tokenizer.eos_token_id
JSON_SCHEMA = '{"type": "object"}'
BACKENDS = ("xgrammar", "guidance")
MAX_WAIT_SECONDS = 5
NUM_SPEC_TOKENS = 8
@pytest.fixture(scope="module")
def tokenizer():
return AutoTokenizer.from_pretrained(TOKENIZER)
@dataclass(frozen=True)
class FlowCase:
"""One validate -> bitmask -> accept -> post-accept bitmask scenario."""
raw_drafts: tuple[str, ...]
expected_row_pattern: str
expected_reasoning: bool | None
expect_terminated: bool
prefix: str = ""
expected_validated: tuple[str, ...] | None = None
reasoning_ended: bool | None = False
xfail_guidance: str | None = None
class MockReasoner:
def __init__(self, tokenizer, marker: int | None = None):
self.marker = marker
def is_reasoning_end(self, input_ids):
if self.marker is None:
return True
return self.marker in list(input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids):
return self.is_reasoning_end(delta_ids)
def _single_token(tokenizer, text: str) -> int:
token_ids = tokenizer.encode(text)
assert len(token_ids) == 1, (text, token_ids)
return token_ids[0]
def _to_token_ids(tokenizer, texts: tuple[str, ...]) -> list[int]:
"""Convert literal text (or EOS sentinel) to token IDs."""
return [
tokenizer.eos_token_id if t == EOS else _single_token(tokenizer, t)
for t in texts
]
def _wait_for_grammar(request: Request) -> None:
structured_req = request.structured_output_request
assert structured_req is not None
deadline = time.time() + MAX_WAIT_SECONDS
while not structured_req._check_grammar_completion():
if time.time() > deadline:
pytest.fail("Grammar compilation timed out")
time.sleep(0.01)
def _build_harness(
tokenizer,
backend: str,
prefix: str = "",
use_reasoner: bool = True,
reasoning_ended: bool | None = None,
enable_in_reasoning: bool = False,
reasoning_parser_kwargs: dict | None = None,
) -> tuple[StructuredOutputManager, Request]:
vllm_config = VllmConfig(
model_config=ModelConfig(tokenizer=TOKENIZER),
device_config=DeviceConfig(device="cpu"),
structured_outputs_config=StructuredOutputsConfig(
backend=backend,
enable_in_reasoning=enable_in_reasoning,
),
speculative_config=SpeculativeConfig(
model="[ngram]",
num_speculative_tokens=NUM_SPEC_TOKENS,
),
)
manager = StructuredOutputManager(vllm_config)
if use_reasoner:
manager.reasoner_cls = MockReasoner
sampling_params = SamplingParams(
structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA)
)
sampling_params.structured_outputs._backend = backend
sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
prompt_ids = tokenizer.encode(prefix) if prefix else []
request = Request(
request_id=f"{backend}-flow",
prompt_token_ids=prompt_ids,
sampling_params=sampling_params,
pooling_params=None,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs=reasoning_parser_kwargs,
)
manager.grammar_init(request)
_wait_for_grammar(request)
structured_req = request.structured_output_request
assert request.prompt_token_ids is not None
assert structured_req is not None
assert structured_req.grammar is not None
if prompt_ids:
assert structured_req.grammar.accept_tokens(
request.request_id,
prompt_ids,
)
return manager, request
def _row_pattern(bitmask) -> str:
return "".join("U" if (row == -1).all() else "C" for row in bitmask)
def _run_real_flow(
manager: StructuredOutputManager,
request: Request,
raw_drafts: list[int],
expected_validated: list[int],
expected_row_pattern: str,
expected_reasoning: bool | None,
expect_terminated: bool,
) -> None:
assert len(expected_validated) <= len(raw_drafts)
validated = manager.validate_tokens(request, list(raw_drafts))
assert validated == expected_validated
padded = validated + [-1] * (len(raw_drafts) - len(validated))
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: padded},
)
assert bitmask is not None
assert bitmask.shape[0] == len(raw_drafts) + 1
assert _row_pattern(bitmask) == expected_row_pattern
# grammar_bitmask() must rollback any speculative state it advanced.
assert manager.validate_tokens(request, list(raw_drafts)) == expected_validated
structured_req = request.structured_output_request
assert structured_req is not None
grammar = structured_req.grammar
assert grammar is not None
assert not grammar.is_terminated()
# `bitmask[i]` is the grammar state after the first `i` scheduled tokens.
# These tests commit the validated prefix, so the matching post-accept
# state is already present in the scheduled bitmask.
expected_post_accept_row = bitmask[len(expected_validated)].copy()
# Mirror the scheduler flow: sampled tokens are appended before
# StructuredOutputManager.accept_tokens() trims them to the
# grammar-constrained suffix.
request.append_output_token_ids(list(expected_validated))
assert manager.accept_tokens(request, list(expected_validated))
post_accept_bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={},
)
assert post_accept_bitmask is not None
assert post_accept_bitmask.shape[0] == 1
assert (post_accept_bitmask[0] == expected_post_accept_row).all()
assert structured_req.reasoning_ended == expected_reasoning
assert grammar.is_terminated() is expect_terminated
FLOW_CASES = [
pytest.param(
FlowCase(
raw_drafts=(" ", "z", " "),
expected_row_pattern="UUUU",
expected_reasoning=False,
expect_terminated=False,
),
id="inactive",
),
pytest.param(
FlowCase(
prefix='{"a"',
raw_drafts=(":", ' "', "b", '"}'),
expected_row_pattern="CCCCC",
expected_reasoning=True,
expect_terminated=False,
reasoning_ended=True,
),
id="active_all_valid",
),
pytest.param(
FlowCase(
prefix='{"a"',
raw_drafts=("z", ":", ' "', "b"),
expected_validated=(),
expected_row_pattern="CUUUU",
expected_reasoning=True,
expect_terminated=False,
reasoning_ended=True,
),
id="active_first_token_invalid",
),
pytest.param(
FlowCase(
prefix='{"a"',
raw_drafts=(":", "z", ' "', "b"),
expected_validated=(":",),
expected_row_pattern="CCUUU",
expected_reasoning=True,
expect_terminated=False,
reasoning_ended=True,
),
id="active_later_token_invalid",
),
pytest.param(
FlowCase(
prefix='{"a": "b"}',
raw_drafts=(EOS, " ", " "),
expected_validated=(EOS,),
expected_row_pattern="CUUU",
expected_reasoning=True,
expect_terminated=True,
reasoning_ended=True,
xfail_guidance=(
"guidance validate_tokens rejects EOS on an already-complete "
"object even though accept_tokens would accept it"
),
),
id="active_first_token_terminates",
),
pytest.param(
FlowCase(
prefix='{"a"',
raw_drafts=(":", ' "', "b", '"}', EOS, " "),
expected_validated=(":", ' "', "b", '"}', EOS),
expected_row_pattern="CCCCCUU",
expected_reasoning=True,
expect_terminated=True,
reasoning_ended=True,
),
id="active_later_token_terminates",
),
pytest.param(
FlowCase(
raw_drafts=(THINK_END, "{", ' "', "b"),
expected_row_pattern="UCCCC",
expected_reasoning=True,
expect_terminated=False,
),
id="becomes_active_on_first_token",
),
pytest.param(
FlowCase(
raw_drafts=(" ", THINK_END, "{", ' "'),
expected_row_pattern="UUCCC",
expected_reasoning=True,
expect_terminated=False,
),
id="becomes_active_on_middle_token",
),
pytest.param(
FlowCase(
raw_drafts=(" ", " ", THINK_END),
expected_row_pattern="UUUC",
expected_reasoning=True,
expect_terminated=False,
),
id="becomes_active_on_last_token",
),
pytest.param(
FlowCase(
raw_drafts=(" ", THINK_END, "z", "{"),
expected_validated=(" ", THINK_END),
expected_row_pattern="UUCUU",
expected_reasoning=True,
expect_terminated=False,
),
id="becomes_active_first_token_invalid",
),
pytest.param(
FlowCase(
raw_drafts=(" ", THINK_END, "{", "z"),
expected_validated=(" ", THINK_END, "{"),
expected_row_pattern="UUCCU",
expected_reasoning=True,
expect_terminated=False,
),
id="becomes_active_later_token_invalid",
),
pytest.param(
FlowCase(
raw_drafts=(" ", THINK_END, "{", "}", EOS, " "),
expected_validated=(" ", THINK_END, "{", "}", EOS),
expected_row_pattern="UUCCCUU",
expected_reasoning=True,
expect_terminated=True,
),
id="becomes_active_terminates",
),
]
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize("case", FLOW_CASES)
def test_real_flow(
tokenizer,
backend: str,
case: FlowCase,
):
if backend == "guidance" and case.xfail_guidance:
pytest.xfail(case.xfail_guidance)
reasoner_kwargs = {"marker": _single_token(tokenizer, THINK_END)}
manager, request = _build_harness(
tokenizer,
backend,
prefix=case.prefix,
reasoning_ended=case.reasoning_ended,
reasoning_parser_kwargs=reasoner_kwargs,
)
raw_drafts = _to_token_ids(tokenizer, case.raw_drafts)
expected_texts = (
case.raw_drafts if case.expected_validated is None else case.expected_validated
)
expected_validated = _to_token_ids(tokenizer, expected_texts)
_run_real_flow(
manager,
request,
raw_drafts=raw_drafts,
expected_validated=expected_validated,
expected_row_pattern=case.expected_row_pattern,
expected_reasoning=case.expected_reasoning,
expect_terminated=case.expect_terminated,
)
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize(
("use_reasoner", "reasoning_ended", "enable_in_reasoning"),
[
pytest.param(True, False, True, id="enable_in_reasoning"),
pytest.param(True, True, False, id="reasoning_ended"),
pytest.param(False, False, False, id="no_reasoner"),
pytest.param(True, None, False, id="inferred"),
],
)
def test_initial_constraint_activation(
tokenizer,
backend: str,
use_reasoner: bool,
reasoning_ended: bool | None,
enable_in_reasoning: bool,
):
manager, request = _build_harness(
tokenizer,
backend,
use_reasoner=use_reasoner,
reasoning_ended=reasoning_ended,
enable_in_reasoning=enable_in_reasoning,
reasoning_parser_kwargs={
"marker": _single_token(tokenizer, THINK_END)
if reasoning_ended is not None
else None
},
)
# "{" is valid JSON start; "z" is not. Truncation proves grammar is active.
open_brace = _single_token(tokenizer, "{")
z = _single_token(tokenizer, "z")
assert manager.validate_tokens(request, [open_brace, z]) == [open_brace]
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={},
)
assert bitmask is not None
request.append_output_token_ids([open_brace])
assert manager.accept_tokens(request, [open_brace])
# Short-circuit paths must not construct a request-local reasoner (#52573).
if enable_in_reasoning or reasoning_ended is True:
structured_req = request.structured_output_request
assert structured_req is not None
assert structured_req.reasoner is None