[Refactor] StructuredOutputManager x Speculative Decoding Refactor (#48200)

This commit is contained in:
yzong-rh
2026-09-15 09:48:49 -04:00
committed by GitHub
parent 64856080b0
commit e6960af33b
12 changed files with 596 additions and 1192 deletions
@@ -1,557 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for reasoning-aware structured output functionality (PR #25515)."""
from collections.abc import Sequence
from unittest.mock import Mock
import pytest
from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
class UntouchableTokenIds(Sequence[int]):
"""A token history that fails the test if anything reads it."""
def __init__(self, length: int) -> None:
self._length = length
def __len__(self) -> int:
return self._length
def __getitem__(self, item):
raise AssertionError("token history must not be read")
def __iter__(self):
raise AssertionError("token history must not be iterated")
def make_qwen3_reasoner():
"""A real engine-based reasoning adapter over a tiny mock vocabulary."""
from tests.parser.engine.conftest import make_mock_tokenizer
from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter
tokenizer = make_mock_tokenizer(
{"<think>": 200, "</think>": 201, "<tool_call>": 202, "</tool_call>": 203}
)
return Qwen3ParserReasoningAdapter(tokenizer)
class MockReasoner:
def __init__(self, tokenizer):
self.is_reasoning_end = Mock(return_value=False)
self.is_reasoning_end_streaming = Mock(return_value=False)
class TestReasoningStructuredOutput:
"""Test reasoning-aware structured output functionality."""
@pytest.fixture
def mock_model_config(self):
"""Create a mock ModelConfig."""
config = Mock(spec=ModelConfig)
config.skip_tokenizer_init = True # Skip tokenizer init to avoid network calls
config.get_vocab_size = Mock(return_value=50000)
# Add missing runner_type attribute that tokenizer initialization expects
config.runner_type = "generate"
# Add other attributes that tokenizer initialization might need
config.tokenizer = "test-tokenizer"
config.tokenizer_mode = "auto"
config.trust_remote_code = False
config.tokenizer_revision = None
return config
@pytest.fixture
def mock_scheduler_config(self):
"""Create a mock SchedulerConfig."""
config = Mock(spec=SchedulerConfig)
config.max_num_seqs = 128
return config
@pytest.fixture
def mock_vllm_config(self, mock_model_config, mock_scheduler_config):
"""Create a mock VllmConfig."""
config = Mock(spec=VllmConfig)
config.model_config = mock_model_config
config.scheduler_config = mock_scheduler_config
config.structured_outputs_config = Mock()
config.structured_outputs_config.reasoning_parser = None
config.structured_outputs_config.enable_in_reasoning = False
config.speculative_config = None
return config
@pytest.fixture
def mock_request_with_structured_output(self):
"""Create a mock request with structured output."""
request = Mock(spec=Request)
request.structured_output_request = Mock()
request.structured_output_request.reasoning_ended = None
request.structured_output_request.grammar = Mock()
request.structured_output_request.reasoning_parser_kwargs = None
request.structured_output_request.reasoner = None
request.structured_output_request.grammar.is_terminated = Mock(
return_value=False
)
request.use_structured_output = True
request.prompt_token_ids = [1, 2, 3, 4, 5]
request.all_token_ids = [1, 2, 3, 4, 5, 6, 7, 8]
request.num_computed_tokens = 5
request.num_output_placeholders = 0
request.request_id = "mock_req"
return request
@pytest.fixture
def manager_with_reasoner(self, mock_vllm_config):
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner_cls = MockReasoner
manager.tokenizer = Mock()
return manager
def test_should_fill_bitmask_with_enable_in_reasoning(
self, manager_with_reasoner, mock_request_with_structured_output
):
"""Test should_fill_bitmask when enable_in_reasoning is True."""
manager_with_reasoner.enable_in_reasoning = True
# Should always return True when enable_in_reasoning is enabled
result = manager_with_reasoner.should_fill_bitmask(
mock_request_with_structured_output
)
assert result is True
assert (
mock_request_with_structured_output.structured_output_request.reasoner
is None
)
def test_should_fill_bitmask_reasoning_already_ended(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""An active grammar does not need a request-local reasoner."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = True
result = manager_with_reasoner.should_fill_bitmask(
mock_request_with_structured_output
)
assert result is True
assert structured_req.reasoner is None
def test_grammar_bitmask_skips_reasoner_when_already_active(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Bitmask generation skips reasoning-boundary detection."""
manager_with_reasoner.vllm_config.num_speculative_tokens = 1
manager_with_reasoner._grammar_bitmask = Mock()
manager_with_reasoner._grammar_bitmask.shape = (1, 1)
expected_bitmask = Mock()
manager_with_reasoner._grammar_bitmask.numpy.return_value = expected_bitmask
manager_with_reasoner._fill_bitmasks = Mock()
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = True
request_id = mock_request_with_structured_output.request_id
bitmask = manager_with_reasoner.grammar_bitmask(
requests={request_id: mock_request_with_structured_output},
structured_output_request_ids=[request_id],
scheduled_spec_decode_tokens={},
)
assert bitmask is expected_bitmask
assert structured_req.reasoner is None
def test_should_fill_bitmask_without_enable_in_reasoning(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Test should_fill_bitmask when enable_in_reasoning is False."""
# Keep enable_in_reasoning as False (default)
config = manager_with_reasoner.vllm_config.structured_outputs_config
assert config.enable_in_reasoning is False
result = manager_with_reasoner.should_fill_bitmask(
mock_request_with_structured_output
)
# Should set reasoning_ended and return its value
assert (
mock_request_with_structured_output.structured_output_request.reasoning_ended
is False
)
assert result is False
def test_should_fill_bitmask_no_reasoner(
self, mock_vllm_config, mock_request_with_structured_output
):
"""Test should_fill_bitmask when no reasoner is configured."""
manager = StructuredOutputManager(mock_vllm_config)
result = manager.should_fill_bitmask(mock_request_with_structured_output)
# Should default to True when no reasoner
assert result is True
def test_should_fill_bitmask_uses_request_reasoning_parser_kwargs(
self, mock_vllm_config, mock_request_with_structured_output
):
"""Test request-level parser kwargs override the default reasoner."""
class KwargReasoner:
def __init__(self, tokenizer, chat_template_kwargs=None):
self.chat_template_kwargs = chat_template_kwargs or {}
def is_reasoning_end(self, input_ids):
return not self.chat_template_kwargs.get("enable_thinking", False)
manager = StructuredOutputManager(mock_vllm_config)
manager.reasoner_cls = KwargReasoner
manager.tokenizer = Mock()
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_parser_kwargs = {
"chat_template_kwargs": {"enable_thinking": True}
}
result = manager.should_fill_bitmask(mock_request_with_structured_output)
assert result is False
assert (
mock_request_with_structured_output.structured_output_request.reasoner
is not None
)
def test_should_advance_with_enable_in_reasoning(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Test should_advance when enable_in_reasoning is True."""
# Enable enable_in_reasoning
manager_with_reasoner.enable_in_reasoning = True
# Should always return True when enable_in_reasoning is enabled
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
assert result is True
assert (
mock_request_with_structured_output.structured_output_request.reasoner
is None
)
def test_should_advance_reasoning_not_ended(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Test should_advance when reasoning has not ended."""
# Set reasoning as not ended
(
mock_request_with_structured_output.structured_output_request
).reasoning_ended = False
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# Should return False since reasoning hasn't ended
assert result is False
def test_should_advance_reasoning_just_ended(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Test should_advance when reasoning ends in current step."""
# Set reasoning as not ended initially, but ends in this step
(
mock_request_with_structured_output.structured_output_request
).reasoning_ended = False
reasoner = MockReasoner(tokenizer=Mock())
reasoner.is_reasoning_end_streaming.return_value = True
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoner = reasoner
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# The scheduler trims the reasoning prefix before advancing the grammar.
assert (
mock_request_with_structured_output.structured_output_request.reasoning_ended
is True
)
assert result is True
def test_should_advance_reasoning_already_ended(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Test should_advance when reasoning has already ended."""
# Set reasoning as already ended
(
mock_request_with_structured_output.structured_output_request
).reasoning_ended = True
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# Should return True since reasoning has ended
assert result is True
assert (
mock_request_with_structured_output.structured_output_request.reasoner
is None
)
def test_should_advance_uses_new_token_ids_when_provided(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Regression for #43388: when caller passes new_token_ids, the
reasoner sees the exact multi-token delta rather than the
placeholder-derived window.
"""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
end_token_id = 248069
reasoner = MockReasoner(tokenizer=Mock())
# Detection mirrors the real Qwen3 parser: end token in the delta.
reasoner.is_reasoning_end_streaming = Mock(
side_effect=lambda input_ids, delta_ids: end_token_id in list(delta_ids)
)
structured_req.reasoner = reasoner
# Scenario from #43388: async + spec decode K=4, 4 tokens accepted
# but only 1 placeholder remains (some drafts were rejected).
# The placeholder math would yield delta=[271] and miss </think>.
# Passing new_token_ids must override that.
new_token_ids = [9, 198, end_token_id, 271]
mock_request_with_structured_output.all_token_ids = [
1,
2,
3,
4,
5,
] + new_token_ids
mock_request_with_structured_output.num_computed_tokens = 9
mock_request_with_structured_output.num_output_placeholders = 1
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output,
new_token_ids=new_token_ids,
)
# First call to is_reasoning_end_streaming was with the full
# new_token_ids (not the truncated placeholder window).
first_call = reasoner.is_reasoning_end_streaming.call_args_list[0]
_, called_delta = first_call.args
assert list(called_delta) == new_token_ids
assert structured_req.reasoning_ended is True
assert result is True
def test_should_advance_without_new_token_ids_falls_back(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Backward compat: callers that don't pass new_token_ids keep
the original placeholder-derived delta window.
"""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
reasoner = MockReasoner(tokenizer=Mock())
reasoner.is_reasoning_end_streaming.return_value = False
structured_req.reasoner = reasoner
mock_request_with_structured_output.all_token_ids = [1, 2, 3, 4, 5]
mock_request_with_structured_output.num_computed_tokens = 5
mock_request_with_structured_output.num_output_placeholders = 2
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output
)
# placeholder window: start = 5 - 2 = 3, delta = [4, 5]
_, called_delta = reasoner.is_reasoning_end_streaming.call_args[0]
assert list(called_delta) == [4, 5]
assert result is False
def test_should_advance_whole_delta_fallback_pins_last_sequence_index(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""When only the whole-delta predicate fires, the boundary is the
last index of the sequence, even if the placeholder window
overshoots it and the delta is empty.
"""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
reasoner = MockReasoner(tokenizer=Mock())
# Base-default shape: rescans the whole sequence, ignores the delta.
reasoner.is_reasoning_end_streaming.return_value = True
structured_req.reasoner = reasoner
mock_request_with_structured_output.all_token_ids = [1, 2, 3, 4, 5]
# placeholder window: start = 9 - 1 = 8, past the end -> delta = []
mock_request_with_structured_output.num_computed_tokens = 9
mock_request_with_structured_output.num_output_placeholders = 1
assert manager_with_reasoner.should_advance(mock_request_with_structured_output)
assert structured_req.reasoning_ended is True
assert structured_req.reasoning_end_token_index == 4
def test_should_advance_trims_reasoning_prefix_for_json(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""JSON uses the common trim-then-advance path at the boundary."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.structured_output_key = (
StructuredOutputOptions.JSON_OBJECT,
"{}",
)
marker = 248069
class MarkerReasoner:
def __init__(self, *_, **__):
pass
def is_reasoning_end_streaming(self, input_ids, delta_ids):
return marker in list(delta_ids)
structured_req.reasoner = MarkerReasoner()
new_token_ids = [9, 198, marker, 271, 5005]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
result = manager_with_reasoner.should_advance(
mock_request_with_structured_output,
new_token_ids=new_token_ids,
)
structured_req.grammar.accept_tokens.assert_not_called()
assert structured_req.reasoning_ended is True
assert result is True
assert structured_req.reasoning_end_token_index == 5
assert manager_with_reasoner.trim_reasoning_for_advance(
mock_request_with_structured_output, new_token_ids
) == [271, 5005]
def test_should_advance_engine_adapter_scans_only_the_delta(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Engine-based parsers locate the boundary from the delta alone."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
reasoner = make_qwen3_reasoner()
assert reasoner.reasoning_end_token_ids == {201, 202}
reasoner.is_reasoning_end_streaming = Mock(
side_effect=AssertionError("engine path must not rescan the prefix")
)
structured_req.reasoner = reasoner
new_token_ids = [9, 198, 201, 271]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
assert manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_ended is True
assert structured_req.reasoning_end_token_index == 5
assert manager_with_reasoner.trim_reasoning_for_advance(
mock_request_with_structured_output, new_token_ids
) == [271]
def test_should_advance_engine_adapter_tool_start_ends_reasoning(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
new_token_ids = [9, 202, 271]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
assert manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_end_token_index == 4
def test_should_advance_engine_adapter_no_boundary(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
new_token_ids = [9, 198, 271]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
assert not manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_ended is False
def test_should_advance_engine_adapter_never_reads_history(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Per-step cost must not depend on how much has been generated."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
new_token_ids = [9, 201]
total = (1 << 20) + len(new_token_ids)
mock_request_with_structured_output.all_token_ids = UntouchableTokenIds(total)
assert manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_end_token_index == total - 1
def test_should_advance_engine_adapter_without_new_tokens_persists(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Draft-validation callers persist the boundary like the legacy path."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
mock_request_with_structured_output.all_token_ids = [1, 2, 3, 4, 5, 201, 7]
# placeholder window: start = 7 - 2 = 5, delta = [201, 7]
mock_request_with_structured_output.num_computed_tokens = 7
mock_request_with_structured_output.num_output_placeholders = 2
assert manager_with_reasoner.should_advance(mock_request_with_structured_output)
assert structured_req.reasoning_ended is True
assert structured_req.reasoning_end_token_index == 5
@@ -0,0 +1,414 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from dataclasses import dataclass
import pytest
from transformers import AutoTokenizer
from vllm.config import DeviceConfig, StructuredOutputsConfig, VllmConfig
from vllm.config.model import ModelConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
TOKENIZER = "gpt2"
THINK_END = "\n" # reasoning-end marker (single GPT-2 token)
EOS = "<|eos|>" # resolved to tokenizer.eos_token_id
JSON_SCHEMA = '{"type": "object"}'
BACKENDS = ("xgrammar", "guidance")
MAX_WAIT_SECONDS = 5
NUM_SPEC_TOKENS = 8
@pytest.fixture(scope="module")
def tokenizer():
return AutoTokenizer.from_pretrained(TOKENIZER)
@dataclass(frozen=True)
class FlowCase:
"""One validate -> bitmask -> accept -> post-accept bitmask scenario."""
raw_drafts: tuple[str, ...]
expected_row_pattern: str
expected_reasoning: bool | None
expect_terminated: bool
prefix: str = ""
expected_validated: tuple[str, ...] | None = None
reasoning_ended: bool | None = False
xfail_guidance: str | None = None
class MockReasoner:
def __init__(self, tokenizer, marker: int | None = None):
self.marker = marker
def is_reasoning_end(self, input_ids):
if self.marker is None:
return True
return self.marker in list(input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids):
return self.is_reasoning_end(delta_ids)
def _single_token(tokenizer, text: str) -> int:
token_ids = tokenizer.encode(text)
assert len(token_ids) == 1, (text, token_ids)
return token_ids[0]
def _to_token_ids(tokenizer, texts: tuple[str, ...]) -> list[int]:
"""Convert literal text (or EOS sentinel) to token IDs."""
return [
tokenizer.eos_token_id if t == EOS else _single_token(tokenizer, t)
for t in texts
]
def _wait_for_grammar(request: Request) -> None:
structured_req = request.structured_output_request
assert structured_req is not None
deadline = time.time() + MAX_WAIT_SECONDS
while not structured_req._check_grammar_completion():
if time.time() > deadline:
pytest.fail("Grammar compilation timed out")
time.sleep(0.01)
def _build_harness(
tokenizer,
backend: str,
prefix: str = "",
use_reasoner: bool = True,
reasoning_ended: bool | None = None,
enable_in_reasoning: bool = False,
reasoning_parser_kwargs: dict | None = None,
) -> tuple[StructuredOutputManager, Request]:
vllm_config = VllmConfig(
model_config=ModelConfig(tokenizer=TOKENIZER),
device_config=DeviceConfig(device="cpu"),
structured_outputs_config=StructuredOutputsConfig(
backend=backend,
enable_in_reasoning=enable_in_reasoning,
),
speculative_config=SpeculativeConfig(
model="[ngram]",
num_speculative_tokens=NUM_SPEC_TOKENS,
),
)
manager = StructuredOutputManager(vllm_config)
if use_reasoner:
manager.reasoner_cls = MockReasoner
sampling_params = SamplingParams(
structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA)
)
sampling_params.structured_outputs._backend = backend
sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)
prompt_ids = tokenizer.encode(prefix) if prefix else []
request = Request(
request_id=f"{backend}-flow",
prompt_token_ids=prompt_ids,
sampling_params=sampling_params,
pooling_params=None,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs=reasoning_parser_kwargs,
)
manager.grammar_init(request)
_wait_for_grammar(request)
structured_req = request.structured_output_request
assert request.prompt_token_ids is not None
assert structured_req is not None
assert structured_req.grammar is not None
if prompt_ids:
assert structured_req.grammar.accept_tokens(
request.request_id,
prompt_ids,
)
return manager, request
def _row_pattern(bitmask) -> str:
return "".join("U" if (row == -1).all() else "C" for row in bitmask)
def _run_real_flow(
manager: StructuredOutputManager,
request: Request,
raw_drafts: list[int],
expected_validated: list[int],
expected_row_pattern: str,
expected_reasoning: bool | None,
expect_terminated: bool,
) -> None:
assert len(expected_validated) <= len(raw_drafts)
validated = manager.validate_tokens(request, list(raw_drafts))
assert validated == expected_validated
padded = validated + [-1] * (len(raw_drafts) - len(validated))
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: padded},
)
assert bitmask is not None
assert bitmask.shape[0] == len(raw_drafts) + 1
assert _row_pattern(bitmask) == expected_row_pattern
# grammar_bitmask() must rollback any speculative state it advanced.
assert manager.validate_tokens(request, list(raw_drafts)) == expected_validated
structured_req = request.structured_output_request
assert structured_req is not None
grammar = structured_req.grammar
assert grammar is not None
assert not grammar.is_terminated()
# `bitmask[i]` is the grammar state after the first `i` scheduled tokens.
# These tests commit the validated prefix, so the matching post-accept
# state is already present in the scheduled bitmask.
expected_post_accept_row = bitmask[len(expected_validated)].copy()
# Mirror the scheduler flow: sampled tokens are appended before
# StructuredOutputManager.accept_tokens() trims them to the
# grammar-constrained suffix.
request.append_output_token_ids(list(expected_validated))
assert manager.accept_tokens(request, list(expected_validated))
post_accept_bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={},
)
assert post_accept_bitmask is not None
assert post_accept_bitmask.shape[0] == 1
assert (post_accept_bitmask[0] == expected_post_accept_row).all()
assert structured_req.reasoning_ended == expected_reasoning
assert grammar.is_terminated() is expect_terminated
FLOW_CASES = [
pytest.param(
FlowCase(
raw_drafts=(" ", "z", " "),
expected_row_pattern="UUUU",
expected_reasoning=False,
expect_terminated=False,
),
id="inactive",
),
pytest.param(
FlowCase(
prefix='{"a"',
raw_drafts=(":", ' "', "b", '"}'),
expected_row_pattern="CCCCC",
expected_reasoning=True,
expect_terminated=False,
reasoning_ended=True,
),
id="active_all_valid",
),
pytest.param(
FlowCase(
prefix='{"a"',
raw_drafts=("z", ":", ' "', "b"),
expected_validated=(),
expected_row_pattern="CUUUU",
expected_reasoning=True,
expect_terminated=False,
reasoning_ended=True,
),
id="active_first_token_invalid",
),
pytest.param(
FlowCase(
prefix='{"a"',
raw_drafts=(":", "z", ' "', "b"),
expected_validated=(":",),
expected_row_pattern="CCUUU",
expected_reasoning=True,
expect_terminated=False,
reasoning_ended=True,
),
id="active_later_token_invalid",
),
pytest.param(
FlowCase(
prefix='{"a": "b"}',
raw_drafts=(EOS, " ", " "),
expected_validated=(EOS,),
expected_row_pattern="CUUU",
expected_reasoning=True,
expect_terminated=True,
reasoning_ended=True,
xfail_guidance=(
"guidance validate_tokens rejects EOS on an already-complete "
"object even though accept_tokens would accept it"
),
),
id="active_first_token_terminates",
),
pytest.param(
FlowCase(
prefix='{"a"',
raw_drafts=(":", ' "', "b", '"}', EOS, " "),
expected_validated=(":", ' "', "b", '"}', EOS),
expected_row_pattern="CCCCCUU",
expected_reasoning=True,
expect_terminated=True,
reasoning_ended=True,
),
id="active_later_token_terminates",
),
pytest.param(
FlowCase(
raw_drafts=(THINK_END, "{", ' "', "b"),
expected_row_pattern="UCCCC",
expected_reasoning=True,
expect_terminated=False,
),
id="becomes_active_on_first_token",
),
pytest.param(
FlowCase(
raw_drafts=(" ", THINK_END, "{", ' "'),
expected_row_pattern="UUCCC",
expected_reasoning=True,
expect_terminated=False,
),
id="becomes_active_on_middle_token",
),
pytest.param(
FlowCase(
raw_drafts=(" ", " ", THINK_END),
expected_row_pattern="UUUC",
expected_reasoning=True,
expect_terminated=False,
),
id="becomes_active_on_last_token",
),
pytest.param(
FlowCase(
raw_drafts=(" ", THINK_END, "z", "{"),
expected_validated=(" ", THINK_END),
expected_row_pattern="UUCUU",
expected_reasoning=True,
expect_terminated=False,
),
id="becomes_active_first_token_invalid",
),
pytest.param(
FlowCase(
raw_drafts=(" ", THINK_END, "{", "z"),
expected_validated=(" ", THINK_END, "{"),
expected_row_pattern="UUCCU",
expected_reasoning=True,
expect_terminated=False,
),
id="becomes_active_later_token_invalid",
),
pytest.param(
FlowCase(
raw_drafts=(" ", THINK_END, "{", "}", EOS, " "),
expected_validated=(" ", THINK_END, "{", "}", EOS),
expected_row_pattern="UUCCCUU",
expected_reasoning=True,
expect_terminated=True,
),
id="becomes_active_terminates",
),
]
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize("case", FLOW_CASES)
def test_real_flow(
tokenizer,
backend: str,
case: FlowCase,
):
if backend == "guidance" and case.xfail_guidance:
pytest.xfail(case.xfail_guidance)
reasoner_kwargs = {"marker": _single_token(tokenizer, THINK_END)}
manager, request = _build_harness(
tokenizer,
backend,
prefix=case.prefix,
reasoning_ended=case.reasoning_ended,
reasoning_parser_kwargs=reasoner_kwargs,
)
raw_drafts = _to_token_ids(tokenizer, case.raw_drafts)
expected_texts = (
case.raw_drafts if case.expected_validated is None else case.expected_validated
)
expected_validated = _to_token_ids(tokenizer, expected_texts)
_run_real_flow(
manager,
request,
raw_drafts=raw_drafts,
expected_validated=expected_validated,
expected_row_pattern=case.expected_row_pattern,
expected_reasoning=case.expected_reasoning,
expect_terminated=case.expect_terminated,
)
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize(
("use_reasoner", "reasoning_ended", "enable_in_reasoning"),
[
pytest.param(True, False, True, id="enable_in_reasoning"),
pytest.param(True, True, False, id="reasoning_ended"),
pytest.param(False, False, False, id="no_reasoner"),
pytest.param(True, None, False, id="inferred"),
],
)
def test_initial_constraint_activation(
tokenizer,
backend: str,
use_reasoner: bool,
reasoning_ended: bool | None,
enable_in_reasoning: bool,
):
manager, request = _build_harness(
tokenizer,
backend,
use_reasoner=use_reasoner,
reasoning_ended=reasoning_ended,
enable_in_reasoning=enable_in_reasoning,
reasoning_parser_kwargs={
"marker": _single_token(tokenizer, THINK_END)
if reasoning_ended is not None
else None
},
)
# "{" is valid JSON start; "z" is not. Truncation proves grammar is active.
open_brace = _single_token(tokenizer, "{")
z = _single_token(tokenizer, "z")
assert manager.validate_tokens(request, [open_brace, z]) == [open_brace]
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={},
)
assert bitmask is not None
request.append_output_token_ids([open_brace])
assert manager.accept_tokens(request, [open_brace])
# Short-circuit paths must not construct a request-local reasoner (#52573).
if enable_in_reasoning or reasoning_ended is True:
structured_req = request.structured_output_request
assert structured_req is not None
assert structured_req.reasoner is None