[Perf] Eliminate full-history reasoning scans for structured outputs (#55223)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
Signed-off-by: Flora Feng <4florafeng@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Flora Feng
2026-09-08 14:07:16 -07:00
committed by GitHub
co-authored by Wentao Ye
parent 73f61d12aa
commit 29f46ac475
8 changed files with 475 additions and 42 deletions
+11
View File
@@ -416,6 +416,17 @@ class TestImplicitReasoningEnd:
args = json.loads(tool_calls[0].arguments)
assert args == {"location": "NYC"}
def test_reasoning_end_token_ids_drop_multi_token_tool_opener(
self, thinking_parser
):
# Like the real DeepSeek-V4 tokenizer, the mock vocab has no single
# token for the DSML tool opener, so only </think> is tracked on the
# token-ID fast path; the implicit tool-call exit still goes through
# the text lexer.
assert thinking_parser.reasoning_end_token_ids == {_THINK_END_ID}
assert thinking_parser.find_reasoning_end_offset([7, _THINK_END_ID, 8]) == 1
assert thinking_parser.find_reasoning_end_offset([7, 8]) is None
def test_streaming_reasoning_implicit_end(self, thinking_parser):
chunks = [
"Let me look up the weather.\n\n",
+63
View File
@@ -7,6 +7,7 @@ DeltaMessage / ExtractedToolCallInformation protocol.
from __future__ import annotations
import dataclasses
import json
from types import SimpleNamespace
from unittest.mock import MagicMock
@@ -132,6 +133,68 @@ def _make_engine(
)
# ── TestReasoningEndTokenIds ─────────────────────────────────────────
def _with_reasoning_exits(
*exits: tuple[str, ParserState, tuple[EventType, ...]],
) -> ParserEngineConfig:
"""Combined config plus extra transitions out of REASONING."""
base = _combined_config()
transitions = dict(base.transitions)
for terminal, next_state, events in exits:
transitions[(ParserState.REASONING, terminal)] = Transition(next_state, events)
return dataclasses.replace(base, transitions=transitions)
class TestReasoningEndTokenIds:
"""ParserEngine derives the reasoning-end token set from its config."""
def test_every_reasoning_end_exit_contributes(self):
cfg = _with_reasoning_exits(
(
"TOOL_START",
ParserState.TOOL_ARGS,
(EventType.REASONING_END, EventType.TOOL_CALL_START),
),
)
assert _make_engine(cfg).reasoning_end_token_ids == {201, 202}
def test_transition_staying_in_reasoning_is_ignored(self):
cfg = _with_reasoning_exits(("THINK_START", ParserState.REASONING, ()))
assert _make_engine(cfg).reasoning_end_token_ids == {201}
def test_unreported_exit_fails_closed(self):
cfg = _with_reasoning_exits(
("TOOL_START", ParserState.TOOL_ARGS, (EventType.TOOL_CALL_START,)),
)
assert _make_engine(cfg).reasoning_end_token_ids == frozenset()
def test_unresolved_think_end_fails_closed(self):
vocab = {k: v for k, v in _VOCAB.items() if k != "</think>"}
assert _make_engine(vocab=vocab).reasoning_end_token_ids == frozenset()
text_only = dataclasses.replace(
_combined_config(), token_id_terminals={"THINK_START": "<think>"}
)
assert _make_engine(text_only).reasoning_end_token_ids == frozenset()
def test_config_without_reasoning_has_empty_set(self):
assert _make_engine(_hermes_config()).reasoning_end_token_ids == frozenset()
def test_find_reasoning_end_offset_returns_first_match(self):
engine = _make_engine()
assert engine.find_reasoning_end_offset([5, 201, 6, 201]) == 1
assert engine.find_reasoning_end_offset([5, 6]) is None
assert engine.find_reasoning_end_offset([]) is None
# Rejected-draft placeholders never match.
assert engine.find_reasoning_end_offset([-1, 201]) == 1
def test_find_reasoning_end_offset_with_empty_set_returns_none(self):
engine = _make_engine(_hermes_config())
assert engine.find_reasoning_end_offset([201]) is None
# ── TestEventsToDelta ────────────────────────────────────────────────
@@ -8,6 +8,7 @@ from transformers import AutoTokenizer
from vllm.config import StructuredOutputsConfig, VllmConfig
from vllm.config.model import ModelConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
@@ -390,3 +391,119 @@ def test_trim_reasoning_for_advance():
next_step = [post, post]
request.append_output_token_ids(next_step)
assert manager.trim_reasoning_for_advance(request, next_step) == next_step
class _EngineReasonerStub(ParserEngineReasoningAdapter):
"""Adapter-typed reasoner with a fixed end-token set and no real engine."""
def __init__(self, end_token_ids):
self._end_token_ids = frozenset(end_token_ids)
self.windows: list[list[int]] = []
@property
def reasoning_end_token_ids(self):
return self._end_token_ids
def find_reasoning_end_offset(self, token_ids):
self.windows.append(list(token_ids))
for offset, token in enumerate(token_ids):
if token in self._end_token_ids:
return offset
return None
def is_reasoning_end(self, input_ids):
return any(token in self._end_token_ids for token in input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids):
raise AssertionError("engine path must not rescan draft prefixes")
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bitmask_engine_reasoner_ends_midwindow_with_padding(backend):
"""Engine reasoners see the draft window once, without -1 padding."""
tokenizer, manager, request, prompt = _make_manager_and_request(backend)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
marker = tokenizer.encode("\n")[0]
reasoner = _EngineReasonerStub({marker})
manager.reasoner_cls = _EngineReasonerStub
request.structured_output_request.reasoner = reasoner
request.structured_output_request.reasoning_ended = False
pre = tokenizer.encode(" ")[0]
post = tokenizer.encode(",")[0]
drafts = [pre, marker, post, -1]
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert bitmask.shape[0] == len(drafts) + 1
assert (bitmask[0] == -1).all()
assert (bitmask[1] == -1).all()
assert not (bitmask[2] == -1).all()
assert reasoner.windows == [[pre, marker, post]]
assert not grammar.is_terminated()
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bitmask_legacy_order_sensitive_predicate_probed_per_token(backend):
"""Legacy predicates are probed one draft at a time, never on the whole
window. A predicate that reports which marker is *newest* (KimiK3) would
otherwise answer False for a window that closes and then reopens
reasoning, leaving the post-marker drafts and the bonus row
unconstrained.
"""
tokenizer, manager, request, prompt = _make_manager_and_request(backend)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
close = tokenizer.encode("\n")[0]
reopen = tokenizer.encode("#")[0]
assert close != reopen
class NewestMarkerReasoner:
def __init__(self, *_, **__):
pass
def is_reasoning_end(self, input_ids):
return self.is_reasoning_end_streaming(input_ids, input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids):
for token in reversed(list(delta_ids)):
if token == close:
return True
if token == reopen:
return False
return False
manager.reasoner_cls = NewestMarkerReasoner
request.structured_output_request.reasoner = NewestMarkerReasoner()
request.structured_output_request.reasoning_ended = False
pre = tokenizer.encode(" ")[0]
post = tokenizer.encode(",")[0]
drafts = [pre, close, post, reopen]
# Sanity: the whole window says "reopened", so only per-token probing
# can find the close marker.
assert not NewestMarkerReasoner().is_reasoning_end_streaming([], drafts)
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert bitmask.shape[0] == len(drafts) + 1
assert (bitmask[0] == -1).all()
assert (bitmask[1] == -1).all()
assert not (bitmask[2] == -1).all()
assert not (bitmask[-1] == -1).all()
assert not grammar.is_terminated()
@@ -3,6 +3,7 @@
"""Unit tests for reasoning-aware structured output functionality (PR #25515)."""
from collections.abc import Sequence
from unittest.mock import Mock
import pytest
@@ -13,6 +14,33 @@ from vllm.v1.structured_output import StructuredOutputManager
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
class UntouchableTokenIds(Sequence[int]):
"""A token history that fails the test if anything reads it."""
def __init__(self, length: int) -> None:
self._length = length
def __len__(self) -> int:
return self._length
def __getitem__(self, item):
raise AssertionError("token history must not be read")
def __iter__(self):
raise AssertionError("token history must not be iterated")
def make_qwen3_reasoner():
"""A real engine-based reasoning adapter over a tiny mock vocabulary."""
from tests.parser.engine.conftest import make_mock_tokenizer
from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter
tokenizer = make_mock_tokenizer(
{"<think>": 200, "</think>": 201, "<tool_call>": 202, "</tool_call>": 203}
)
return Qwen3ParserReasoningAdapter(tokenizer)
class MockReasoner:
def __init__(self, tokenizer):
self.is_reasoning_end = Mock(return_value=False)
@@ -364,6 +392,31 @@ class TestReasoningStructuredOutput:
assert list(called_delta) == [4, 5]
assert result is False
def test_should_advance_whole_delta_fallback_pins_last_sequence_index(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""When only the whole-delta predicate fires, the boundary is the
last index of the sequence, even if the placeholder window
overshoots it and the delta is empty.
"""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
reasoner = MockReasoner(tokenizer=Mock())
# Base-default shape: rescans the whole sequence, ignores the delta.
reasoner.is_reasoning_end_streaming.return_value = True
structured_req.reasoner = reasoner
mock_request_with_structured_output.all_token_ids = [1, 2, 3, 4, 5]
# placeholder window: start = 9 - 1 = 8, past the end -> delta = []
mock_request_with_structured_output.num_computed_tokens = 9
mock_request_with_structured_output.num_output_placeholders = 1
assert manager_with_reasoner.should_advance(mock_request_with_structured_output)
assert structured_req.reasoning_ended is True
assert structured_req.reasoning_end_token_index == 4
def test_should_advance_trims_reasoning_prefix_for_json(
self,
manager_with_reasoner,
@@ -403,3 +456,102 @@ class TestReasoningStructuredOutput:
assert manager_with_reasoner.trim_reasoning_for_advance(
mock_request_with_structured_output, new_token_ids
) == [271, 5005]
def test_should_advance_engine_adapter_scans_only_the_delta(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Engine-based parsers locate the boundary from the delta alone."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
reasoner = make_qwen3_reasoner()
assert reasoner.reasoning_end_token_ids == {201, 202}
reasoner.is_reasoning_end_streaming = Mock(
side_effect=AssertionError("engine path must not rescan the prefix")
)
structured_req.reasoner = reasoner
new_token_ids = [9, 198, 201, 271]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
assert manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_ended is True
assert structured_req.reasoning_end_token_index == 5
assert manager_with_reasoner.trim_reasoning_for_advance(
mock_request_with_structured_output, new_token_ids
) == [271]
def test_should_advance_engine_adapter_tool_start_ends_reasoning(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
new_token_ids = [9, 202, 271]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
assert manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_end_token_index == 4
def test_should_advance_engine_adapter_no_boundary(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
new_token_ids = [9, 198, 271]
mock_request_with_structured_output.all_token_ids = [1, 2, 3] + new_token_ids
assert not manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_ended is False
def test_should_advance_engine_adapter_never_reads_history(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Per-step cost must not depend on how much has been generated."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
new_token_ids = [9, 201]
total = (1 << 20) + len(new_token_ids)
mock_request_with_structured_output.all_token_ids = UntouchableTokenIds(total)
assert manager_with_reasoner.should_advance(
mock_request_with_structured_output, new_token_ids=new_token_ids
)
assert structured_req.reasoning_end_token_index == total - 1
def test_should_advance_engine_adapter_without_new_tokens_persists(
self,
manager_with_reasoner,
mock_request_with_structured_output,
):
"""Draft-validation callers persist the boundary like the legacy path."""
structured_req = mock_request_with_structured_output.structured_output_request
structured_req.reasoning_ended = False
structured_req.reasoner = make_qwen3_reasoner()
mock_request_with_structured_output.all_token_ids = [1, 2, 3, 4, 5, 201, 7]
# placeholder window: start = 7 - 2 = 5, delta = [201, 7]
mock_request_with_structured_output.num_computed_tokens = 7
mock_request_with_structured_output.num_output_placeholders = 2
assert manager_with_reasoner.should_advance(mock_request_with_structured_output)
assert structured_req.reasoning_ended is True
assert structured_req.reasoning_end_token_index == 5