[Perf] Eliminate full-history reasoning scans for structured outputs (#55223)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
Signed-off-by: Flora Feng <4florafeng@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Flora Feng
2026-09-08 14:07:16 -07:00
committed by GitHub
co-authored by Wentao Ye
parent 73f61d12aa
commit 29f46ac475
8 changed files with 475 additions and 42 deletions
@@ -8,6 +8,7 @@ from transformers import AutoTokenizer
from vllm.config import StructuredOutputsConfig, VllmConfig
from vllm.config.model import ModelConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
@@ -390,3 +391,119 @@ def test_trim_reasoning_for_advance():
next_step = [post, post]
request.append_output_token_ids(next_step)
assert manager.trim_reasoning_for_advance(request, next_step) == next_step
class _EngineReasonerStub(ParserEngineReasoningAdapter):
"""Adapter-typed reasoner with a fixed end-token set and no real engine."""
def __init__(self, end_token_ids):
self._end_token_ids = frozenset(end_token_ids)
self.windows: list[list[int]] = []
@property
def reasoning_end_token_ids(self):
return self._end_token_ids
def find_reasoning_end_offset(self, token_ids):
self.windows.append(list(token_ids))
for offset, token in enumerate(token_ids):
if token in self._end_token_ids:
return offset
return None
def is_reasoning_end(self, input_ids):
return any(token in self._end_token_ids for token in input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids):
raise AssertionError("engine path must not rescan draft prefixes")
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bitmask_engine_reasoner_ends_midwindow_with_padding(backend):
"""Engine reasoners see the draft window once, without -1 padding."""
tokenizer, manager, request, prompt = _make_manager_and_request(backend)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
marker = tokenizer.encode("\n")[0]
reasoner = _EngineReasonerStub({marker})
manager.reasoner_cls = _EngineReasonerStub
request.structured_output_request.reasoner = reasoner
request.structured_output_request.reasoning_ended = False
pre = tokenizer.encode(" ")[0]
post = tokenizer.encode(",")[0]
drafts = [pre, marker, post, -1]
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert bitmask.shape[0] == len(drafts) + 1
assert (bitmask[0] == -1).all()
assert (bitmask[1] == -1).all()
assert not (bitmask[2] == -1).all()
assert reasoner.windows == [[pre, marker, post]]
assert not grammar.is_terminated()
@pytest.mark.parametrize("backend", ["xgrammar", "guidance"])
def test_bitmask_legacy_order_sensitive_predicate_probed_per_token(backend):
"""Legacy predicates are probed one draft at a time, never on the whole
window. A predicate that reports which marker is *newest* (KimiK3) would
otherwise answer False for a window that closes and then reopens
reasoning, leaving the post-marker drafts and the bonus row
unconstrained.
"""
tokenizer, manager, request, prompt = _make_manager_and_request(backend)
grammar = request.structured_output_request.grammar
assert grammar.accept_tokens(request.request_id, prompt)
close = tokenizer.encode("\n")[0]
reopen = tokenizer.encode("#")[0]
assert close != reopen
class NewestMarkerReasoner:
def __init__(self, *_, **__):
pass
def is_reasoning_end(self, input_ids):
return self.is_reasoning_end_streaming(input_ids, input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids):
for token in reversed(list(delta_ids)):
if token == close:
return True
if token == reopen:
return False
return False
manager.reasoner_cls = NewestMarkerReasoner
request.structured_output_request.reasoner = NewestMarkerReasoner()
request.structured_output_request.reasoning_ended = False
pre = tokenizer.encode(" ")[0]
post = tokenizer.encode(",")[0]
drafts = [pre, close, post, reopen]
# Sanity: the whole window says "reopened", so only per-token probing
# can find the close marker.
assert not NewestMarkerReasoner().is_reasoning_end_streaming([], drafts)
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert bitmask.shape[0] == len(drafts) + 1
assert (bitmask[0] == -1).all()
assert (bitmask[1] == -1).all()
assert not (bitmask[2] == -1).all()
assert not (bitmask[-1] == -1).all()
assert not grammar.is_terminated()