[Bugfix][Structured Output] Mask request stop tokens in xgrammar until grammar terminates (#49227)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-08-10 14:29:20 -04:00
committed by GitHub
parent 405bc86768
commit 05f0a80016
7 changed files with 114 additions and 6 deletions
@@ -0,0 +1,82 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for issue #42403.
A request's stop-token set (the generation_config eos list plus any
user-supplied ``stop_token_ids``) is invisible to xgrammar, which only knows
the tokenizer's single eos. Such tokens can therefore escape the grammar
bitmask while the FSM is still mid-object and truncate structured output.
``compile_grammar`` now forwards ``all_stop_token_ids`` to the matcher as
``override_stop_tokens`` so xgrammar masks them until the grammar completes.
"""
import pytest
from transformers import AutoTokenizer
from vllm.config import StructuredOutputsConfig, VllmConfig
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
TOKENIZER = "openai-community/gpt2"
VOCAB_SIZE = 50257
# gpt2 token ids used to drive a `{"type": "string"}` grammar deterministically.
EOS = 50256 # <|endoftext|> -- the tokenizer's only default stop token
QUOTE = 1 # standalone `"`; opens then closes the JSON string
LETTER = 55 # `X`: valid string content, not a special/stop token by default
def _token_allowed(row, token_id: int) -> bool:
word = int(row[token_id // 32].item()) & 0xFFFFFFFF
return bool(word & (1 << (token_id % 32)))
@pytest.fixture(scope="module")
def backend() -> XgrammarBackend:
vllm_config = VllmConfig(
structured_outputs_config=StructuredOutputsConfig(backend="xgrammar")
)
return XgrammarBackend(
vllm_config,
tokenizer=AutoTokenizer.from_pretrained(TOKENIZER),
vocab_size=VOCAB_SIZE,
)
def test_request_stop_tokens_gated_to_grammar_terminal(backend: XgrammarBackend):
schema = '{"type": "string"}'
default = backend.compile_grammar(StructuredOutputOptions.JSON, schema)
override = backend.compile_grammar(
StructuredOutputOptions.JSON, schema, stop_token_ids={EOS, LETTER}
)
# Open the string: both grammars are now in a non-terminal state.
for grammar in (default, override):
assert grammar.accept_tokens("req", [QUOTE])
bm_default = backend.allocate_token_bitmask(1)
bm_override = backend.allocate_token_bitmask(1)
default.fill_bitmask(bm_default, 0)
override.fill_bitmask(bm_override, 0)
# Mid-string, the plain token is valid content, so the default grammar
# leaves it samplable -- this is the leak. Registering it as a stop token
# masks it until the grammar can terminate.
assert _token_allowed(bm_default[0], LETTER)
assert not _token_allowed(bm_override[0], LETTER)
# Close the string -> accepting state (grammar complete, not yet terminated).
for grammar in (default, override):
assert grammar.accept_tokens("req", [QUOTE])
assert not grammar.is_terminated()
default.fill_bitmask(bm_default, 0)
override.fill_bitmask(bm_override, 0)
# The extra stop token may now terminate under the override, never under
# the default grammar -- and the tokenizer's own eos still terminates both,
# so default termination is preserved.
assert not _token_allowed(bm_default[0], LETTER)
assert _token_allowed(bm_override[0], LETTER)
assert _token_allowed(bm_default[0], EOS)
assert _token_allowed(bm_override[0], EOS)
+8 -1
View File
@@ -184,7 +184,14 @@ class StructuredOutputManager:
try:
request_type, grammar_spec = struct_request.structured_output_key
assert self.backend is not None
return self.backend.compile_grammar(request_type, grammar_spec)
stop_token_ids = (
request.sampling_params.all_stop_token_ids
if request.sampling_params is not None
else None
)
return self.backend.compile_grammar(
request_type, grammar_spec, stop_token_ids=stop_token_ids
)
except Exception:
logger.exception(
"Failed to compile grammar for request %s", request.request_id
@@ -106,7 +106,10 @@ class GuidanceBackend(StructuredOutputBackend):
)
def compile_grammar(
self, request_type: StructuredOutputOptions, grammar_spec: str
self,
request_type: StructuredOutputOptions,
grammar_spec: str,
stop_token_ids: set[int] | None = None,
) -> StructuredOutputGrammar:
self.serialized_grammar = serialize_guidance_grammar(
request_type,
@@ -99,7 +99,10 @@ class LMFormatEnforcerBackend(StructuredOutputBackend):
)
def compile_grammar(
self, request_type: StructuredOutputOptions, grammar_spec: str
self,
request_type: StructuredOutputOptions,
grammar_spec: str,
stop_token_ids: set[int] | None = None,
) -> StructuredOutputGrammar:
character_level_parser: lmformatenforcer.CharacterLevelParser
if request_type == StructuredOutputOptions.JSON:
@@ -71,7 +71,10 @@ class OutlinesBackend(StructuredOutputBackend):
return index
def compile_grammar(
self, request_type: StructuredOutputOptions, grammar_spec: str
self,
request_type: StructuredOutputOptions,
grammar_spec: str,
stop_token_ids: set[int] | None = None,
) -> StructuredOutputGrammar:
if request_type == StructuredOutputOptions.JSON:
regex = json_schema.build_regex_from_schema(grammar_spec)
+7 -1
View File
@@ -105,7 +105,10 @@ class StructuredOutputBackend(ABC):
@abstractmethod
def compile_grammar(
self, request_type: StructuredOutputOptions, grammar_spec: str
self,
request_type: StructuredOutputOptions,
grammar_spec: str,
stop_token_ids: set[int] | None = None,
) -> StructuredOutputGrammar:
"""
Compiles a grammar specification into a structured output grammar.
@@ -114,6 +117,9 @@ class StructuredOutputBackend(ABC):
request_type (StructuredOutputOptions): The type of structured
output request.
grammar_spec (str): The grammar specification to compile.
stop_token_ids (set[int] | None): The request's EOS and user
stop-token ids (``SamplingParams.all_stop_token_ids``), masked
until the grammar terminates.
Returns:
StructuredOutputGrammar: The compiled structured output grammar.
@@ -76,7 +76,10 @@ class XgrammarBackend(StructuredOutputBackend):
)
def compile_grammar(
self, request_type: StructuredOutputOptions, grammar_spec: str
self,
request_type: StructuredOutputOptions,
grammar_spec: str,
stop_token_ids: set[int] | None = None,
) -> StructuredOutputGrammar:
if request_type == StructuredOutputOptions.JSON:
ctx = self.compiler.compile_json_schema(
@@ -119,6 +122,7 @@ class XgrammarBackend(StructuredOutputBackend):
return XgrammarGrammar(
matcher=xgr.GrammarMatcher(
ctx,
override_stop_tokens=list(stop_token_ids) if stop_token_ids else None,
max_rollback_tokens=self.num_speculative_tokens,
),
vocab_size=self.vocab_size,