diff --git a/tests/v1/structured_output/test_backend_xgrammar_stop_tokens.py b/tests/v1/structured_output/test_backend_xgrammar_stop_tokens.py new file mode 100644 index 0000000000..e910d29aa3 --- /dev/null +++ b/tests/v1/structured_output/test_backend_xgrammar_stop_tokens.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for issue #42403. + +A request's stop-token set (the generation_config eos list plus any +user-supplied ``stop_token_ids``) is invisible to xgrammar, which only knows +the tokenizer's single eos. Such tokens can therefore escape the grammar +bitmask while the FSM is still mid-object and truncate structured output. +``compile_grammar`` now forwards ``all_stop_token_ids`` to the matcher as +``override_stop_tokens`` so xgrammar masks them until the grammar completes. +""" + +import pytest +from transformers import AutoTokenizer + +from vllm.config import StructuredOutputsConfig, VllmConfig +from vllm.v1.structured_output.backend_types import StructuredOutputOptions +from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend + +TOKENIZER = "openai-community/gpt2" +VOCAB_SIZE = 50257 + +# gpt2 token ids used to drive a `{"type": "string"}` grammar deterministically. +EOS = 50256 # <|endoftext|> -- the tokenizer's only default stop token +QUOTE = 1 # standalone `"`; opens then closes the JSON string +LETTER = 55 # `X`: valid string content, not a special/stop token by default + + +def _token_allowed(row, token_id: int) -> bool: + word = int(row[token_id // 32].item()) & 0xFFFFFFFF + return bool(word & (1 << (token_id % 32))) + + +@pytest.fixture(scope="module") +def backend() -> XgrammarBackend: + vllm_config = VllmConfig( + structured_outputs_config=StructuredOutputsConfig(backend="xgrammar") + ) + return XgrammarBackend( + vllm_config, + tokenizer=AutoTokenizer.from_pretrained(TOKENIZER), + vocab_size=VOCAB_SIZE, + ) + + +def test_request_stop_tokens_gated_to_grammar_terminal(backend: XgrammarBackend): + schema = '{"type": "string"}' + default = backend.compile_grammar(StructuredOutputOptions.JSON, schema) + override = backend.compile_grammar( + StructuredOutputOptions.JSON, schema, stop_token_ids={EOS, LETTER} + ) + + # Open the string: both grammars are now in a non-terminal state. + for grammar in (default, override): + assert grammar.accept_tokens("req", [QUOTE]) + + bm_default = backend.allocate_token_bitmask(1) + bm_override = backend.allocate_token_bitmask(1) + default.fill_bitmask(bm_default, 0) + override.fill_bitmask(bm_override, 0) + + # Mid-string, the plain token is valid content, so the default grammar + # leaves it samplable -- this is the leak. Registering it as a stop token + # masks it until the grammar can terminate. + assert _token_allowed(bm_default[0], LETTER) + assert not _token_allowed(bm_override[0], LETTER) + + # Close the string -> accepting state (grammar complete, not yet terminated). + for grammar in (default, override): + assert grammar.accept_tokens("req", [QUOTE]) + assert not grammar.is_terminated() + + default.fill_bitmask(bm_default, 0) + override.fill_bitmask(bm_override, 0) + + # The extra stop token may now terminate under the override, never under + # the default grammar -- and the tokenizer's own eos still terminates both, + # so default termination is preserved. + assert not _token_allowed(bm_default[0], LETTER) + assert _token_allowed(bm_override[0], LETTER) + assert _token_allowed(bm_default[0], EOS) + assert _token_allowed(bm_override[0], EOS) diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 08bb237c04..f2ba4ac368 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -184,7 +184,14 @@ class StructuredOutputManager: try: request_type, grammar_spec = struct_request.structured_output_key assert self.backend is not None - return self.backend.compile_grammar(request_type, grammar_spec) + stop_token_ids = ( + request.sampling_params.all_stop_token_ids + if request.sampling_params is not None + else None + ) + return self.backend.compile_grammar( + request_type, grammar_spec, stop_token_ids=stop_token_ids + ) except Exception: logger.exception( "Failed to compile grammar for request %s", request.request_id diff --git a/vllm/v1/structured_output/backend_guidance.py b/vllm/v1/structured_output/backend_guidance.py index 19b2c76e9d..30ecbfa065 100644 --- a/vllm/v1/structured_output/backend_guidance.py +++ b/vllm/v1/structured_output/backend_guidance.py @@ -106,7 +106,10 @@ class GuidanceBackend(StructuredOutputBackend): ) def compile_grammar( - self, request_type: StructuredOutputOptions, grammar_spec: str + self, + request_type: StructuredOutputOptions, + grammar_spec: str, + stop_token_ids: set[int] | None = None, ) -> StructuredOutputGrammar: self.serialized_grammar = serialize_guidance_grammar( request_type, diff --git a/vllm/v1/structured_output/backend_lm_format_enforcer.py b/vllm/v1/structured_output/backend_lm_format_enforcer.py index 69eded30fc..898aaa136e 100644 --- a/vllm/v1/structured_output/backend_lm_format_enforcer.py +++ b/vllm/v1/structured_output/backend_lm_format_enforcer.py @@ -99,7 +99,10 @@ class LMFormatEnforcerBackend(StructuredOutputBackend): ) def compile_grammar( - self, request_type: StructuredOutputOptions, grammar_spec: str + self, + request_type: StructuredOutputOptions, + grammar_spec: str, + stop_token_ids: set[int] | None = None, ) -> StructuredOutputGrammar: character_level_parser: lmformatenforcer.CharacterLevelParser if request_type == StructuredOutputOptions.JSON: diff --git a/vllm/v1/structured_output/backend_outlines.py b/vllm/v1/structured_output/backend_outlines.py index 91627ff154..e66ef6361a 100644 --- a/vllm/v1/structured_output/backend_outlines.py +++ b/vllm/v1/structured_output/backend_outlines.py @@ -71,7 +71,10 @@ class OutlinesBackend(StructuredOutputBackend): return index def compile_grammar( - self, request_type: StructuredOutputOptions, grammar_spec: str + self, + request_type: StructuredOutputOptions, + grammar_spec: str, + stop_token_ids: set[int] | None = None, ) -> StructuredOutputGrammar: if request_type == StructuredOutputOptions.JSON: regex = json_schema.build_regex_from_schema(grammar_spec) diff --git a/vllm/v1/structured_output/backend_types.py b/vllm/v1/structured_output/backend_types.py index 5c09b7b063..d9826c18f5 100644 --- a/vllm/v1/structured_output/backend_types.py +++ b/vllm/v1/structured_output/backend_types.py @@ -105,7 +105,10 @@ class StructuredOutputBackend(ABC): @abstractmethod def compile_grammar( - self, request_type: StructuredOutputOptions, grammar_spec: str + self, + request_type: StructuredOutputOptions, + grammar_spec: str, + stop_token_ids: set[int] | None = None, ) -> StructuredOutputGrammar: """ Compiles a grammar specification into a structured output grammar. @@ -114,6 +117,9 @@ class StructuredOutputBackend(ABC): request_type (StructuredOutputOptions): The type of structured output request. grammar_spec (str): The grammar specification to compile. + stop_token_ids (set[int] | None): The request's EOS and user + stop-token ids (``SamplingParams.all_stop_token_ids``), masked + until the grammar terminates. Returns: StructuredOutputGrammar: The compiled structured output grammar. diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 4f199a1a27..58b726bf72 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -76,7 +76,10 @@ class XgrammarBackend(StructuredOutputBackend): ) def compile_grammar( - self, request_type: StructuredOutputOptions, grammar_spec: str + self, + request_type: StructuredOutputOptions, + grammar_spec: str, + stop_token_ids: set[int] | None = None, ) -> StructuredOutputGrammar: if request_type == StructuredOutputOptions.JSON: ctx = self.compiler.compile_json_schema( @@ -119,6 +122,7 @@ class XgrammarBackend(StructuredOutputBackend): return XgrammarGrammar( matcher=xgr.GrammarMatcher( ctx, + override_stop_tokens=list(stop_token_ids) if stop_token_ids else None, max_rollback_tokens=self.num_speculative_tokens, ), vocab_size=self.vocab_size,