From 818f70c295137ddf9d2ba63262adb6f12e954b58 Mon Sep 17 00:00:00 2001 From: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:38:23 +0400 Subject: [PATCH] [Bugfix] Mask special added tokens in xgrammar JSON, regex and EBNF grammars (#60749) Signed-off-by: Vadim Gimpelson Signed-off-by: Artem Perevedentsev Co-authored-by: Claude Co-authored-by: Artem Perevedentsev --- .../test_backend_xgrammar_stop_tokens.py | 93 +++++++++++++++++++ vllm/v1/structured_output/backend_xgrammar.py | 86 +++++++++++++++-- 2 files changed, 170 insertions(+), 9 deletions(-) diff --git a/tests/v1/structured_output/test_backend_xgrammar_stop_tokens.py b/tests/v1/structured_output/test_backend_xgrammar_stop_tokens.py index e910d29aa3..5b0baf44fd 100644 --- a/tests/v1/structured_output/test_backend_xgrammar_stop_tokens.py +++ b/tests/v1/structured_output/test_backend_xgrammar_stop_tokens.py @@ -8,8 +8,15 @@ the tokenizer's single eos. Such tokens can therefore escape the grammar bitmask while the FSM is still mid-object and truncate structured output. ``compile_grammar`` now forwards ``all_stop_token_ids`` to the matcher as ``override_stop_tokens`` so xgrammar masks them until the grammar completes. + +Issue #60580: xgrammar only treats tokens that decode to an empty string as +special, so a tokenizer's control tokens (e.g. ``<|im_start|>``) matched +grammar text and could be sampled inside a JSON string. Structural tags and +Lark grammars, which can name such tokens, must still be able to emit them. """ +import json + import pytest from transformers import AutoTokenizer @@ -25,6 +32,14 @@ EOS = 50256 # <|endoftext|> -- the tokenizer's only default stop token QUOTE = 1 # standalone `"`; opens then closes the JSON string LETTER = 55 # `X`: valid string content, not a special/stop token by default +QWEN_TOKENIZER = "Qwen/Qwen3-0.6B" +QWEN_VOCAB_SIZE = 151936 + +# Qwen3 special added tokens. +ENDOFTEXT = 151643 # <|endoftext|>: a generation_config stop token +IM_START = 151644 # <|im_start|>: never a stop token +IM_END = 151645 # <|im_end|>: the tokenizer's eos + def _token_allowed(row, token_id: int) -> bool: word = int(row[token_id // 32].item()) & 0xFFFFFFFF @@ -80,3 +95,81 @@ def test_request_stop_tokens_gated_to_grammar_terminal(backend: XgrammarBackend) assert _token_allowed(bm_override[0], LETTER) assert _token_allowed(bm_default[0], EOS) assert _token_allowed(bm_override[0], EOS) + + +@pytest.fixture(scope="module") +def qwen_tokenizer(): + return AutoTokenizer.from_pretrained(QWEN_TOKENIZER) + + +@pytest.fixture(scope="module") +def qwen_backend(qwen_tokenizer) -> XgrammarBackend: + vllm_config = VllmConfig( + structured_outputs_config=StructuredOutputsConfig(backend="xgrammar") + ) + return XgrammarBackend( + vllm_config, tokenizer=qwen_tokenizer, vocab_size=QWEN_VOCAB_SIZE + ) + + +@pytest.mark.parametrize( + "request_type, grammar_spec", + [ + (StructuredOutputOptions.JSON, '{"type": "string"}'), + (StructuredOutputOptions.REGEX, '"[^"]*"'), + (StructuredOutputOptions.GRAMMAR, 'root ::= "\\"" [^"]* "\\""'), + ], +) +def test_special_tokens_masked_until_grammar_ends( + qwen_backend: XgrammarBackend, qwen_tokenizer, request_type, grammar_spec +): + grammar = qwen_backend.compile_grammar( + request_type, grammar_spec, stop_token_ids={IM_END, ENDOFTEXT} + ) + bitmask = qwen_backend.allocate_token_bitmask(1) + + text = qwen_tokenizer.encode('"The company is', add_special_tokens=False) + assert grammar.accept_tokens("req", text) + grammar.fill_bitmask(bitmask, 0) + for token_id in (IM_START, IM_END, ENDOFTEXT): + assert not _token_allowed(bitmask[0], token_id) + + quote = qwen_tokenizer.encode('"', add_special_tokens=False) + assert grammar.accept_tokens("req", quote) + grammar.fill_bitmask(bitmask, 0) + assert not _token_allowed(bitmask[0], IM_START) + assert _token_allowed(bitmask[0], IM_END) + assert _token_allowed(bitmask[0], ENDOFTEXT) + + +@pytest.mark.parametrize( + "request_type, grammar_spec", + [ + ( + StructuredOutputOptions.STRUCTURAL_TAG, + json.dumps( + { + "type": "structural_tag", + "format": { + "type": "tag", + "begin": "<|im_start|>", + "content": { + "type": "json_schema", + "json_schema": {"type": "string"}, + }, + "end": "\n", + }, + } + ), + ), + (StructuredOutputOptions.GRAMMAR, f'start: <[{IM_START}]> "assistant"'), + ], +) +def test_token_naming_grammars_can_emit_special_tokens( + qwen_backend: XgrammarBackend, request_type, grammar_spec +): + grammar = qwen_backend.compile_grammar(request_type, grammar_spec) + bitmask = qwen_backend.allocate_token_bitmask(1) + grammar.fill_bitmask(bitmask, 0) + assert _token_allowed(bitmask[0], IM_START) + assert grammar.accept_tokens("req", [IM_START]) diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 9631766d92..94ccc55b8b 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -26,6 +26,8 @@ from vllm.v1.structured_output.utils import ( if TYPE_CHECKING: import xgrammar as xgr + + from vllm.tokenizers import TokenizerLike else: xgr = LazyLoader("xgr", globals(), "xgrammar") @@ -43,6 +45,7 @@ class XgrammarBackend(StructuredOutputBackend): model_config is not None and model_config.hf_config.model_type == "plamo3" ) + full_vocab_tokenizer_info = None if is_mistral_tokenizer(self.tokenizer): # NOTE: ideally, xgrammar should handle this accordingly. # refer to https://github.com/mlc-ai/xgrammar/blob/d77c0a0173ef14779c918e3be7966ba852f7910f/python/xgrammar/tokenizer_info.py#L98 @@ -66,15 +69,46 @@ class XgrammarBackend(StructuredOutputBackend): ): tokenizer_info, _ = init_xgrammar() else: - tokenizer_info = xgr.TokenizerInfo.from_huggingface( + # tokenizer for structural tags + # accepts some of the special tokens + full_vocab_tokenizer_info = xgr.TokenizerInfo.from_huggingface( self.tokenizer, vocab_size=self.vocab_size, ) - self.compiler = xgr.GrammarCompiler( - tokenizer_info, - max_threads=8, - cache_enabled=True, - cache_limit_bytes=vllm.envs.VLLM_XGRAMMAR_CACHE_MB * 1024 * 1024, + # tokenizer for json + # does NOT accept any special tokens + tokenizer_info = self._filter_special_tokens( + self.tokenizer, full_vocab_tokenizer_info + ) + + # FIXME(arpera): + # Here we create not one but TWO compilers + # because xgrammar currently does not properly handle + # special tokens. + # Proper special tokens handling depends on structured output + # request type: json, structural_tag, etc. + # Structural tag may accept some of the special tokens + # At the same time json must NOT accept any special tokens + # Currently xgrammar does not properly handle all the cases + # So, we have to implement additional logic on our side + # We maintain TWO compilers for different structural output types + # If in future xgrammar will fix this problem, + # then we can consider to remove this + # Also, note that other backends like guidance + # do NOT have this problem (as far as I see, I may be wrong). + # So, if someone wants to go and fix the problem on xgrammar side + # you should first definitely check how other backends deal with it + # Some work on xgrammar side has been started: + # https://github.com/mlc-ai/xgrammar/issues/832 + + # Compiler for json. Must NOT accept any special tokens. + self.compiler = self._create_compiler(tokenizer_info) + + # Compiler for structural tag. May accept some of the special tokens. + self.full_vocab_compiler = ( + self.compiler + if full_vocab_tokenizer_info is None + else self._create_compiler(full_vocab_tokenizer_info) ) self.num_speculative_tokens = 0 @@ -122,7 +156,7 @@ class XgrammarBackend(StructuredOutputBackend): ) elif request_type == StructuredOutputOptions.GRAMMAR: if grammar_is_likely_lark(grammar_spec): - ctx = self.compiler.compile_lark(grammar_spec) + ctx = self.full_vocab_compiler.compile_lark(grammar_spec) else: ctx = self.compiler.compile_grammar(grammar_spec) elif request_type == StructuredOutputOptions.REGEX: @@ -142,9 +176,11 @@ class XgrammarBackend(StructuredOutputBackend): ) for s in s_tag["structures"] ] - ctx = self.compiler.compile_structural_tag(tags, s_tag["triggers"]) + ctx = self.full_vocab_compiler.compile_structural_tag( + tags, s_tag["triggers"] + ) else: - ctx = self.compiler.compile_structural_tag(grammar_spec) + ctx = self.full_vocab_compiler.compile_structural_tag(grammar_spec) else: logger.error( "Validation should have already occurred. Please file an issue." @@ -168,6 +204,38 @@ class XgrammarBackend(StructuredOutputBackend): def destroy(self): del self.compiler + del self.full_vocab_compiler + + @staticmethod + def _create_compiler( + tokenizer_info: "xgr.TokenizerInfo", + ) -> "xgr.GrammarCompiler": + return xgr.GrammarCompiler( + tokenizer_info, + max_threads=8, + cache_enabled=True, + cache_limit_bytes=vllm.envs.VLLM_XGRAMMAR_CACHE_MB * 1024 * 1024, + ) + + @staticmethod + def _filter_special_tokens( + tokenizer: "TokenizerLike", tokenizer_info: "xgr.TokenizerInfo" + ) -> "xgr.TokenizerInfo": + """Filter ALL special tokens from tokenizer_info""" + special_token_ids = { + token_id + for token_id, token in getattr( + tokenizer, "added_tokens_decoder", {} + ).items() + if token.special + } + encoded_vocab = [""] * tokenizer_info.vocab_size + for token, token_id in tokenizer.get_vocab().items(): + if token_id < len(encoded_vocab) and token_id not in special_token_ids: + encoded_vocab[token_id] = token + return xgr.TokenizerInfo.from_vocab_and_metadata( + encoded_vocab, tokenizer_info.dump_metadata() + ) @dataclass