[Bugfix] Mask special added tokens in xgrammar JSON, regex and EBNF grammars (#60749)

Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
Vadim Gimpelson
2026-10-13 14:38:23 +03:00
committed by GitHub
co-authored by Claude Artem Perevedentsev
parent cd7517fb17
commit 818f70c295
2 changed files with 170 additions and 9 deletions
@@ -8,8 +8,15 @@ the tokenizer's single eos. Such tokens can therefore escape the grammar
bitmask while the FSM is still mid-object and truncate structured output.
``compile_grammar`` now forwards ``all_stop_token_ids`` to the matcher as
``override_stop_tokens`` so xgrammar masks them until the grammar completes.
Issue #60580: xgrammar only treats tokens that decode to an empty string as
special, so a tokenizer's control tokens (e.g. ``<|im_start|>``) matched
grammar text and could be sampled inside a JSON string. Structural tags and
Lark grammars, which can name such tokens, must still be able to emit them.
"""
import json
import pytest
from transformers import AutoTokenizer
@@ -25,6 +32,14 @@ EOS = 50256 # <|endoftext|> -- the tokenizer's only default stop token
QUOTE = 1 # standalone `"`; opens then closes the JSON string
LETTER = 55 # `X`: valid string content, not a special/stop token by default
QWEN_TOKENIZER = "Qwen/Qwen3-0.6B"
QWEN_VOCAB_SIZE = 151936
# Qwen3 special added tokens.
ENDOFTEXT = 151643 # <|endoftext|>: a generation_config stop token
IM_START = 151644 # <|im_start|>: never a stop token
IM_END = 151645 # <|im_end|>: the tokenizer's eos
def _token_allowed(row, token_id: int) -> bool:
word = int(row[token_id // 32].item()) & 0xFFFFFFFF
@@ -80,3 +95,81 @@ def test_request_stop_tokens_gated_to_grammar_terminal(backend: XgrammarBackend)
assert _token_allowed(bm_override[0], LETTER)
assert _token_allowed(bm_default[0], EOS)
assert _token_allowed(bm_override[0], EOS)
@pytest.fixture(scope="module")
def qwen_tokenizer():
return AutoTokenizer.from_pretrained(QWEN_TOKENIZER)
@pytest.fixture(scope="module")
def qwen_backend(qwen_tokenizer) -> XgrammarBackend:
vllm_config = VllmConfig(
structured_outputs_config=StructuredOutputsConfig(backend="xgrammar")
)
return XgrammarBackend(
vllm_config, tokenizer=qwen_tokenizer, vocab_size=QWEN_VOCAB_SIZE
)
@pytest.mark.parametrize(
"request_type, grammar_spec",
[
(StructuredOutputOptions.JSON, '{"type": "string"}'),
(StructuredOutputOptions.REGEX, '"[^"]*"'),
(StructuredOutputOptions.GRAMMAR, 'root ::= "\\"" [^"]* "\\""'),
],
)
def test_special_tokens_masked_until_grammar_ends(
qwen_backend: XgrammarBackend, qwen_tokenizer, request_type, grammar_spec
):
grammar = qwen_backend.compile_grammar(
request_type, grammar_spec, stop_token_ids={IM_END, ENDOFTEXT}
)
bitmask = qwen_backend.allocate_token_bitmask(1)
text = qwen_tokenizer.encode('"The company is', add_special_tokens=False)
assert grammar.accept_tokens("req", text)
grammar.fill_bitmask(bitmask, 0)
for token_id in (IM_START, IM_END, ENDOFTEXT):
assert not _token_allowed(bitmask[0], token_id)
quote = qwen_tokenizer.encode('"', add_special_tokens=False)
assert grammar.accept_tokens("req", quote)
grammar.fill_bitmask(bitmask, 0)
assert not _token_allowed(bitmask[0], IM_START)
assert _token_allowed(bitmask[0], IM_END)
assert _token_allowed(bitmask[0], ENDOFTEXT)
@pytest.mark.parametrize(
"request_type, grammar_spec",
[
(
StructuredOutputOptions.STRUCTURAL_TAG,
json.dumps(
{
"type": "structural_tag",
"format": {
"type": "tag",
"begin": "<|im_start|>",
"content": {
"type": "json_schema",
"json_schema": {"type": "string"},
},
"end": "\n",
},
}
),
),
(StructuredOutputOptions.GRAMMAR, f'start: <[{IM_START}]> "assistant"'),
],
)
def test_token_naming_grammars_can_emit_special_tokens(
qwen_backend: XgrammarBackend, request_type, grammar_spec
):
grammar = qwen_backend.compile_grammar(request_type, grammar_spec)
bitmask = qwen_backend.allocate_token_bitmask(1)
grammar.fill_bitmask(bitmask, 0)
assert _token_allowed(bitmask[0], IM_START)
assert grammar.accept_tokens("req", [IM_START])
+77 -9
View File
@@ -26,6 +26,8 @@ from vllm.v1.structured_output.utils import (
if TYPE_CHECKING:
import xgrammar as xgr
from vllm.tokenizers import TokenizerLike
else:
xgr = LazyLoader("xgr", globals(), "xgrammar")
@@ -43,6 +45,7 @@ class XgrammarBackend(StructuredOutputBackend):
model_config is not None and model_config.hf_config.model_type == "plamo3"
)
full_vocab_tokenizer_info = None
if is_mistral_tokenizer(self.tokenizer):
# NOTE: ideally, xgrammar should handle this accordingly.
# refer to https://github.com/mlc-ai/xgrammar/blob/d77c0a0173ef14779c918e3be7966ba852f7910f/python/xgrammar/tokenizer_info.py#L98
@@ -66,15 +69,46 @@ class XgrammarBackend(StructuredOutputBackend):
):
tokenizer_info, _ = init_xgrammar()
else:
tokenizer_info = xgr.TokenizerInfo.from_huggingface(
# tokenizer for structural tags
# accepts some of the special tokens
full_vocab_tokenizer_info = xgr.TokenizerInfo.from_huggingface(
self.tokenizer,
vocab_size=self.vocab_size,
)
self.compiler = xgr.GrammarCompiler(
tokenizer_info,
max_threads=8,
cache_enabled=True,
cache_limit_bytes=vllm.envs.VLLM_XGRAMMAR_CACHE_MB * 1024 * 1024,
# tokenizer for json
# does NOT accept any special tokens
tokenizer_info = self._filter_special_tokens(
self.tokenizer, full_vocab_tokenizer_info
)
# FIXME(arpera):
# Here we create not one but TWO compilers
# because xgrammar currently does not properly handle
# special tokens.
# Proper special tokens handling depends on structured output
# request type: json, structural_tag, etc.
# Structural tag may accept some of the special tokens
# At the same time json must NOT accept any special tokens
# Currently xgrammar does not properly handle all the cases
# So, we have to implement additional logic on our side
# We maintain TWO compilers for different structural output types
# If in future xgrammar will fix this problem,
# then we can consider to remove this
# Also, note that other backends like guidance
# do NOT have this problem (as far as I see, I may be wrong).
# So, if someone wants to go and fix the problem on xgrammar side
# you should first definitely check how other backends deal with it
# Some work on xgrammar side has been started:
# https://github.com/mlc-ai/xgrammar/issues/832
# Compiler for json. Must NOT accept any special tokens.
self.compiler = self._create_compiler(tokenizer_info)
# Compiler for structural tag. May accept some of the special tokens.
self.full_vocab_compiler = (
self.compiler
if full_vocab_tokenizer_info is None
else self._create_compiler(full_vocab_tokenizer_info)
)
self.num_speculative_tokens = 0
@@ -122,7 +156,7 @@ class XgrammarBackend(StructuredOutputBackend):
)
elif request_type == StructuredOutputOptions.GRAMMAR:
if grammar_is_likely_lark(grammar_spec):
ctx = self.compiler.compile_lark(grammar_spec)
ctx = self.full_vocab_compiler.compile_lark(grammar_spec)
else:
ctx = self.compiler.compile_grammar(grammar_spec)
elif request_type == StructuredOutputOptions.REGEX:
@@ -142,9 +176,11 @@ class XgrammarBackend(StructuredOutputBackend):
)
for s in s_tag["structures"]
]
ctx = self.compiler.compile_structural_tag(tags, s_tag["triggers"])
ctx = self.full_vocab_compiler.compile_structural_tag(
tags, s_tag["triggers"]
)
else:
ctx = self.compiler.compile_structural_tag(grammar_spec)
ctx = self.full_vocab_compiler.compile_structural_tag(grammar_spec)
else:
logger.error(
"Validation should have already occurred. Please file an issue."
@@ -168,6 +204,38 @@ class XgrammarBackend(StructuredOutputBackend):
def destroy(self):
del self.compiler
del self.full_vocab_compiler
@staticmethod
def _create_compiler(
tokenizer_info: "xgr.TokenizerInfo",
) -> "xgr.GrammarCompiler":
return xgr.GrammarCompiler(
tokenizer_info,
max_threads=8,
cache_enabled=True,
cache_limit_bytes=vllm.envs.VLLM_XGRAMMAR_CACHE_MB * 1024 * 1024,
)
@staticmethod
def _filter_special_tokens(
tokenizer: "TokenizerLike", tokenizer_info: "xgr.TokenizerInfo"
) -> "xgr.TokenizerInfo":
"""Filter ALL special tokens from tokenizer_info"""
special_token_ids = {
token_id
for token_id, token in getattr(
tokenizer, "added_tokens_decoder", {}
).items()
if token.special
}
encoded_vocab = [""] * tokenizer_info.vocab_size
for token, token_id in tokenizer.get_vocab().items():
if token_id < len(encoded_vocab) and token_id not in special_token_ids:
encoded_vocab[token_id] = token
return xgr.TokenizerInfo.from_vocab_and_metadata(
encoded_vocab, tokenizer_info.dump_metadata()
)
@dataclass