[Bugfix] Mask special added tokens in xgrammar JSON, regex and EBNF grammars (#60749)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
co-authored by
Claude
Artem Perevedentsev
parent
cd7517fb17
commit
818f70c295
@@ -8,8 +8,15 @@ the tokenizer's single eos. Such tokens can therefore escape the grammar
|
||||
bitmask while the FSM is still mid-object and truncate structured output.
|
||||
``compile_grammar`` now forwards ``all_stop_token_ids`` to the matcher as
|
||||
``override_stop_tokens`` so xgrammar masks them until the grammar completes.
|
||||
|
||||
Issue #60580: xgrammar only treats tokens that decode to an empty string as
|
||||
special, so a tokenizer's control tokens (e.g. ``<|im_start|>``) matched
|
||||
grammar text and could be sampled inside a JSON string. Structural tags and
|
||||
Lark grammars, which can name such tokens, must still be able to emit them.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
@@ -25,6 +32,14 @@ EOS = 50256 # <|endoftext|> -- the tokenizer's only default stop token
|
||||
QUOTE = 1 # standalone `"`; opens then closes the JSON string
|
||||
LETTER = 55 # `X`: valid string content, not a special/stop token by default
|
||||
|
||||
QWEN_TOKENIZER = "Qwen/Qwen3-0.6B"
|
||||
QWEN_VOCAB_SIZE = 151936
|
||||
|
||||
# Qwen3 special added tokens.
|
||||
ENDOFTEXT = 151643 # <|endoftext|>: a generation_config stop token
|
||||
IM_START = 151644 # <|im_start|>: never a stop token
|
||||
IM_END = 151645 # <|im_end|>: the tokenizer's eos
|
||||
|
||||
|
||||
def _token_allowed(row, token_id: int) -> bool:
|
||||
word = int(row[token_id // 32].item()) & 0xFFFFFFFF
|
||||
@@ -80,3 +95,81 @@ def test_request_stop_tokens_gated_to_grammar_terminal(backend: XgrammarBackend)
|
||||
assert _token_allowed(bm_override[0], LETTER)
|
||||
assert _token_allowed(bm_default[0], EOS)
|
||||
assert _token_allowed(bm_override[0], EOS)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen_tokenizer():
|
||||
return AutoTokenizer.from_pretrained(QWEN_TOKENIZER)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen_backend(qwen_tokenizer) -> XgrammarBackend:
|
||||
vllm_config = VllmConfig(
|
||||
structured_outputs_config=StructuredOutputsConfig(backend="xgrammar")
|
||||
)
|
||||
return XgrammarBackend(
|
||||
vllm_config, tokenizer=qwen_tokenizer, vocab_size=QWEN_VOCAB_SIZE
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_type, grammar_spec",
|
||||
[
|
||||
(StructuredOutputOptions.JSON, '{"type": "string"}'),
|
||||
(StructuredOutputOptions.REGEX, '"[^"]*"'),
|
||||
(StructuredOutputOptions.GRAMMAR, 'root ::= "\\"" [^"]* "\\""'),
|
||||
],
|
||||
)
|
||||
def test_special_tokens_masked_until_grammar_ends(
|
||||
qwen_backend: XgrammarBackend, qwen_tokenizer, request_type, grammar_spec
|
||||
):
|
||||
grammar = qwen_backend.compile_grammar(
|
||||
request_type, grammar_spec, stop_token_ids={IM_END, ENDOFTEXT}
|
||||
)
|
||||
bitmask = qwen_backend.allocate_token_bitmask(1)
|
||||
|
||||
text = qwen_tokenizer.encode('"The company is', add_special_tokens=False)
|
||||
assert grammar.accept_tokens("req", text)
|
||||
grammar.fill_bitmask(bitmask, 0)
|
||||
for token_id in (IM_START, IM_END, ENDOFTEXT):
|
||||
assert not _token_allowed(bitmask[0], token_id)
|
||||
|
||||
quote = qwen_tokenizer.encode('"', add_special_tokens=False)
|
||||
assert grammar.accept_tokens("req", quote)
|
||||
grammar.fill_bitmask(bitmask, 0)
|
||||
assert not _token_allowed(bitmask[0], IM_START)
|
||||
assert _token_allowed(bitmask[0], IM_END)
|
||||
assert _token_allowed(bitmask[0], ENDOFTEXT)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_type, grammar_spec",
|
||||
[
|
||||
(
|
||||
StructuredOutputOptions.STRUCTURAL_TAG,
|
||||
json.dumps(
|
||||
{
|
||||
"type": "structural_tag",
|
||||
"format": {
|
||||
"type": "tag",
|
||||
"begin": "<|im_start|>",
|
||||
"content": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"type": "string"},
|
||||
},
|
||||
"end": "\n",
|
||||
},
|
||||
}
|
||||
),
|
||||
),
|
||||
(StructuredOutputOptions.GRAMMAR, f'start: <[{IM_START}]> "assistant"'),
|
||||
],
|
||||
)
|
||||
def test_token_naming_grammars_can_emit_special_tokens(
|
||||
qwen_backend: XgrammarBackend, request_type, grammar_spec
|
||||
):
|
||||
grammar = qwen_backend.compile_grammar(request_type, grammar_spec)
|
||||
bitmask = qwen_backend.allocate_token_bitmask(1)
|
||||
grammar.fill_bitmask(bitmask, 0)
|
||||
assert _token_allowed(bitmask[0], IM_START)
|
||||
assert grammar.accept_tokens("req", [IM_START])
|
||||
|
||||
@@ -26,6 +26,8 @@ from vllm.v1.structured_output.utils import (
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import xgrammar as xgr
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
else:
|
||||
xgr = LazyLoader("xgr", globals(), "xgrammar")
|
||||
|
||||
@@ -43,6 +45,7 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
model_config is not None and model_config.hf_config.model_type == "plamo3"
|
||||
)
|
||||
|
||||
full_vocab_tokenizer_info = None
|
||||
if is_mistral_tokenizer(self.tokenizer):
|
||||
# NOTE: ideally, xgrammar should handle this accordingly.
|
||||
# refer to https://github.com/mlc-ai/xgrammar/blob/d77c0a0173ef14779c918e3be7966ba852f7910f/python/xgrammar/tokenizer_info.py#L98
|
||||
@@ -66,15 +69,46 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
):
|
||||
tokenizer_info, _ = init_xgrammar()
|
||||
else:
|
||||
tokenizer_info = xgr.TokenizerInfo.from_huggingface(
|
||||
# tokenizer for structural tags
|
||||
# accepts some of the special tokens
|
||||
full_vocab_tokenizer_info = xgr.TokenizerInfo.from_huggingface(
|
||||
self.tokenizer,
|
||||
vocab_size=self.vocab_size,
|
||||
)
|
||||
self.compiler = xgr.GrammarCompiler(
|
||||
tokenizer_info,
|
||||
max_threads=8,
|
||||
cache_enabled=True,
|
||||
cache_limit_bytes=vllm.envs.VLLM_XGRAMMAR_CACHE_MB * 1024 * 1024,
|
||||
# tokenizer for json
|
||||
# does NOT accept any special tokens
|
||||
tokenizer_info = self._filter_special_tokens(
|
||||
self.tokenizer, full_vocab_tokenizer_info
|
||||
)
|
||||
|
||||
# FIXME(arpera):
|
||||
# Here we create not one but TWO compilers
|
||||
# because xgrammar currently does not properly handle
|
||||
# special tokens.
|
||||
# Proper special tokens handling depends on structured output
|
||||
# request type: json, structural_tag, etc.
|
||||
# Structural tag may accept some of the special tokens
|
||||
# At the same time json must NOT accept any special tokens
|
||||
# Currently xgrammar does not properly handle all the cases
|
||||
# So, we have to implement additional logic on our side
|
||||
# We maintain TWO compilers for different structural output types
|
||||
# If in future xgrammar will fix this problem,
|
||||
# then we can consider to remove this
|
||||
# Also, note that other backends like guidance
|
||||
# do NOT have this problem (as far as I see, I may be wrong).
|
||||
# So, if someone wants to go and fix the problem on xgrammar side
|
||||
# you should first definitely check how other backends deal with it
|
||||
# Some work on xgrammar side has been started:
|
||||
# https://github.com/mlc-ai/xgrammar/issues/832
|
||||
|
||||
# Compiler for json. Must NOT accept any special tokens.
|
||||
self.compiler = self._create_compiler(tokenizer_info)
|
||||
|
||||
# Compiler for structural tag. May accept some of the special tokens.
|
||||
self.full_vocab_compiler = (
|
||||
self.compiler
|
||||
if full_vocab_tokenizer_info is None
|
||||
else self._create_compiler(full_vocab_tokenizer_info)
|
||||
)
|
||||
|
||||
self.num_speculative_tokens = 0
|
||||
@@ -122,7 +156,7 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
)
|
||||
elif request_type == StructuredOutputOptions.GRAMMAR:
|
||||
if grammar_is_likely_lark(grammar_spec):
|
||||
ctx = self.compiler.compile_lark(grammar_spec)
|
||||
ctx = self.full_vocab_compiler.compile_lark(grammar_spec)
|
||||
else:
|
||||
ctx = self.compiler.compile_grammar(grammar_spec)
|
||||
elif request_type == StructuredOutputOptions.REGEX:
|
||||
@@ -142,9 +176,11 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
)
|
||||
for s in s_tag["structures"]
|
||||
]
|
||||
ctx = self.compiler.compile_structural_tag(tags, s_tag["triggers"])
|
||||
ctx = self.full_vocab_compiler.compile_structural_tag(
|
||||
tags, s_tag["triggers"]
|
||||
)
|
||||
else:
|
||||
ctx = self.compiler.compile_structural_tag(grammar_spec)
|
||||
ctx = self.full_vocab_compiler.compile_structural_tag(grammar_spec)
|
||||
else:
|
||||
logger.error(
|
||||
"Validation should have already occurred. Please file an issue."
|
||||
@@ -168,6 +204,38 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
|
||||
def destroy(self):
|
||||
del self.compiler
|
||||
del self.full_vocab_compiler
|
||||
|
||||
@staticmethod
|
||||
def _create_compiler(
|
||||
tokenizer_info: "xgr.TokenizerInfo",
|
||||
) -> "xgr.GrammarCompiler":
|
||||
return xgr.GrammarCompiler(
|
||||
tokenizer_info,
|
||||
max_threads=8,
|
||||
cache_enabled=True,
|
||||
cache_limit_bytes=vllm.envs.VLLM_XGRAMMAR_CACHE_MB * 1024 * 1024,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _filter_special_tokens(
|
||||
tokenizer: "TokenizerLike", tokenizer_info: "xgr.TokenizerInfo"
|
||||
) -> "xgr.TokenizerInfo":
|
||||
"""Filter ALL special tokens from tokenizer_info"""
|
||||
special_token_ids = {
|
||||
token_id
|
||||
for token_id, token in getattr(
|
||||
tokenizer, "added_tokens_decoder", {}
|
||||
).items()
|
||||
if token.special
|
||||
}
|
||||
encoded_vocab = [""] * tokenizer_info.vocab_size
|
||||
for token, token_id in tokenizer.get_vocab().items():
|
||||
if token_id < len(encoded_vocab) and token_id not in special_token_ids:
|
||||
encoded_vocab[token_id] = token
|
||||
return xgr.TokenizerInfo.from_vocab_and_metadata(
|
||||
encoded_vocab, tokenizer_info.dump_metadata()
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in New Issue
Block a user