[Bugfix][Structured Output] Mask request stop tokens in xgrammar until grammar terminates (#49227)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for issue #42403.
|
||||
|
||||
A request's stop-token set (the generation_config eos list plus any
|
||||
user-supplied ``stop_token_ids``) is invisible to xgrammar, which only knows
|
||||
the tokenizer's single eos. Such tokens can therefore escape the grammar
|
||||
bitmask while the FSM is still mid-object and truncate structured output.
|
||||
``compile_grammar`` now forwards ``all_stop_token_ids`` to the matcher as
|
||||
``override_stop_tokens`` so xgrammar masks them until the grammar completes.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.config import StructuredOutputsConfig, VllmConfig
|
||||
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
|
||||
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
|
||||
|
||||
TOKENIZER = "openai-community/gpt2"
|
||||
VOCAB_SIZE = 50257
|
||||
|
||||
# gpt2 token ids used to drive a `{"type": "string"}` grammar deterministically.
|
||||
EOS = 50256 # <|endoftext|> -- the tokenizer's only default stop token
|
||||
QUOTE = 1 # standalone `"`; opens then closes the JSON string
|
||||
LETTER = 55 # `X`: valid string content, not a special/stop token by default
|
||||
|
||||
|
||||
def _token_allowed(row, token_id: int) -> bool:
|
||||
word = int(row[token_id // 32].item()) & 0xFFFFFFFF
|
||||
return bool(word & (1 << (token_id % 32)))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def backend() -> XgrammarBackend:
|
||||
vllm_config = VllmConfig(
|
||||
structured_outputs_config=StructuredOutputsConfig(backend="xgrammar")
|
||||
)
|
||||
return XgrammarBackend(
|
||||
vllm_config,
|
||||
tokenizer=AutoTokenizer.from_pretrained(TOKENIZER),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
)
|
||||
|
||||
|
||||
def test_request_stop_tokens_gated_to_grammar_terminal(backend: XgrammarBackend):
|
||||
schema = '{"type": "string"}'
|
||||
default = backend.compile_grammar(StructuredOutputOptions.JSON, schema)
|
||||
override = backend.compile_grammar(
|
||||
StructuredOutputOptions.JSON, schema, stop_token_ids={EOS, LETTER}
|
||||
)
|
||||
|
||||
# Open the string: both grammars are now in a non-terminal state.
|
||||
for grammar in (default, override):
|
||||
assert grammar.accept_tokens("req", [QUOTE])
|
||||
|
||||
bm_default = backend.allocate_token_bitmask(1)
|
||||
bm_override = backend.allocate_token_bitmask(1)
|
||||
default.fill_bitmask(bm_default, 0)
|
||||
override.fill_bitmask(bm_override, 0)
|
||||
|
||||
# Mid-string, the plain token is valid content, so the default grammar
|
||||
# leaves it samplable -- this is the leak. Registering it as a stop token
|
||||
# masks it until the grammar can terminate.
|
||||
assert _token_allowed(bm_default[0], LETTER)
|
||||
assert not _token_allowed(bm_override[0], LETTER)
|
||||
|
||||
# Close the string -> accepting state (grammar complete, not yet terminated).
|
||||
for grammar in (default, override):
|
||||
assert grammar.accept_tokens("req", [QUOTE])
|
||||
assert not grammar.is_terminated()
|
||||
|
||||
default.fill_bitmask(bm_default, 0)
|
||||
override.fill_bitmask(bm_override, 0)
|
||||
|
||||
# The extra stop token may now terminate under the override, never under
|
||||
# the default grammar -- and the tokenizer's own eos still terminates both,
|
||||
# so default termination is preserved.
|
||||
assert not _token_allowed(bm_default[0], LETTER)
|
||||
assert _token_allowed(bm_override[0], LETTER)
|
||||
assert _token_allowed(bm_default[0], EOS)
|
||||
assert _token_allowed(bm_override[0], EOS)
|
||||
@@ -184,7 +184,14 @@ class StructuredOutputManager:
|
||||
try:
|
||||
request_type, grammar_spec = struct_request.structured_output_key
|
||||
assert self.backend is not None
|
||||
return self.backend.compile_grammar(request_type, grammar_spec)
|
||||
stop_token_ids = (
|
||||
request.sampling_params.all_stop_token_ids
|
||||
if request.sampling_params is not None
|
||||
else None
|
||||
)
|
||||
return self.backend.compile_grammar(
|
||||
request_type, grammar_spec, stop_token_ids=stop_token_ids
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to compile grammar for request %s", request.request_id
|
||||
|
||||
@@ -106,7 +106,10 @@ class GuidanceBackend(StructuredOutputBackend):
|
||||
)
|
||||
|
||||
def compile_grammar(
|
||||
self, request_type: StructuredOutputOptions, grammar_spec: str
|
||||
self,
|
||||
request_type: StructuredOutputOptions,
|
||||
grammar_spec: str,
|
||||
stop_token_ids: set[int] | None = None,
|
||||
) -> StructuredOutputGrammar:
|
||||
self.serialized_grammar = serialize_guidance_grammar(
|
||||
request_type,
|
||||
|
||||
@@ -99,7 +99,10 @@ class LMFormatEnforcerBackend(StructuredOutputBackend):
|
||||
)
|
||||
|
||||
def compile_grammar(
|
||||
self, request_type: StructuredOutputOptions, grammar_spec: str
|
||||
self,
|
||||
request_type: StructuredOutputOptions,
|
||||
grammar_spec: str,
|
||||
stop_token_ids: set[int] | None = None,
|
||||
) -> StructuredOutputGrammar:
|
||||
character_level_parser: lmformatenforcer.CharacterLevelParser
|
||||
if request_type == StructuredOutputOptions.JSON:
|
||||
|
||||
@@ -71,7 +71,10 @@ class OutlinesBackend(StructuredOutputBackend):
|
||||
return index
|
||||
|
||||
def compile_grammar(
|
||||
self, request_type: StructuredOutputOptions, grammar_spec: str
|
||||
self,
|
||||
request_type: StructuredOutputOptions,
|
||||
grammar_spec: str,
|
||||
stop_token_ids: set[int] | None = None,
|
||||
) -> StructuredOutputGrammar:
|
||||
if request_type == StructuredOutputOptions.JSON:
|
||||
regex = json_schema.build_regex_from_schema(grammar_spec)
|
||||
|
||||
@@ -105,7 +105,10 @@ class StructuredOutputBackend(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def compile_grammar(
|
||||
self, request_type: StructuredOutputOptions, grammar_spec: str
|
||||
self,
|
||||
request_type: StructuredOutputOptions,
|
||||
grammar_spec: str,
|
||||
stop_token_ids: set[int] | None = None,
|
||||
) -> StructuredOutputGrammar:
|
||||
"""
|
||||
Compiles a grammar specification into a structured output grammar.
|
||||
@@ -114,6 +117,9 @@ class StructuredOutputBackend(ABC):
|
||||
request_type (StructuredOutputOptions): The type of structured
|
||||
output request.
|
||||
grammar_spec (str): The grammar specification to compile.
|
||||
stop_token_ids (set[int] | None): The request's EOS and user
|
||||
stop-token ids (``SamplingParams.all_stop_token_ids``), masked
|
||||
until the grammar terminates.
|
||||
|
||||
Returns:
|
||||
StructuredOutputGrammar: The compiled structured output grammar.
|
||||
|
||||
@@ -76,7 +76,10 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
)
|
||||
|
||||
def compile_grammar(
|
||||
self, request_type: StructuredOutputOptions, grammar_spec: str
|
||||
self,
|
||||
request_type: StructuredOutputOptions,
|
||||
grammar_spec: str,
|
||||
stop_token_ids: set[int] | None = None,
|
||||
) -> StructuredOutputGrammar:
|
||||
if request_type == StructuredOutputOptions.JSON:
|
||||
ctx = self.compiler.compile_json_schema(
|
||||
@@ -119,6 +122,7 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
return XgrammarGrammar(
|
||||
matcher=xgr.GrammarMatcher(
|
||||
ctx,
|
||||
override_stop_tokens=list(stop_token_ids) if stop_token_ids else None,
|
||||
max_rollback_tokens=self.num_speculative_tokens,
|
||||
),
|
||||
vocab_size=self.vocab_size,
|
||||
|
||||
Reference in New Issue
Block a user