[Bugfix][Structured Output] Mask request stop tokens in xgrammar until grammar terminates (#50595)

Signed-off-by: Yifan Zong <yzong@redhat.com>
This commit is contained in:
yzong-rh
2026-08-13 14:07:59 -04:00
committed by GitHub
parent 48825acc23
commit e6b2a8ad5e
2 changed files with 41 additions and 14 deletions
+38 -9
View File
@@ -12,10 +12,9 @@ from openai_harmony import (
RenderConversationConfig, RenderConversationConfig,
Role, Role,
) )
from transformers import AutoTokenizer from transformers import AutoTokenizer, GenerationConfig
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.config import StructuredOutputsConfig, VllmConfig
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import FunctionCall from vllm.entrypoints.openai.engine.protocol import FunctionCall
from vllm.entrypoints.openai.parser.harmony_utils import ( from vllm.entrypoints.openai.parser.harmony_utils import (
@@ -25,6 +24,8 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.harmony import HarmonyParser from vllm.parser.harmony import HarmonyParser
from vllm.parser.parser_manager import ParserManager from vllm.parser.parser_manager import ParserManager
from vllm.sampling_params import StructuredOutputsParams from vllm.sampling_params import StructuredOutputsParams
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
REASONING_MODEL_NAME = "openai/gpt-oss-20b" REASONING_MODEL_NAME = "openai/gpt-oss-20b"
@@ -34,6 +35,26 @@ def gpt_oss_tokenizer():
return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
@pytest.fixture(scope="module")
def gpt_oss_stop_token_ids() -> set[int]:
eos_token_id = GenerationConfig.from_pretrained(REASONING_MODEL_NAME).eos_token_id
if isinstance(eos_token_id, int):
return {eos_token_id}
return set(eos_token_id)
@pytest.fixture(scope="module")
def xgrammar_backend(gpt_oss_tokenizer) -> XgrammarBackend:
vllm_config = VllmConfig(
structured_outputs_config=StructuredOutputsConfig(backend="xgrammar")
)
return XgrammarBackend(
vllm_config,
tokenizer=gpt_oss_tokenizer,
vocab_size=len(gpt_oss_tokenizer),
)
@pytest.fixture @pytest.fixture
def harmony_parser(gpt_oss_tokenizer): def harmony_parser(gpt_oss_tokenizer):
parser_cls = ParserManager.get_parser( parser_cls = ParserManager.get_parser(
@@ -1036,21 +1057,25 @@ class TestAdjustRequest:
cls, cls,
adjusted_request: ChatCompletionRequest | ResponsesRequest, adjusted_request: ChatCompletionRequest | ResponsesRequest,
expected_admission: Sequence[str], expected_admission: Sequence[str],
xgrammar_backend: XgrammarBackend,
stop_token_ids: set[int],
) -> None: ) -> None:
structured_outputs = adjusted_request.structured_outputs structured_outputs = adjusted_request.structured_outputs
assert structured_outputs is not None assert structured_outputs is not None
assert structured_outputs.structural_tag is not None assert structured_outputs.structural_tag is not None
assert structured_outputs.all_non_structural_tag_constraints_none() assert structured_outputs.all_non_structural_tag_constraints_none()
grammar = Grammar.from_structural_tag(structured_outputs.structural_tag) grammar = xgrammar_backend.compile_grammar(
StructuredOutputOptions.STRUCTURAL_TAG,
structured_outputs.structural_tag,
stop_token_ids=stop_token_ids,
)
expected_admission_set = set(expected_admission) expected_admission_set = set(expected_admission)
for sample_name in cls.ADMISSION_SAMPLES: for sample_name in cls.ADMISSION_SAMPLES:
admitted = _is_grammar_accept_string( tokens = encode_output(getattr(cls, sample_name))
grammar, accepted = grammar.validate_tokens(tokens)
getattr(cls, sample_name), admitted = accepted == tokens
require_termination=False,
)
should_admit = sample_name in expected_admission_set should_admit = sample_name in expected_admission_set
assert admitted is should_admit, ( assert admitted is should_admit, (
f"Expected structured_outputs admission for {sample_name} " f"Expected structured_outputs admission for {sample_name} "
@@ -1183,6 +1208,8 @@ class TestAdjustRequest:
def test_adjust_request( def test_adjust_request(
self, self,
harmony_parser, harmony_parser,
xgrammar_backend,
gpt_oss_stop_token_ids,
request_kind, request_kind,
request_kwargs, request_kwargs,
expected_admission, expected_admission,
@@ -1193,4 +1220,6 @@ class TestAdjustRequest:
self._assert_structured_outputs_admission( self._assert_structured_outputs_admission(
adjusted_request, adjusted_request,
expected_admission, expected_admission,
xgrammar_backend,
gpt_oss_stop_token_ids,
) )
+3 -5
View File
@@ -388,11 +388,9 @@ class HarmonyParser(DelegatingParser):
# Harmomy's stop tokens are <|return|>, <|call|>, <|endoftext|> # Harmomy's stop tokens are <|return|>, <|call|>, <|endoftext|>
# <|return|> is represented as "" since it's the default stop token, which xgrammar # They are represented as "" since xgrammar disallows stop tokens while
# disallows under constraints, leading to bad or infinite generation. # under constraints, leading to bad or infinite generations.
# StreamableParser doesn't consider <|endoftext|> as a message end, so it's excluded _END_TAG = ["<|end|>", ""]
# TODO: Remove <|call|> once #50595 lands.
_END_TAG = ["<|end|>", "<|call|>", ""]
_FINAL_BEGIN = "<|channel|>final{constrain}<|message|>" _FINAL_BEGIN = "<|channel|>final{constrain}<|message|>"
_TOOL_CALL_CHANNELS = [ _TOOL_CALL_CHANNELS = [
"<|channel|>commentary", "<|channel|>commentary",