From 1f0bc49ee61be3406d7cb32355980c6f90865c5e Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 25 Sep 2026 12:35:10 +0800 Subject: [PATCH] [Structured Outputs] Parse Lark grammars natively in the xgrammar backend (#58321) Co-authored-by: Claude Opus 5.5 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao --- .../llm/test_struct_output_generate.py | 2 +- .../test_backend_xgrammar_lark.py | 143 ++++++++++++++++++ vllm/sampling_params.py | 17 +++ vllm/v1/structured_output/backend_xgrammar.py | 24 ++- vllm/v1/structured_output/utils.py | 130 ---------------- 5 files changed, 171 insertions(+), 145 deletions(-) create mode 100644 tests/v1/structured_output/test_backend_xgrammar_lark.py diff --git a/tests/entrypoints/llm/test_struct_output_generate.py b/tests/entrypoints/llm/test_struct_output_generate.py index 030b0207e4..e6b02721f4 100644 --- a/tests/entrypoints/llm/test_struct_output_generate.py +++ b/tests/entrypoints/llm/test_struct_output_generate.py @@ -455,7 +455,7 @@ def test_structured_output( structured_outputs=StructuredOutputsParams(grammar="not a grammar"), ) with pytest.raises( - VLLMValidationError, match="Failed to convert the grammar " + VLLMValidationError, match="Invalid grammar specification" ): runner.llm.generate( ( diff --git a/tests/v1/structured_output/test_backend_xgrammar_lark.py b/tests/v1/structured_output/test_backend_xgrammar_lark.py new file mode 100644 index 0000000000..6b13f270d9 --- /dev/null +++ b/tests/v1/structured_output/test_backend_xgrammar_lark.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Lark grammars in the xgrammar backend. + +`structured_outputs.grammar` accepts either EBNF or Lark. The xgrammar backend +parses Lark natively, both when validating a request and when compiling it, so +the request's grammar is never rewritten and frontends that skip Python +request validation get the same behavior. +""" + +import pytest +from mistral_common.protocol.instruct.tool_calls import Tool, ToolChoiceEnum +from transformers import AutoTokenizer + +from vllm.config import StructuredOutputsConfig, VllmConfig +from vllm.exceptions import VLLMValidationError +from vllm.sampling_params import SamplingParams, StructuredOutputsParams +from vllm.tokenizers.mistral import MistralTokenizer +from vllm.v1.structured_output.backend_types import StructuredOutputOptions +from vllm.v1.structured_output.backend_xgrammar import ( + XgrammarBackend, + validate_xgrammar_grammar, +) + +pytestmark = pytest.mark.cpu_test + +TOKENIZER = "openai-community/gpt2" +VOCAB_SIZE = 50257 +MISTRAL_TOKENIZER = "mistralai/Ministral-3-3B-Instruct-2512" + +# Uses Lark features beyond plain rules: a regex terminal, `%import common`, +# and a rule alias. The Codex `apply_patch` tool grammar has the same shape. +LARK_GRAMMAR = """ +start: "id=" NUMBER tag? LF? +tag: " #" /[a-z]+/ -> label +NUMBER: /[0-9]+/ +%import common.LF +""" + +EBNF_GRAMMAR = 'root ::= "id=" [0-9]+' + + +def _validate( + grammar: str, backend: str = "xgrammar", tokenizer: object | None = None +) -> StructuredOutputsParams: + params = SamplingParams(structured_outputs=StructuredOutputsParams(grammar=grammar)) + params._validate_structured_outputs( + _StubModelConfig(), + StructuredOutputsConfig(backend=backend), + tokenizer=tokenizer or object(), + ) + assert params.structured_outputs is not None + return params.structured_outputs + + +class _StubModelConfig: + is_diffusion = False + + +GRAMMARS = [ + pytest.param(LARK_GRAMMAR, id="lark"), + pytest.param(EBNF_GRAMMAR, id="ebnf"), +] + + +@pytest.mark.parametrize("grammar", GRAMMARS) +def test_validation_accepts_grammar_without_rewriting_it(grammar: str): + assert _validate(grammar).grammar == grammar + + +def test_auto_backend_selects_xgrammar_for_lark(): + assert _validate(LARK_GRAMMAR, backend="auto")._backend == "xgrammar" + + +@pytest.mark.parametrize("mode", [ToolChoiceEnum.required, ToolChoiceEnum.none]) +def test_auto_backend_selects_guidance_for_mistral_lark(mode: ToolChoiceEnum): + """The Mistral tool parser generates Lark grammars for llguidance, which + keeps special tokens such as `[TOOL_CALLS]` out of regex matches.""" + tokenizer = MistralTokenizer.from_pretrained(MISTRAL_TOKENIZER) + factory = tokenizer.grammar_factory + tool = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + grammar = factory.get_lark_from_jinja( + template=factory.select_jinja_template(), + mode=mode, + tools=[Tool.from_openai(tool)], + json_schema=None, + parallel_tool_calls=True, + json_only=False, + ) + assert _validate(grammar, "auto", tokenizer)._backend == "guidance" + + +@pytest.mark.parametrize( + "grammar", + [ + pytest.param('rule: "hello"', id="missing-start-rule"), + pytest.param("start: 'hello'", id="single-quoted-literal"), + ], +) +def test_validation_rejects_invalid_lark(grammar: str): + params = SamplingParams(structured_outputs=StructuredOutputsParams(grammar=grammar)) + with pytest.raises(VLLMValidationError, match="Invalid grammar specification"): + validate_xgrammar_grammar(params) + + +@pytest.fixture(scope="module") +def backend() -> XgrammarBackend: + vllm_config = VllmConfig( + structured_outputs_config=StructuredOutputsConfig(backend="xgrammar") + ) + return XgrammarBackend( + vllm_config, + tokenizer=AutoTokenizer.from_pretrained(TOKENIZER), + vocab_size=VOCAB_SIZE, + ) + + +@pytest.mark.parametrize("grammar", GRAMMARS) +@pytest.mark.parametrize( + "text, accepted", + [("id=42", True), ("id=x", False)], +) +def test_compile_grammar_matches_lark_and_ebnf( + backend: XgrammarBackend, grammar: str, text: str, accepted: bool +): + compiled = backend.compile_grammar(StructuredOutputOptions.GRAMMAR, grammar) + token_ids = backend.tokenizer.encode(text, add_special_tokens=False) + assert compiled.accept_tokens("req", token_ids) is accepted + + +def test_compile_grammar_uses_lark_semantics(backend: XgrammarBackend): + compiled = backend.compile_grammar(StructuredOutputOptions.GRAMMAR, LARK_GRAMMAR) + token_ids = backend.tokenizer.encode("id=7 #ok\n", add_special_tokens=False) + assert compiled.accept_tokens("req", token_ids) + assert compiled.is_terminated() or compiled.accept_tokens( + "req", [backend.tokenizer.eos_token_id] + ) diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 77dd91b6b6..88a8978d69 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -1178,6 +1178,7 @@ class SamplingParams( validate_structured_output_request_outlines, ) from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar + from vllm.v1.structured_output.utils import grammar_is_likely_lark if backend.startswith("xgrammar"): # xgrammar with no fallback @@ -1210,6 +1211,22 @@ class SamplingParams( "backends or tokenizer_mode='hf' instead." ) validate_structured_output_request_lm_format_enforcer(self) + elif ( + backend == "auto" + and is_mistral_tokenizer(tokenizer) + and tokenizer.is_tekken + and self.structured_outputs.grammar + and grammar_is_likely_lark(self.structured_outputs.grammar) + ): + # Lark grammars for Tekken Mistral tokenizers, including the ones + # the Mistral tool parser generates, go to guidance. llguidance + # takes the tokenizer from mistral-common and never lets a regex + # match a special token. The xgrammar backend declares no special + # tokens for Mistral tokenizers, so a regex like `.` can match + # `[TOOL_CALLS]` or `[INST]` as plain text. + validate_guidance_grammar(self, tokenizer=_get_llg_tokenizer(tokenizer)) + self.structured_outputs._backend = "guidance" + self.structured_outputs._backend_was_auto = True else: # NOTE: backend must be "auto" here, because we have # checked supported_backends above. diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 3c48ff64a0..5d2c2a4985 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -21,7 +21,6 @@ from vllm.v1.structured_output.backend_types import ( from vllm.v1.structured_output.utils import ( choice_as_grammar, compile_regex_with_timeout, - convert_lark_to_ebnf, grammar_is_likely_lark, ) @@ -91,7 +90,10 @@ class XgrammarBackend(StructuredOutputBackend): '{"type": "object"}', any_whitespace=not self.disable_any_whitespace ) elif request_type == StructuredOutputOptions.GRAMMAR: - ctx = self.compiler.compile_grammar(grammar_spec) + if grammar_is_likely_lark(grammar_spec): + ctx = self.compiler.compile_lark(grammar_spec) + else: + ctx = self.compiler.compile_grammar(grammar_spec) elif request_type == StructuredOutputOptions.REGEX: ctx = compile_regex_with_timeout( self.compiler.compile_regex, @@ -404,19 +406,13 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None: return if so_params.grammar: - if grammar_is_likely_lark(so_params.grammar): - # xgrammar supports EBNF grammars only - try: - so_params.grammar = convert_lark_to_ebnf(so_params.grammar) - except ValueError as e: - raise VLLMValidationError( - "Failed to convert the grammar from Lark to EBNF. " - ) from e - - # Test parsing EBNF grammar, possibly already converted from Lark + # Parse the grammar with the same syntax `compile_grammar` will use, + # but don't compile it. The grammar is passed on unchanged. try: - # parse the grammar, but we aren't compiling it. - xgr.Grammar.from_ebnf(so_params.grammar) + if grammar_is_likely_lark(so_params.grammar): + xgr.Grammar.from_lark(so_params.grammar) + else: + xgr.Grammar.from_ebnf(so_params.grammar) except Exception as e: raise VLLMValidationError("Invalid grammar specification.") from e return diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index 4bb438d6f3..5a0b95c17b 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -435,136 +435,6 @@ def grammar_is_likely_lark(grammar_str: str) -> bool: return True -def convert_lark_to_ebnf(grammar_str: str) -> str: - """Convert a Lark grammar string to EBNF format. - - EBNF reference: - https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md - Lark grammar reference: - https://lark-parser.readthedocs.io/en/latest/grammar.html - - Args: - grammar_str: Input grammar in Lark format - - Returns: - str: Converted grammar in EBNF format - - Examples: - >>> print(convert_lark_to_ebnf("rule: 'hello'")) - root ::= rule - rule ::= "hello" - - """ - if not isinstance(grammar_str, str): - raise ValueError(f"Grammar must be a string, got {type(grammar_str)}") - if not grammar_str.strip(): - raise ValueError("Grammar string cannot be empty") - - defined_rules = set() - referenced_rules = set() - output_lines = [] - - def clean_line(line: str) -> str: - """Remove comments and whitespace from line.""" - return re.sub(r"(#|//).*$", "", line).strip() - - def check_quotes(text: str, rule_name: str, line_num: int) -> None: - """Validate quote matching in text.""" - if text.count("'") % 2 != 0 or text.count('"') % 2 != 0: - raise ValueError(f"Mismatched quotes in {rule_name} on line {line_num}") - - def extract_references(text: str) -> set[str]: - """Extract rule references from text.""" - # Remove quoted strings and special characters - text = re.sub(r'"[^"]*"', "", text) - text = re.sub(r"[+*?()|\[\]{}]", " ", text) - return set(re.findall(r"\b[a-zA-Z_][a-zA-Z0-9_]*\b", text)) - - # First pass: Find root rule and validate rule definitions - lines = [clean_line(line) for line in grammar_str.split("\n")] - first_rule = None - - for line_num, line in enumerate(lines, 1): - if not line or line.startswith("|"): - continue - - if ":" in line: - try: - name = line.split(":", 1)[0].strip().strip("?") - defined_rules.add(name) - if first_rule is None: - first_rule = name - if name == "start": - first_rule = "start" - except IndexError as e: - raise ValueError( - f"Invalid rule format on line {line_num}. " - "Expected 'rule_name: definition'" - ) from e - - if not defined_rules: - raise ValueError("No valid rules found in grammar") - - # Add root rule - output_lines.append(f"root ::= {first_rule}") - - # Second pass: Process rule definitions and alternatives - current_rule = None - current_definition = [] - - for line_num, line in enumerate(lines, 1): - if not line: - continue - - try: - if ":" in line and not line.startswith("|"): - # Save previous rule if exists - if current_rule: - output_lines.append( - f"{current_rule} ::= {' | '.join(current_definition)}" - ) - - # Process new rule - name, definition = line.split(":", 1) - current_rule = name.strip().strip("?") - - check_quotes(definition, f"rule '{current_rule}'", line_num) - definition = re.sub(r"'([^']*)'", r'"\1"', definition) - referenced_rules.update(extract_references(definition)) - current_definition = [definition.strip()] - - elif line.startswith("|"): - if not current_rule: - raise ValueError( - f"Alternative '|' on line {line_num} " - "without a preceding rule definition" - ) - - alt_def = line[1:].strip() - check_quotes( - alt_def, f"alternative for rule '{current_rule}'", line_num - ) - alt_def = re.sub(r"'([^']*)'", r'"\1"', alt_def) - referenced_rules.update(extract_references(alt_def)) - current_definition.append(alt_def) - - except ValueError as e: - raise ValueError(f"Error on line {line_num}: {str(e)}") from e - - # Add final rule if exists - if current_rule: - output_lines.append(f"{current_rule} ::= {' | '.join(current_definition)}") - - # Validate all rules are defined - undefined_rules = referenced_rules - defined_rules - {"root"} - if undefined_rules: - raise ValueError( - f"Referenced rules are not defined: {', '.join(sorted(undefined_rules))}" - ) - - return "\n".join(output_lines) - - def choice_as_grammar(choice: list[str]) -> str: def escape_ebnf_string(s: str) -> str: """Escape EBNF literals, including raw LF, CR, and NUL terminators."""