[Structured Outputs] Parse Lark grammars natively in the xgrammar backend (#58321)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
8a269919f4
commit
1f0bc49ee6
@@ -455,7 +455,7 @@ def test_structured_output(
|
||||
structured_outputs=StructuredOutputsParams(grammar="not a grammar"),
|
||||
)
|
||||
with pytest.raises(
|
||||
VLLMValidationError, match="Failed to convert the grammar "
|
||||
VLLMValidationError, match="Invalid grammar specification"
|
||||
):
|
||||
runner.llm.generate(
|
||||
(
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Lark grammars in the xgrammar backend.
|
||||
|
||||
`structured_outputs.grammar` accepts either EBNF or Lark. The xgrammar backend
|
||||
parses Lark natively, both when validating a request and when compiling it, so
|
||||
the request's grammar is never rewritten and frontends that skip Python
|
||||
request validation get the same behavior.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from mistral_common.protocol.instruct.tool_calls import Tool, ToolChoiceEnum
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.config import StructuredOutputsConfig, VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
|
||||
from vllm.v1.structured_output.backend_xgrammar import (
|
||||
XgrammarBackend,
|
||||
validate_xgrammar_grammar,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
TOKENIZER = "openai-community/gpt2"
|
||||
VOCAB_SIZE = 50257
|
||||
MISTRAL_TOKENIZER = "mistralai/Ministral-3-3B-Instruct-2512"
|
||||
|
||||
# Uses Lark features beyond plain rules: a regex terminal, `%import common`,
|
||||
# and a rule alias. The Codex `apply_patch` tool grammar has the same shape.
|
||||
LARK_GRAMMAR = """
|
||||
start: "id=" NUMBER tag? LF?
|
||||
tag: " #" /[a-z]+/ -> label
|
||||
NUMBER: /[0-9]+/
|
||||
%import common.LF
|
||||
"""
|
||||
|
||||
EBNF_GRAMMAR = 'root ::= "id=" [0-9]+'
|
||||
|
||||
|
||||
def _validate(
|
||||
grammar: str, backend: str = "xgrammar", tokenizer: object | None = None
|
||||
) -> StructuredOutputsParams:
|
||||
params = SamplingParams(structured_outputs=StructuredOutputsParams(grammar=grammar))
|
||||
params._validate_structured_outputs(
|
||||
_StubModelConfig(),
|
||||
StructuredOutputsConfig(backend=backend),
|
||||
tokenizer=tokenizer or object(),
|
||||
)
|
||||
assert params.structured_outputs is not None
|
||||
return params.structured_outputs
|
||||
|
||||
|
||||
class _StubModelConfig:
|
||||
is_diffusion = False
|
||||
|
||||
|
||||
GRAMMARS = [
|
||||
pytest.param(LARK_GRAMMAR, id="lark"),
|
||||
pytest.param(EBNF_GRAMMAR, id="ebnf"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("grammar", GRAMMARS)
|
||||
def test_validation_accepts_grammar_without_rewriting_it(grammar: str):
|
||||
assert _validate(grammar).grammar == grammar
|
||||
|
||||
|
||||
def test_auto_backend_selects_xgrammar_for_lark():
|
||||
assert _validate(LARK_GRAMMAR, backend="auto")._backend == "xgrammar"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [ToolChoiceEnum.required, ToolChoiceEnum.none])
|
||||
def test_auto_backend_selects_guidance_for_mistral_lark(mode: ToolChoiceEnum):
|
||||
"""The Mistral tool parser generates Lark grammars for llguidance, which
|
||||
keeps special tokens such as `[TOOL_CALLS]` out of regex matches."""
|
||||
tokenizer = MistralTokenizer.from_pretrained(MISTRAL_TOKENIZER)
|
||||
factory = tokenizer.grammar_factory
|
||||
tool = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
grammar = factory.get_lark_from_jinja(
|
||||
template=factory.select_jinja_template(),
|
||||
mode=mode,
|
||||
tools=[Tool.from_openai(tool)],
|
||||
json_schema=None,
|
||||
parallel_tool_calls=True,
|
||||
json_only=False,
|
||||
)
|
||||
assert _validate(grammar, "auto", tokenizer)._backend == "guidance"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"grammar",
|
||||
[
|
||||
pytest.param('rule: "hello"', id="missing-start-rule"),
|
||||
pytest.param("start: 'hello'", id="single-quoted-literal"),
|
||||
],
|
||||
)
|
||||
def test_validation_rejects_invalid_lark(grammar: str):
|
||||
params = SamplingParams(structured_outputs=StructuredOutputsParams(grammar=grammar))
|
||||
with pytest.raises(VLLMValidationError, match="Invalid grammar specification"):
|
||||
validate_xgrammar_grammar(params)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def backend() -> XgrammarBackend:
|
||||
vllm_config = VllmConfig(
|
||||
structured_outputs_config=StructuredOutputsConfig(backend="xgrammar")
|
||||
)
|
||||
return XgrammarBackend(
|
||||
vllm_config,
|
||||
tokenizer=AutoTokenizer.from_pretrained(TOKENIZER),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("grammar", GRAMMARS)
|
||||
@pytest.mark.parametrize(
|
||||
"text, accepted",
|
||||
[("id=42", True), ("id=x", False)],
|
||||
)
|
||||
def test_compile_grammar_matches_lark_and_ebnf(
|
||||
backend: XgrammarBackend, grammar: str, text: str, accepted: bool
|
||||
):
|
||||
compiled = backend.compile_grammar(StructuredOutputOptions.GRAMMAR, grammar)
|
||||
token_ids = backend.tokenizer.encode(text, add_special_tokens=False)
|
||||
assert compiled.accept_tokens("req", token_ids) is accepted
|
||||
|
||||
|
||||
def test_compile_grammar_uses_lark_semantics(backend: XgrammarBackend):
|
||||
compiled = backend.compile_grammar(StructuredOutputOptions.GRAMMAR, LARK_GRAMMAR)
|
||||
token_ids = backend.tokenizer.encode("id=7 #ok\n", add_special_tokens=False)
|
||||
assert compiled.accept_tokens("req", token_ids)
|
||||
assert compiled.is_terminated() or compiled.accept_tokens(
|
||||
"req", [backend.tokenizer.eos_token_id]
|
||||
)
|
||||
@@ -1178,6 +1178,7 @@ class SamplingParams(
|
||||
validate_structured_output_request_outlines,
|
||||
)
|
||||
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
|
||||
from vllm.v1.structured_output.utils import grammar_is_likely_lark
|
||||
|
||||
if backend.startswith("xgrammar"):
|
||||
# xgrammar with no fallback
|
||||
@@ -1210,6 +1211,22 @@ class SamplingParams(
|
||||
"backends or tokenizer_mode='hf' instead."
|
||||
)
|
||||
validate_structured_output_request_lm_format_enforcer(self)
|
||||
elif (
|
||||
backend == "auto"
|
||||
and is_mistral_tokenizer(tokenizer)
|
||||
and tokenizer.is_tekken
|
||||
and self.structured_outputs.grammar
|
||||
and grammar_is_likely_lark(self.structured_outputs.grammar)
|
||||
):
|
||||
# Lark grammars for Tekken Mistral tokenizers, including the ones
|
||||
# the Mistral tool parser generates, go to guidance. llguidance
|
||||
# takes the tokenizer from mistral-common and never lets a regex
|
||||
# match a special token. The xgrammar backend declares no special
|
||||
# tokens for Mistral tokenizers, so a regex like `.` can match
|
||||
# `[TOOL_CALLS]` or `[INST]` as plain text.
|
||||
validate_guidance_grammar(self, tokenizer=_get_llg_tokenizer(tokenizer))
|
||||
self.structured_outputs._backend = "guidance"
|
||||
self.structured_outputs._backend_was_auto = True
|
||||
else:
|
||||
# NOTE: backend must be "auto" here, because we have
|
||||
# checked supported_backends above.
|
||||
|
||||
@@ -21,7 +21,6 @@ from vllm.v1.structured_output.backend_types import (
|
||||
from vllm.v1.structured_output.utils import (
|
||||
choice_as_grammar,
|
||||
compile_regex_with_timeout,
|
||||
convert_lark_to_ebnf,
|
||||
grammar_is_likely_lark,
|
||||
)
|
||||
|
||||
@@ -91,7 +90,10 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
'{"type": "object"}', any_whitespace=not self.disable_any_whitespace
|
||||
)
|
||||
elif request_type == StructuredOutputOptions.GRAMMAR:
|
||||
ctx = self.compiler.compile_grammar(grammar_spec)
|
||||
if grammar_is_likely_lark(grammar_spec):
|
||||
ctx = self.compiler.compile_lark(grammar_spec)
|
||||
else:
|
||||
ctx = self.compiler.compile_grammar(grammar_spec)
|
||||
elif request_type == StructuredOutputOptions.REGEX:
|
||||
ctx = compile_regex_with_timeout(
|
||||
self.compiler.compile_regex,
|
||||
@@ -404,19 +406,13 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
return
|
||||
|
||||
if so_params.grammar:
|
||||
if grammar_is_likely_lark(so_params.grammar):
|
||||
# xgrammar supports EBNF grammars only
|
||||
try:
|
||||
so_params.grammar = convert_lark_to_ebnf(so_params.grammar)
|
||||
except ValueError as e:
|
||||
raise VLLMValidationError(
|
||||
"Failed to convert the grammar from Lark to EBNF. "
|
||||
) from e
|
||||
|
||||
# Test parsing EBNF grammar, possibly already converted from Lark
|
||||
# Parse the grammar with the same syntax `compile_grammar` will use,
|
||||
# but don't compile it. The grammar is passed on unchanged.
|
||||
try:
|
||||
# parse the grammar, but we aren't compiling it.
|
||||
xgr.Grammar.from_ebnf(so_params.grammar)
|
||||
if grammar_is_likely_lark(so_params.grammar):
|
||||
xgr.Grammar.from_lark(so_params.grammar)
|
||||
else:
|
||||
xgr.Grammar.from_ebnf(so_params.grammar)
|
||||
except Exception as e:
|
||||
raise VLLMValidationError("Invalid grammar specification.") from e
|
||||
return
|
||||
|
||||
@@ -435,136 +435,6 @@ def grammar_is_likely_lark(grammar_str: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def convert_lark_to_ebnf(grammar_str: str) -> str:
|
||||
"""Convert a Lark grammar string to EBNF format.
|
||||
|
||||
EBNF reference:
|
||||
https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md
|
||||
Lark grammar reference:
|
||||
https://lark-parser.readthedocs.io/en/latest/grammar.html
|
||||
|
||||
Args:
|
||||
grammar_str: Input grammar in Lark format
|
||||
|
||||
Returns:
|
||||
str: Converted grammar in EBNF format
|
||||
|
||||
Examples:
|
||||
>>> print(convert_lark_to_ebnf("rule: 'hello'"))
|
||||
root ::= rule
|
||||
rule ::= "hello"
|
||||
|
||||
"""
|
||||
if not isinstance(grammar_str, str):
|
||||
raise ValueError(f"Grammar must be a string, got {type(grammar_str)}")
|
||||
if not grammar_str.strip():
|
||||
raise ValueError("Grammar string cannot be empty")
|
||||
|
||||
defined_rules = set()
|
||||
referenced_rules = set()
|
||||
output_lines = []
|
||||
|
||||
def clean_line(line: str) -> str:
|
||||
"""Remove comments and whitespace from line."""
|
||||
return re.sub(r"(#|//).*$", "", line).strip()
|
||||
|
||||
def check_quotes(text: str, rule_name: str, line_num: int) -> None:
|
||||
"""Validate quote matching in text."""
|
||||
if text.count("'") % 2 != 0 or text.count('"') % 2 != 0:
|
||||
raise ValueError(f"Mismatched quotes in {rule_name} on line {line_num}")
|
||||
|
||||
def extract_references(text: str) -> set[str]:
|
||||
"""Extract rule references from text."""
|
||||
# Remove quoted strings and special characters
|
||||
text = re.sub(r'"[^"]*"', "", text)
|
||||
text = re.sub(r"[+*?()|\[\]{}]", " ", text)
|
||||
return set(re.findall(r"\b[a-zA-Z_][a-zA-Z0-9_]*\b", text))
|
||||
|
||||
# First pass: Find root rule and validate rule definitions
|
||||
lines = [clean_line(line) for line in grammar_str.split("\n")]
|
||||
first_rule = None
|
||||
|
||||
for line_num, line in enumerate(lines, 1):
|
||||
if not line or line.startswith("|"):
|
||||
continue
|
||||
|
||||
if ":" in line:
|
||||
try:
|
||||
name = line.split(":", 1)[0].strip().strip("?")
|
||||
defined_rules.add(name)
|
||||
if first_rule is None:
|
||||
first_rule = name
|
||||
if name == "start":
|
||||
first_rule = "start"
|
||||
except IndexError as e:
|
||||
raise ValueError(
|
||||
f"Invalid rule format on line {line_num}. "
|
||||
"Expected 'rule_name: definition'"
|
||||
) from e
|
||||
|
||||
if not defined_rules:
|
||||
raise ValueError("No valid rules found in grammar")
|
||||
|
||||
# Add root rule
|
||||
output_lines.append(f"root ::= {first_rule}")
|
||||
|
||||
# Second pass: Process rule definitions and alternatives
|
||||
current_rule = None
|
||||
current_definition = []
|
||||
|
||||
for line_num, line in enumerate(lines, 1):
|
||||
if not line:
|
||||
continue
|
||||
|
||||
try:
|
||||
if ":" in line and not line.startswith("|"):
|
||||
# Save previous rule if exists
|
||||
if current_rule:
|
||||
output_lines.append(
|
||||
f"{current_rule} ::= {' | '.join(current_definition)}"
|
||||
)
|
||||
|
||||
# Process new rule
|
||||
name, definition = line.split(":", 1)
|
||||
current_rule = name.strip().strip("?")
|
||||
|
||||
check_quotes(definition, f"rule '{current_rule}'", line_num)
|
||||
definition = re.sub(r"'([^']*)'", r'"\1"', definition)
|
||||
referenced_rules.update(extract_references(definition))
|
||||
current_definition = [definition.strip()]
|
||||
|
||||
elif line.startswith("|"):
|
||||
if not current_rule:
|
||||
raise ValueError(
|
||||
f"Alternative '|' on line {line_num} "
|
||||
"without a preceding rule definition"
|
||||
)
|
||||
|
||||
alt_def = line[1:].strip()
|
||||
check_quotes(
|
||||
alt_def, f"alternative for rule '{current_rule}'", line_num
|
||||
)
|
||||
alt_def = re.sub(r"'([^']*)'", r'"\1"', alt_def)
|
||||
referenced_rules.update(extract_references(alt_def))
|
||||
current_definition.append(alt_def)
|
||||
|
||||
except ValueError as e:
|
||||
raise ValueError(f"Error on line {line_num}: {str(e)}") from e
|
||||
|
||||
# Add final rule if exists
|
||||
if current_rule:
|
||||
output_lines.append(f"{current_rule} ::= {' | '.join(current_definition)}")
|
||||
|
||||
# Validate all rules are defined
|
||||
undefined_rules = referenced_rules - defined_rules - {"root"}
|
||||
if undefined_rules:
|
||||
raise ValueError(
|
||||
f"Referenced rules are not defined: {', '.join(sorted(undefined_rules))}"
|
||||
)
|
||||
|
||||
return "\n".join(output_lines)
|
||||
|
||||
|
||||
def choice_as_grammar(choice: list[str]) -> str:
|
||||
def escape_ebnf_string(s: str) -> str:
|
||||
"""Escape EBNF literals, including raw LF, CR, and NUL terminators."""
|
||||
|
||||
Reference in New Issue
Block a user