[Bugfix] Escape control characters in xgrammar choice grammar (#48115)

Signed-off-by: john <6712432+CaiJohn@users.noreply.github.com>
Co-authored-by: john <6712432+CaiJohn@users.noreply.github.com>
This commit is contained in:
CaiJohn
2026-09-17 15:57:47 +03:00
committed by GitHub
co-authored by john
parent 0ef8366ba3
commit 75c71390d5
2 changed files with 58 additions and 3 deletions
+46
View File
@@ -5,9 +5,12 @@ import pytest
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.v1.structured_output.backend_xgrammar import (
has_xgrammar_unsupported_json_features,
validate_xgrammar_grammar,
)
from vllm.v1.structured_output.utils import choice_as_grammar
pytestmark = pytest.mark.cpu_test
@@ -232,3 +235,46 @@ def test_pattern_properties_constrains_keys_and_values(pattern_properties_schema
assert grammar_accepts(pattern_properties_schema, '{"grade_1": 5}')
assert not grammar_accepts(pattern_properties_schema, '{"other": 5}')
assert not grammar_accepts(pattern_properties_schema, '{"grade_1": "five"}')
class TestIsGrammarAcceptString:
class TestPR48115Regressions:
@pytest.mark.parametrize("codepoint", [*range(0x20), 0x7F])
def test_choice_as_grammar_preserves_control_characters(self, codepoint):
choice = f"a{chr(codepoint)}f"
grammar = Grammar.from_ebnf(choice_as_grammar([choice]))
assert _is_grammar_accept_string(grammar, choice)
assert not _is_grammar_accept_string(grammar, "a")
assert not _is_grammar_accept_string(grammar, "af")
assert not _is_grammar_accept_string(grammar, f"a\\u{codepoint:04x}f")
@pytest.mark.parametrize(
("choice", "other"),
[
(r"a\nf", "a\nf"),
('a quote " and a backslash \\', 'a quote " and a backslash '),
("café", "cafe"),
("日本語", "日本"),
("😀", "😁"),
],
)
def test_choice_as_grammar_preserves_literal_choices(self, choice, other):
grammar = Grammar.from_ebnf(choice_as_grammar(["yes", choice]))
assert _is_grammar_accept_string(grammar, "yes")
assert _is_grammar_accept_string(grammar, choice)
assert not _is_grammar_accept_string(grammar, other)
assert not _is_grammar_accept_string(grammar, choice + "extra")
def test_validate_xgrammar_preserves_multiline_choice(self):
structured_outputs = StructuredOutputsParams(choice=["yes", "no\nplease"])
validate_xgrammar_grammar(
SamplingParams(structured_outputs=structured_outputs)
)
assert structured_outputs.choice is None
grammar = Grammar.from_ebnf(structured_outputs.grammar)
assert _is_grammar_accept_string(grammar, "yes")
assert _is_grammar_accept_string(grammar, "no\nplease")
assert not _is_grammar_accept_string(grammar, r"no\nplease")
+12 -3
View File
@@ -567,9 +567,18 @@ def convert_lark_to_ebnf(grammar_str: str) -> str:
def choice_as_grammar(choice: list[str]) -> str:
def escape_ebnf_string(s: str) -> str:
"""Escape special characters in a EBNF string."""
# Escape double quotes and backslashes
return re.sub(r'(["\\])', r"\\\1", s)
"""Escape EBNF literals, including raw LF, CR, and NUL terminators."""
escapes = {"\\": r"\\", '"': r"\"", "\n": r"\n", "\r": r"\r", "\t": r"\t"}
def escape_char(ch: str) -> str:
if ch in escapes:
return escapes[ch]
# Escape remaining C0 controls (U+0000-U+001F) and DEL (U+007F).
if ord(ch) < 0x20 or ord(ch) == 0x7F:
return f"\\u{ord(ch):04x}"
return ch
return "".join(escape_char(ch) for ch in s)
escaped_choices = (escape_ebnf_string(c) for c in choice)
grammar = "root ::= " + " | ".join(f'"{c}"' for c in escaped_choices)