[Bugfix] Escape control characters in xgrammar choice grammar (#48115)
Signed-off-by: john <6712432+CaiJohn@users.noreply.github.com> Co-authored-by: john <6712432+CaiJohn@users.noreply.github.com>
This commit is contained in:
@@ -5,9 +5,12 @@ import pytest
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.structured_output.backend_xgrammar import (
|
||||
has_xgrammar_unsupported_json_features,
|
||||
validate_xgrammar_grammar,
|
||||
)
|
||||
from vllm.v1.structured_output.utils import choice_as_grammar
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -232,3 +235,46 @@ def test_pattern_properties_constrains_keys_and_values(pattern_properties_schema
|
||||
assert grammar_accepts(pattern_properties_schema, '{"grade_1": 5}')
|
||||
assert not grammar_accepts(pattern_properties_schema, '{"other": 5}')
|
||||
assert not grammar_accepts(pattern_properties_schema, '{"grade_1": "five"}')
|
||||
|
||||
|
||||
class TestIsGrammarAcceptString:
|
||||
class TestPR48115Regressions:
|
||||
@pytest.mark.parametrize("codepoint", [*range(0x20), 0x7F])
|
||||
def test_choice_as_grammar_preserves_control_characters(self, codepoint):
|
||||
choice = f"a{chr(codepoint)}f"
|
||||
grammar = Grammar.from_ebnf(choice_as_grammar([choice]))
|
||||
|
||||
assert _is_grammar_accept_string(grammar, choice)
|
||||
assert not _is_grammar_accept_string(grammar, "a")
|
||||
assert not _is_grammar_accept_string(grammar, "af")
|
||||
assert not _is_grammar_accept_string(grammar, f"a\\u{codepoint:04x}f")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("choice", "other"),
|
||||
[
|
||||
(r"a\nf", "a\nf"),
|
||||
('a quote " and a backslash \\', 'a quote " and a backslash '),
|
||||
("café", "cafe"),
|
||||
("日本語", "日本"),
|
||||
("😀", "😁"),
|
||||
],
|
||||
)
|
||||
def test_choice_as_grammar_preserves_literal_choices(self, choice, other):
|
||||
grammar = Grammar.from_ebnf(choice_as_grammar(["yes", choice]))
|
||||
|
||||
assert _is_grammar_accept_string(grammar, "yes")
|
||||
assert _is_grammar_accept_string(grammar, choice)
|
||||
assert not _is_grammar_accept_string(grammar, other)
|
||||
assert not _is_grammar_accept_string(grammar, choice + "extra")
|
||||
|
||||
def test_validate_xgrammar_preserves_multiline_choice(self):
|
||||
structured_outputs = StructuredOutputsParams(choice=["yes", "no\nplease"])
|
||||
validate_xgrammar_grammar(
|
||||
SamplingParams(structured_outputs=structured_outputs)
|
||||
)
|
||||
|
||||
assert structured_outputs.choice is None
|
||||
grammar = Grammar.from_ebnf(structured_outputs.grammar)
|
||||
assert _is_grammar_accept_string(grammar, "yes")
|
||||
assert _is_grammar_accept_string(grammar, "no\nplease")
|
||||
assert not _is_grammar_accept_string(grammar, r"no\nplease")
|
||||
|
||||
@@ -567,9 +567,18 @@ def convert_lark_to_ebnf(grammar_str: str) -> str:
|
||||
|
||||
def choice_as_grammar(choice: list[str]) -> str:
|
||||
def escape_ebnf_string(s: str) -> str:
|
||||
"""Escape special characters in a EBNF string."""
|
||||
# Escape double quotes and backslashes
|
||||
return re.sub(r'(["\\])', r"\\\1", s)
|
||||
"""Escape EBNF literals, including raw LF, CR, and NUL terminators."""
|
||||
escapes = {"\\": r"\\", '"': r"\"", "\n": r"\n", "\r": r"\r", "\t": r"\t"}
|
||||
|
||||
def escape_char(ch: str) -> str:
|
||||
if ch in escapes:
|
||||
return escapes[ch]
|
||||
# Escape remaining C0 controls (U+0000-U+001F) and DEL (U+007F).
|
||||
if ord(ch) < 0x20 or ord(ch) == 0x7F:
|
||||
return f"\\u{ord(ch):04x}"
|
||||
return ch
|
||||
|
||||
return "".join(escape_char(ch) for ch in s)
|
||||
|
||||
escaped_choices = (escape_ebnf_string(c) for c in choice)
|
||||
grammar = "root ::= " + " | ".join(f'"{c}"' for c in escaped_choices)
|
||||
|
||||
Reference in New Issue
Block a user