Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+53 -33
View File
@@ -12,8 +12,10 @@ from vllm.reasoning import ReasoningParserManager
from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
from vllm.utils import LazyLoader
from vllm.v1.structured_output.backend_guidance import GuidanceBackend
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
StructuredOutputGrammar)
from vllm.v1.structured_output.backend_types import (
StructuredOutputBackend,
StructuredOutputGrammar,
)
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
if TYPE_CHECKING:
@@ -48,8 +50,7 @@ class StructuredOutputManager:
# - at least 1 CPU
# - at most half the number of CPUs or 8, whichever is less
max_workers = max(1, min(multiprocessing.cpu_count() // 2, 8))
self.executor_for_fillmask = ThreadPoolExecutor(
max_workers=max_workers)
self.executor_for_fillmask = ThreadPoolExecutor(max_workers=max_workers)
if not self.vllm_config.model_config.skip_tokenizer_init:
# The default max_workers if not specified is the number of
@@ -60,12 +61,15 @@ class StructuredOutputManager:
max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
self.executor = ThreadPoolExecutor(max_workers=max_workers)
self.tokenizer = init_tokenizer_from_configs(
model_config=self.vllm_config.model_config)
reasoning_parser = \
self.vllm_config.structured_outputs_config.reasoning_parser
model_config=self.vllm_config.model_config
)
reasoning_parser = (
self.vllm_config.structured_outputs_config.reasoning_parser
)
if reasoning_parser:
reasoner_cls = ReasoningParserManager.get_reasoning_parser(
reasoning_parser)
reasoning_parser
)
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
def grammar_init(self, request: Request) -> None:
@@ -73,8 +77,10 @@ class StructuredOutputManager:
return
if TYPE_CHECKING:
assert request.sampling_params is not None and \
request.sampling_params.structured_outputs is not None
assert (
request.sampling_params is not None
and request.sampling_params.structured_outputs is not None
)
# Initialize the backend the first time it is needed.
#
@@ -98,8 +104,7 @@ class StructuredOutputManager:
vocab_size=vocab_size,
)
elif backend == "outlines":
from vllm.v1.structured_output.backend_outlines import (
OutlinesBackend)
from vllm.v1.structured_output.backend_outlines import OutlinesBackend
self.backend = OutlinesBackend(
self.vllm_config,
@@ -108,15 +113,16 @@ class StructuredOutputManager:
)
elif backend == "lm-format-enforcer":
from vllm.v1.structured_output.backend_lm_format_enforcer import ( # noqa: E501
LMFormatEnforcerBackend)
LMFormatEnforcerBackend,
)
self.backend = LMFormatEnforcerBackend(
self.vllm_config,
tokenizer=self.tokenizer,
vocab_size=vocab_size,
)
else:
raise ValueError(
f"Unsupported structured output backend: {backend}")
raise ValueError(f"Unsupported structured output backend: {backend}")
grammar = self.executor.submit(self._async_create_grammar, request)
request.structured_output_request.grammar = grammar # type: ignore[assignment]
@@ -169,8 +175,9 @@ class StructuredOutputManager:
max_num_spec_tokens = 0
if self.vllm_config.speculative_config is not None:
max_num_spec_tokens = \
max_num_spec_tokens = (
self.vllm_config.speculative_config.num_speculative_tokens
)
if self._grammar_bitmask is None:
assert self.backend is not None
@@ -179,22 +186,23 @@ class StructuredOutputManager:
# Allocate a bitmask for each token needing to be checked:
# one for each speculative position, and one more for the
# bonus token / non-speculative token.
self._grammar_bitmask = \
self.backend.allocate_token_bitmask(
max_batch_size * (1 + max_num_spec_tokens))
self._grammar_bitmask = self.backend.allocate_token_bitmask(
max_batch_size * (1 + max_num_spec_tokens)
)
# Generate a batched bitmask for all structured output requests.
# When speculative decoding is enabled, we need to include multiple
# masks for each request, one for each possible bonus token position.
# These are stored inline in the tensor and unpacked by the gpu runner.
cumulative_index = 0
ordered_seq = sorted(structured_output_request_ids.items(),
key=lambda x: x[1])
ordered_seq = sorted(structured_output_request_ids.items(), key=lambda x: x[1])
# Optimized parallel filling of bitmasks for
# non-spec, large-batch-size cases
if len(ordered_seq) > self.fill_bitmask_parallel_threshold and \
max_num_spec_tokens == 0:
if (
len(ordered_seq) > self.fill_bitmask_parallel_threshold
and max_num_spec_tokens == 0
):
promises = []
batch = []
for req_id, _ in ordered_seq:
@@ -205,8 +213,9 @@ class StructuredOutputManager:
assert structured_output_request.grammar is not None
apply_bitmask = self.should_fill_bitmask(request)
batch.append((structured_output_request.grammar,
cumulative_index, apply_bitmask))
batch.append(
(structured_output_request.grammar, cumulative_index, apply_bitmask)
)
if len(batch) == self.fill_bitmask_parallel_batch_size:
promises.append(self._async_submit_fill_bitmask(batch))
batch = []
@@ -232,18 +241,28 @@ class StructuredOutputManager:
state_advancements = 0
req_tokens = scheduled_spec_decode_tokens.get(req_id, [])
for i, token in enumerate(req_tokens + [None]):
self._fill_bitmasks([(structured_output_request.grammar,
cumulative_index, apply_bitmask)])
self._fill_bitmasks(
[
(
structured_output_request.grammar,
cumulative_index,
apply_bitmask,
)
]
)
if apply_bitmask and token is not None and \
not structured_output_request.grammar.is_terminated():
if (
apply_bitmask
and token is not None
and not structured_output_request.grammar.is_terminated()
):
assert structured_output_request.grammar.accept_tokens(
req_id, [token])
req_id, [token]
)
state_advancements += 1
cumulative_index += 1
if state_advancements > 0:
structured_output_request.grammar.rollback(
state_advancements)
structured_output_request.grammar.rollback(state_advancements)
bitmask_tensor = self._grammar_bitmask
if cumulative_index < bitmask_tensor.shape[0]:
@@ -258,8 +277,9 @@ class StructuredOutputManager:
if self.reasoner is not None:
assert request.structured_output_request is not None
if request.structured_output_request.reasoning_ended is None:
request.structured_output_request.reasoning_ended = \
request.structured_output_request.reasoning_ended = (
self.reasoner.is_reasoning_end(request.prompt_token_ids)
)
return request.structured_output_request.reasoning_ended
return True
+48 -32
View File
@@ -14,9 +14,11 @@ import torch
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
from vllm.utils import LazyLoader
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
StructuredOutputGrammar,
StructuredOutputOptions)
from vllm.v1.structured_output.backend_types import (
StructuredOutputBackend,
StructuredOutputGrammar,
StructuredOutputOptions,
)
from vllm.v1.structured_output.request import get_structured_output_key
if TYPE_CHECKING:
@@ -26,8 +28,7 @@ if TYPE_CHECKING:
else:
llguidance = LazyLoader("llguidance", globals(), "llguidance")
llguidance_hf = LazyLoader("llguidance.hf", globals(), "llguidance.hf")
llguidance_torch = LazyLoader("llguidance.torch", globals(),
"llguidance.torch")
llguidance_torch = LazyLoader("llguidance.torch", globals(), "llguidance.torch")
logger = init_logger(__name__)
@@ -36,16 +37,18 @@ def _walk_json_for_additional_properties(data: object):
if isinstance(data, dict):
for value in data.values():
_walk_json_for_additional_properties(value)
if 'additionalProperties' not in data and \
('properties' in data or 'patternProperties' in data):
data['additionalProperties'] = False
if "additionalProperties" not in data and (
"properties" in data or "patternProperties" in data
):
data["additionalProperties"] = False
elif isinstance(data, list):
for item in data:
_walk_json_for_additional_properties(item)
def process_for_additional_properties(
guide_json: Union[str, dict[str, Any]]) -> dict[str, Any]:
guide_json: Union[str, dict[str, Any]],
) -> dict[str, Any]:
if isinstance(guide_json, str):
guide_json_obj = json.loads(guide_json)
else:
@@ -57,21 +60,27 @@ def process_for_additional_properties(
@dataclass
class GuidanceBackend(StructuredOutputBackend):
def __post_init__(self):
self.disable_any_whitespace = \
self.disable_any_whitespace = (
self.vllm_config.structured_outputs_config.disable_any_whitespace
self.disable_additional_properties = \
)
self.disable_additional_properties = (
self.vllm_config.structured_outputs_config.disable_additional_properties
)
self.ll_tokenizer = llguidance_hf.from_tokenizer(
self.tokenizer, self.vocab_size)
self.tokenizer, self.vocab_size
)
def compile_grammar(self, request_type: StructuredOutputOptions,
grammar_spec: str) -> StructuredOutputGrammar:
def compile_grammar(
self, request_type: StructuredOutputOptions, grammar_spec: str
) -> StructuredOutputGrammar:
self.serialized_grammar = serialize_guidance_grammar(
request_type, grammar_spec, self.disable_any_whitespace,
self.disable_additional_properties)
request_type,
grammar_spec,
self.disable_any_whitespace,
self.disable_additional_properties,
)
ll_matcher = llguidance.LLMatcher(
self.ll_tokenizer,
@@ -90,7 +99,8 @@ class GuidanceBackend(StructuredOutputBackend):
def allocate_token_bitmask(self, max_num_seqs: int):
return llguidance_torch.allocate_token_bitmask(
max_num_seqs, self.ll_tokenizer.vocab_size)
max_num_seqs, self.ll_tokenizer.vocab_size
)
def destroy(self):
pass
@@ -178,15 +188,17 @@ def serialize_guidance_grammar(
disable_any_whitespace: bool = False,
disable_additional_properties: bool = False,
) -> str:
def _process_schema(grammar_spec: Union[str, dict[str, Any]], ) -> str:
def _process_schema(
grammar_spec: Union[str, dict[str, Any]],
) -> str:
if disable_additional_properties:
grammar_spec = process_for_additional_properties(grammar_spec)
return llguidance.LLMatcher.grammar_from_json_schema(
grammar_spec,
defaults={
"whitespace_flexible": not disable_any_whitespace,
})
},
)
if request_type == StructuredOutputOptions.JSON:
return _process_schema(grammar_spec)
@@ -195,7 +207,8 @@ def serialize_guidance_grammar(
'{"type": "object"}',
defaults={
"whitespace_flexible": not disable_any_whitespace,
})
},
)
else:
if request_type == StructuredOutputOptions.REGEX:
tp = "regex"
@@ -215,29 +228,32 @@ def serialize_guidance_grammar(
trig = next((t for t in triggers if begin.startswith(t)), None)
if trig is None:
raise ValueError(
f"Trigger {begin} not found in triggers {triggers}")
f"Trigger {begin} not found in triggers {triggers}"
)
tags.append(
llguidance.StructTag(
trigger=trig,
begin=s["begin"],
grammar=_process_schema(s["schema"]),
end=s["end"],
))
)
)
if not tags:
raise ValueError(
"No structural tags found in the grammar spec.")
raise ValueError("No structural tags found in the grammar spec.")
return llguidance.StructTag.to_grammar(tags)
else:
logger.error("Validation should have already occurred. "
"Please file an issue.")
raise ValueError("grammar is not of valid supported types. "
f"({request_type!s})")
logger.error(
"Validation should have already occurred. Please file an issue."
)
raise ValueError(
f"grammar is not of valid supported types. ({request_type!s})"
)
return llguidance.grammar_from(tp, grammar_spec)
def validate_guidance_grammar(
sampling_params: SamplingParams,
tokenizer: Optional[llguidance.LLTokenizer] = None) -> None:
sampling_params: SamplingParams, tokenizer: Optional[llguidance.LLTokenizer] = None
) -> None:
tp, grm = get_structured_output_key(sampling_params)
guidance_grm = serialize_guidance_grammar(tp, grm)
err = llguidance.LLMatcher.validate_grammar(guidance_grm, tokenizer)
@@ -13,26 +13,31 @@ from transformers import PreTrainedTokenizerBase
from vllm.sampling_params import SamplingParams
from vllm.utils import LazyLoader
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
StructuredOutputGrammar,
StructuredOutputOptions)
from vllm.v1.structured_output.backend_types import (
StructuredOutputBackend,
StructuredOutputGrammar,
StructuredOutputOptions,
)
if TYPE_CHECKING:
import lmformatenforcer
import lmformatenforcer.integrations.vllm as lmfe_vllm
else:
lmformatenforcer = LazyLoader("lmformatenforcer", globals(),
"lmformatenforcer")
lmfe_vllm = LazyLoader("lmformatenforcer.integrations.vllm", globals(),
"lmformatenforcer.integrations.vllm")
lmformatenforcer = LazyLoader("lmformatenforcer", globals(), "lmformatenforcer")
lmfe_vllm = LazyLoader(
"lmformatenforcer.integrations.vllm",
globals(),
"lmformatenforcer.integrations.vllm",
)
@lru_cache
def _cached_build_vllm_token_enforcer_tokenizer_data(
tokenizer: PreTrainedTokenizerBase,
vocab_size: int) -> lmfe_vllm.TokenEnforcerTokenizerData:
tokenizer: PreTrainedTokenizerBase, vocab_size: int
) -> lmfe_vllm.TokenEnforcerTokenizerData:
return lmfe_vllm.build_vllm_token_enforcer_tokenizer_data(
tokenizer, use_bitmask=True, vocab_size=vocab_size)
tokenizer, use_bitmask=True, vocab_size=vocab_size
)
@dataclass
@@ -44,7 +49,8 @@ class LMFormatEnforcerGrammar(StructuredOutputGrammar):
original_len = len(self.current_tokens_prefix)
for token in tokens:
if not self.token_enforcer.get_allowed_tokens(
self.current_tokens_prefix).is_token_allowed(token):
self.current_tokens_prefix
).is_token_allowed(token):
# Rollback partial updates to ensure atomicity.
del self.current_tokens_prefix[original_len:]
return False
@@ -56,8 +62,8 @@ class LMFormatEnforcerGrammar(StructuredOutputGrammar):
prefix = tokens[:prefix_length]
next_token = tokens[prefix_length]
if not self.token_enforcer.get_allowed_tokens(
self.current_tokens_prefix +
prefix).is_token_allowed(next_token):
self.current_tokens_prefix + prefix
).is_token_allowed(next_token):
break
else:
return tokens
@@ -69,14 +75,16 @@ class LMFormatEnforcerGrammar(StructuredOutputGrammar):
def fill_bitmask(self, bitmask: torch.Tensor, batch_index: int) -> None:
allowed_tokens = self.token_enforcer.get_allowed_tokens(
self.current_tokens_prefix)
self.current_tokens_prefix
)
bitmask[batch_index] = allowed_tokens.allowed_tokens
def is_terminated(self) -> bool:
# We are considered terminated if the prefix ends with eos_token_id
return_value = len(
self.current_tokens_prefix) > 0 and self.current_tokens_prefix[
-1] == self.token_enforcer.eos_token_id
return_value = (
len(self.current_tokens_prefix) > 0
and self.current_tokens_prefix[-1] == self.token_enforcer.eos_token_id
)
return return_value
def reset(self):
@@ -85,18 +93,18 @@ class LMFormatEnforcerGrammar(StructuredOutputGrammar):
@dataclass
class LMFormatEnforcerBackend(StructuredOutputBackend):
def __post_init__(self):
self.tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
self.tokenizer, self.vocab_size)
self.tokenizer, self.vocab_size
)
def compile_grammar(self, request_type: StructuredOutputOptions,
grammar_spec: str) -> StructuredOutputGrammar:
def compile_grammar(
self, request_type: StructuredOutputOptions, grammar_spec: str
) -> StructuredOutputGrammar:
character_level_parser: lmformatenforcer.CharacterLevelParser
if request_type == StructuredOutputOptions.JSON:
spec_dict = json.loads(grammar_spec)
character_level_parser = lmformatenforcer.JsonSchemaParser(
spec_dict)
character_level_parser = lmformatenforcer.JsonSchemaParser(spec_dict)
elif request_type == StructuredOutputOptions.JSON_OBJECT:
character_level_parser = lmformatenforcer.JsonSchemaParser(None)
elif request_type == StructuredOutputOptions.REGEX:
@@ -104,14 +112,17 @@ class LMFormatEnforcerBackend(StructuredOutputBackend):
elif request_type == StructuredOutputOptions.CHOICE:
choices = ast.literal_eval(grammar_spec)
character_level_parser = lmformatenforcer.UnionParser(
[lmformatenforcer.StringParser(choice) for choice in choices])
[lmformatenforcer.StringParser(choice) for choice in choices]
)
else:
raise ValueError(
"Invalid request type for LM Format Enforcer backend"
f"({request_type!s})")
f"Invalid request type for LM Format Enforcer backend({request_type!s})"
)
max_rollback_tokens = (
self.vllm_config.speculative_config.num_speculative_tokens
if self.vllm_config.speculative_config is not None else 0)
if self.vllm_config.speculative_config is not None
else 0
)
if max_rollback_tokens > 0:
raise ValueError(
@@ -136,8 +147,7 @@ class LMFormatEnforcerBackend(StructuredOutputBackend):
pass
def validate_structured_output_request_lm_format_enforcer(
params: SamplingParams):
def validate_structured_output_request_lm_format_enforcer(params: SamplingParams):
if params.structured_outputs is None:
return
@@ -163,5 +173,7 @@ def validate_structured_output_request_lm_format_enforcer(
elif so_params.choice:
return
elif so_params.grammar:
raise ValueError("LM Format Enforcer structured outputs backend "
"does not support grammar specifications")
raise ValueError(
"LM Format Enforcer structured outputs backend "
"does not support grammar specifications"
)
+46 -42
View File
@@ -15,20 +15,23 @@ from regex import escape as regex_escape
from vllm.sampling_params import SamplingParams
from vllm.utils import LazyLoader
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
StructuredOutputGrammar,
StructuredOutputOptions)
from vllm.v1.structured_output.utils import (OutlinesVocabulary,
get_outlines_cache,
get_outlines_vocabulary)
from vllm.v1.structured_output.backend_types import (
StructuredOutputBackend,
StructuredOutputGrammar,
StructuredOutputOptions,
)
from vllm.v1.structured_output.utils import (
OutlinesVocabulary,
get_outlines_cache,
get_outlines_vocabulary,
)
if TYPE_CHECKING:
import outlines_core as oc
import outlines_core.json_schema as json_schema
else:
oc = LazyLoader("oc", globals(), "outlines_core")
json_schema = LazyLoader("json_schema", globals(),
"outlines_core.json_schema")
json_schema = LazyLoader("json_schema", globals(), "outlines_core.json_schema")
# Python 3.11+ sre_parse and sre_constants
# are deprecated, so we must import them from re
@@ -46,13 +49,13 @@ else:
@dataclass
class OutlinesBackend(StructuredOutputBackend):
def __post_init__(self):
self.vocabulary = get_outlines_vocabulary(self.tokenizer)
self.cache = get_outlines_cache()
def _compile_index(self, regex_string: str,
vocabulary: OutlinesVocabulary) -> oc.Index:
def _compile_index(
self, regex_string: str, vocabulary: OutlinesVocabulary
) -> oc.Index:
cache_key = f"{vocabulary._hash}_{regex_string}"
if cache_key in self.cache:
return self.cache[cache_key]
@@ -62,8 +65,9 @@ class OutlinesBackend(StructuredOutputBackend):
return index
def compile_grammar(self, request_type: StructuredOutputOptions,
grammar_spec: str) -> StructuredOutputGrammar:
def compile_grammar(
self, request_type: StructuredOutputOptions, grammar_spec: str
) -> StructuredOutputGrammar:
if request_type == StructuredOutputOptions.JSON:
regex = json_schema.build_regex_from_schema(grammar_spec)
elif request_type == StructuredOutputOptions.REGEX:
@@ -79,10 +83,13 @@ class OutlinesBackend(StructuredOutputBackend):
index = self._compile_index(regex, self.vocabulary)
max_rollback_tokens = (
self.vllm_config.speculative_config.num_speculative_tokens
if self.vllm_config.speculative_config is not None else 0)
return OutlinesGrammar(vocab_size=self.vocab_size,
guide=oc.Guide(
index, max_rollback=max_rollback_tokens))
if self.vllm_config.speculative_config is not None
else 0
)
return OutlinesGrammar(
vocab_size=self.vocab_size,
guide=oc.Guide(index, max_rollback=max_rollback_tokens),
)
def allocate_token_bitmask(self, max_num_seqs: int) -> torch.Tensor:
return torch.full(
@@ -98,20 +105,15 @@ class OutlinesBackend(StructuredOutputBackend):
@dataclass
class OutlinesGrammar(StructuredOutputGrammar):
vocab_size: int
guide: oc.Guide = field(hash=False)
num_processed_tokens: int = field(default_factory=lambda: 0,
repr=False,
hash=False,
init=False)
num_processed_tokens: int = field(
default_factory=lambda: 0, repr=False, hash=False, init=False
)
# outlines_core signals done on DFA accept; vLLM expects done after EOS.
# We delay the finished flag by one step so EOS can still be emitted.
_prev_finished: bool = field(default=False,
init=False,
repr=False,
hash=False)
_prev_finished: bool = field(default=False, init=False, repr=False, hash=False)
def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
"""Accepts a list of tokens and advances the FSM.
@@ -142,8 +144,7 @@ class OutlinesGrammar(StructuredOutputGrammar):
def fill_bitmask(self, bitmask: torch.Tensor, idx: int) -> None:
mask = bitmask[idx]
self.guide.write_mask_into(mask.data_ptr(), mask.numel(),
mask.element_size())
self.guide.write_mask_into(mask.data_ptr(), mask.numel(), mask.element_size())
def is_terminated(self) -> bool:
curr = self.guide.is_finished()
@@ -187,8 +188,10 @@ def validate_structured_output_request_outlines(params: SamplingParams):
regex = "(" + "|".join(choices) + ")"
validate_regex_is_buildable(regex)
elif so_params.grammar:
raise ValueError("Outlines structured outputs backend "
"does not support grammar specifications")
raise ValueError(
"Outlines structured outputs backend "
"does not support grammar specifications"
)
def _prefix_needs_context(parsed) -> bool:
@@ -196,7 +199,7 @@ def _prefix_needs_context(parsed) -> bool:
def subpattern_consumes(parsed) -> bool:
"""Return True if subpattern can consume at least one character."""
tokens = parsed.data if hasattr(parsed, 'data') else parsed
tokens = parsed.data if hasattr(parsed, "data") else parsed
for ttype, tval in tokens:
# literal, character class, or dot always consumes
if ttype in (sre_parse.LITERAL, sre_parse.IN, sre_parse.ANY):
@@ -212,17 +215,18 @@ def _prefix_needs_context(parsed) -> bool:
if any(subpattern_consumes(br) for br in branches):
return True
# grouped subpattern: recurse into its contents
elif ttype == sre_parse.SUBPATTERN and subpattern_consumes(
tval[3]):
elif ttype == sre_parse.SUBPATTERN and subpattern_consumes(tval[3]):
return True
# No consumers, return False
return False
tokens = parsed.data if hasattr(parsed, 'data') else parsed
tokens = parsed.data if hasattr(parsed, "data") else parsed
for ttype, tval in tokens:
# Direct anchors or look-around
if ttype == sre_parse.AT or ttype in (sre_constants.ASSERT,
sre_constants.ASSERT_NOT):
if ttype == sre_parse.AT or ttype in (
sre_constants.ASSERT,
sre_constants.ASSERT_NOT,
):
return True
# Nested subpattern: check
@@ -261,9 +265,8 @@ def _prefix_needs_context(parsed) -> bool:
def _check_unsupported(parsed) -> None:
"""Check for regex features unsupported by regex-automata"""
tokens = parsed.data if hasattr(parsed, 'data') else parsed
tokens = parsed.data if hasattr(parsed, "data") else parsed
for ttype, tval in tokens:
# backreference
if ttype in (sre_parse.GROUPREF, sre_parse.GROUPREF_EXISTS):
raise ValueError("Backreferences are unsupported.")
@@ -274,8 +277,7 @@ def _check_unsupported(parsed) -> None:
# unicode word boundaries
elif ttype == sre_parse.AT:
if tval in (sre_constants.AT_BOUNDARY,
sre_constants.AT_NON_BOUNDARY):
if tval in (sre_constants.AT_BOUNDARY, sre_constants.AT_NON_BOUNDARY):
raise ValueError("Unicode word boundaries are unsupported.")
elif ttype == sre_parse.BRANCH:
@@ -308,7 +310,8 @@ def validate_regex_is_buildable(pattern: str) -> None:
raise ValueError(
f"Regex uses unsupported feature for structured outputs: {e}. "
"Only basic matching constructs are supported—lookarounds, "
"backreferences, and unicode boundaries are not.") from e
"backreferences, and unicode boundaries are not."
) from e
if _prefix_needs_context(parsed):
raise ValueError(
@@ -317,4 +320,5 @@ def validate_regex_is_buildable(pattern: str) -> None:
"in a way which requires context before any token is matched."
"structured outputs needs regexes that can match without needing "
"that context. Try rewriting the pattern without using these "
f"constructs. Pattern:\n{pattern}")
f"constructs. Pattern:\n{pattern}"
)
+3 -2
View File
@@ -103,8 +103,9 @@ class StructuredOutputBackend(ABC):
vocab_size: int
@abstractmethod
def compile_grammar(self, request_type: StructuredOutputOptions,
grammar_spec: str) -> StructuredOutputGrammar:
def compile_grammar(
self, request_type: StructuredOutputOptions, grammar_spec: str
) -> StructuredOutputGrammar:
"""
Compiles a grammar specification into a structured output grammar.
+74 -42
View File
@@ -14,12 +14,16 @@ from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
from vllm.utils import LazyLoader
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
StructuredOutputGrammar,
StructuredOutputOptions)
from vllm.v1.structured_output.utils import (choice_as_grammar,
convert_lark_to_ebnf,
grammar_is_likely_lark)
from vllm.v1.structured_output.backend_types import (
StructuredOutputBackend,
StructuredOutputGrammar,
StructuredOutputOptions,
)
from vllm.v1.structured_output.utils import (
choice_as_grammar,
convert_lark_to_ebnf,
grammar_is_likely_lark,
)
if TYPE_CHECKING:
import xgrammar as xgr
@@ -31,10 +35,10 @@ logger = init_logger(__name__)
@dataclass
class XgrammarBackend(StructuredOutputBackend):
def __post_init__(self):
self.disable_any_whitespace = \
self.disable_any_whitespace = (
self.vllm_config.structured_outputs_config.disable_any_whitespace
)
if isinstance(self.tokenizer, MistralTokenizer):
# NOTE: ideally, xgrammar should handle this accordingly.
@@ -44,27 +48,33 @@ class XgrammarBackend(StructuredOutputBackend):
encoded_vocab = self.tokenizer._vocab
else:
encoded_vocab = [
token for token, _ in sorted(
token
for token, _ in sorted(
self.tokenizer.get_vocab().items(),
key=lambda x: x[1],
)
]
stop_token_ids = None
if (hasattr(
if (
hasattr(
self.tokenizer,
"eos_token_id",
) and self.tokenizer.eos_token_id is not None):
)
and self.tokenizer.eos_token_id is not None
):
stop_token_ids = [self.tokenizer.eos_token_id]
except AttributeError as e:
raise ValueError(
f"Cannot get the vocabulary of the tokenizer "
f"{type(self.tokenizer)}. The tokenizer should have a "
"get_vocab method.") from e
"get_vocab method."
) from e
tokenizer_info = xgr.TokenizerInfo( # type: ignore
encoded_vocab=encoded_vocab,
# NOTE: https://github.com/mlc-ai/xgrammar/blob/5e141f6ff1ca02bc31f9e512e68b61f2a8ae88e5/tests/python/test_tokenizer_info.py#L43 # noqa: E501
vocab_type=xgr.VocabType.RAW
if self.tokenizer.is_tekken else xgr.VocabType.BYTE_FALLBACK,
if self.tokenizer.is_tekken
else xgr.VocabType.BYTE_FALLBACK,
vocab_size=self.vocab_size,
stop_token_ids=stop_token_ids,
add_prefix_space=True,
@@ -83,18 +93,21 @@ class XgrammarBackend(StructuredOutputBackend):
self.num_speculative_tokens = 0
if self.vllm_config.speculative_config is not None:
self.num_speculative_tokens = \
self.num_speculative_tokens = (
self.vllm_config.speculative_config.num_speculative_tokens
)
def compile_grammar(self, request_type: StructuredOutputOptions,
grammar_spec: str) -> StructuredOutputGrammar:
def compile_grammar(
self, request_type: StructuredOutputOptions, grammar_spec: str
) -> StructuredOutputGrammar:
if request_type == StructuredOutputOptions.JSON:
ctx = self.compiler.compile_json_schema(
grammar_spec, any_whitespace=not self.disable_any_whitespace)
grammar_spec, any_whitespace=not self.disable_any_whitespace
)
elif request_type == StructuredOutputOptions.JSON_OBJECT:
ctx = self.compiler.compile_json_schema(
'{"type": "object"}',
any_whitespace=not self.disable_any_whitespace)
'{"type": "object"}', any_whitespace=not self.disable_any_whitespace
)
elif request_type == StructuredOutputOptions.GRAMMAR:
ctx = self.compiler.compile_grammar(grammar_spec)
elif request_type == StructuredOutputOptions.REGEX:
@@ -106,17 +119,20 @@ class XgrammarBackend(StructuredOutputBackend):
begin=s["begin"],
schema=json.dumps(s["schema"]),
end=s["end"],
) for s in s_tag["structures"]
)
for s in s_tag["structures"]
]
structural_tag = xgr.StructuralTag.from_legacy_structural_tag(
tags, s_tag["triggers"])
tags, s_tag["triggers"]
)
ctx = self.compiler.compile_structural_tag(structural_tag)
else:
logger.error(
"Validation should have already occurred. Please file an issue."
)
raise ValueError(
f"grammar is not of valid supported types. ({request_type!s})")
f"grammar is not of valid supported types. ({request_type!s})"
)
return XgrammarGrammar(
matcher=xgr.GrammarMatcher(
@@ -146,10 +162,9 @@ class XgrammarGrammar(StructuredOutputGrammar):
vocab_size: int
matcher: xgr.GrammarMatcher = field(hash=False)
ctx: xgr.CompiledGrammar = field(hash=False)
num_processed_tokens: int = field(default_factory=lambda: 0,
repr=False,
hash=False,
init=False)
num_processed_tokens: int = field(
default_factory=lambda: 0, repr=False, hash=False, init=False
)
_is_terminated: bool = field(default=False, repr=False, hash=False)
def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
@@ -164,7 +179,10 @@ class XgrammarGrammar(StructuredOutputGrammar):
if not self.matcher.accept_token(token):
logger.error(
"Failed to advance FSM for request %s "
"for tokens %s. Please file an issue.", request_id, token)
"for tokens %s. Please file an issue.",
request_id,
token,
)
return False
self.num_processed_tokens += 1
self._is_terminated = self.matcher.is_terminated()
@@ -216,8 +234,9 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
# Check for array unsupported keywords
if obj.get("type") == "array" and any(
key in obj for key in ("uniqueItems", "contains",
"minContains", "maxContains")):
key in obj
for key in ("uniqueItems", "contains", "minContains", "maxContains")
):
return True
# Unsupported keywords for strings
@@ -226,8 +245,14 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
# Unsupported keywords for objects
if obj.get("type") == "object" and any(
key in obj for key in ("minProperties", "maxProperties",
"propertyNames", "patternProperties")):
key in obj
for key in (
"minProperties",
"maxProperties",
"propertyNames",
"patternProperties",
)
):
return True
# Recursively check all nested objects and arrays
@@ -259,16 +284,18 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
try:
xgr.Grammar.from_regex(so_params.regex)
except Exception as err:
raise ValueError("Failed to transform regex into a grammar: "
f"{err}") from err
raise ValueError(
f"Failed to transform regex into a grammar: {err}"
) from err
if so_params.choice:
choice_grammar = choice_as_grammar(so_params.choice)
try:
xgr.Grammar.from_ebnf(choice_grammar)
except Exception as err:
raise ValueError("Failed to transform choices into a grammar: "
"{err}") from err
raise ValueError(
"Failed to transform choices into a grammar: {err}"
) from err
so_params.choice = None
so_params.grammar = choice_grammar
return
@@ -285,12 +312,14 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
try:
xgr.Grammar.from_json_schema(schema)
except Exception as err:
raise ValueError("Failed to transform json schema into a grammar: "
f"{err}") from err
raise ValueError(
f"Failed to transform json schema into a grammar: {err}"
) from err
if has_xgrammar_unsupported_json_features(schema):
raise ValueError("The provided JSON schema contains features not "
"supported by xgrammar.")
raise ValueError(
"The provided JSON schema contains features not supported by xgrammar."
)
return
if so_params.grammar:
@@ -300,7 +329,8 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
so_params.grammar = convert_lark_to_ebnf(so_params.grammar)
except ValueError as e:
raise ValueError(
"Failed to convert the grammar from Lark to EBNF. ") from e
"Failed to convert the grammar from Lark to EBNF. "
) from e
# Test parsing EBNF grammar, possibly already converted from Lark
try:
@@ -318,10 +348,12 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
begin=s["begin"],
schema=json.dumps(s["schema"]),
end=s["end"],
) for s in s_tag["structures"]
)
for s in s_tag["structures"]
]
structural_tag = xgr.StructuralTag.from_legacy_structural_tag(
tags, s_tag["triggers"])
tags, s_tag["triggers"]
)
xgr.Grammar.from_structural_tag(structural_tag)
except Exception as e:
raise ValueError("Invalid structural tag specification.") from e
+15 -12
View File
@@ -10,17 +10,19 @@ from concurrent.futures._base import TimeoutError
from typing import Optional, Union, cast
from vllm.sampling_params import SamplingParams
from vllm.v1.structured_output.backend_types import (StructuredOutputGrammar,
StructuredOutputKey,
StructuredOutputOptions)
from vllm.v1.structured_output.backend_types import (
StructuredOutputGrammar,
StructuredOutputKey,
StructuredOutputOptions,
)
@dataclasses.dataclass
class StructuredOutputRequest:
sampling_params: SamplingParams
_grammar: Optional[Union[Future[StructuredOutputGrammar],
StructuredOutputGrammar]] = None
_grammar: Optional[
Union[Future[StructuredOutputGrammar], StructuredOutputGrammar]
] = None
reasoning_ended: Optional[bool] = None
def _check_grammar_completion(self) -> bool:
@@ -43,13 +45,15 @@ class StructuredOutputRequest:
@property
def grammar(self) -> Optional[StructuredOutputGrammar]:
completed = self._check_grammar_completion()
return cast(Optional[StructuredOutputGrammar],
self._grammar) if completed else None
return (
cast(Optional[StructuredOutputGrammar], self._grammar)
if completed
else None
)
@grammar.setter
def grammar(
self, grammar: Union[StructuredOutputGrammar,
Future[StructuredOutputGrammar]]
self, grammar: Union[StructuredOutputGrammar, Future[StructuredOutputGrammar]]
) -> None:
self._grammar = grammar
@@ -58,8 +62,7 @@ class StructuredOutputRequest:
return get_structured_output_key(self.sampling_params)
def get_structured_output_key(
sampling_params: SamplingParams) -> StructuredOutputKey:
def get_structured_output_key(sampling_params: SamplingParams) -> StructuredOutputKey:
params = sampling_params.structured_outputs
assert params is not None, "params can't be None."
if params.json is not None:
+83 -73
View File
@@ -76,27 +76,31 @@ def apply_grammar_bitmask(
for req_id, batch_index in seq:
logit_index = batch_index + cumulative_offset
cumulative_offset += len(
scheduler_output.scheduled_spec_decode_tokens.get(req_id, []))
scheduler_output.scheduled_spec_decode_tokens.get(req_id, [])
)
if req_id in scheduler_output.structured_output_request_ids:
struct_out_req_batch_indices[req_id] = logit_index
out_indices = []
# Reorder the bitmask to match the order of the requests in the batch.
sorted_bitmask = np.full(shape=(logits.shape[0], grammar_bitmask.shape[1]),
fill_value=-1,
dtype=grammar_bitmask.dtype)
sorted_bitmask = np.full(
shape=(logits.shape[0], grammar_bitmask.shape[1]),
fill_value=-1,
dtype=grammar_bitmask.dtype,
)
cumulative_index = 0
seq = sorted(scheduler_output.structured_output_request_ids.items(),
key=lambda x: x[1])
seq = sorted(
scheduler_output.structured_output_request_ids.items(), key=lambda x: x[1]
)
for req_id, _ in seq:
num_spec_tokens = len(
scheduler_output.scheduled_spec_decode_tokens.get(req_id, []))
scheduler_output.scheduled_spec_decode_tokens.get(req_id, [])
)
if req_id in struct_out_req_batch_indices:
logit_index = struct_out_req_batch_indices[req_id]
for i in range(1 + num_spec_tokens):
sorted_bitmask[logit_index + i] = \
grammar_bitmask[cumulative_index + i]
sorted_bitmask[logit_index + i] = grammar_bitmask[cumulative_index + i]
out_indices.append(logit_index + i)
cumulative_index += 1 + num_spec_tokens
grammar_bitmask = sorted_bitmask
@@ -128,8 +132,7 @@ class OutlinesVocabulary:
self.inner = vocabulary
# Have to do abs(hash()) because python hashes can
# be negative, and we are using hash as a cache key.
hex_str = hashlib.sha256(
vocabulary.__repr__().encode('utf-8')).hexdigest()
hex_str = hashlib.sha256(vocabulary.__repr__().encode("utf-8")).hexdigest()
hash_int = int(hex_str, 16)
self._hash = hash_int
@@ -165,16 +168,18 @@ def get_outlines_cache():
cache_dir = get_outlines_cache_path()
if envs.VLLM_V1_USE_OUTLINES_CACHE:
logger.warning("Enabling outlines cache. This is an unbounded on-disk "
"cache. It may consume a lot of disk space and should "
"not be used with untrusted clients.")
logger.warning(
"Enabling outlines cache. This is an unbounded on-disk "
"cache. It may consume a lot of disk space and should "
"not be used with untrusted clients."
)
cache = Cache(cache_dir, eviction_policy="none", cull_limit=0)
outlines_version = importlib.metadata.version("outlines_core")
cached_version = cache.get('__version__', None)
cached_version = cache.get("__version__", None)
if cached_version != outlines_version:
cache.clear()
cache.set('__version__', outlines_version)
cache.set("__version__", outlines_version)
return cache
else:
return LRUCache(maxsize=128)
@@ -194,19 +199,17 @@ def _reduced_vocabulary(
A Dict of token string -> equivalent token ids
"""
unicode_to_bytes = {
v: k
for k, v in tokenization_gpt2.bytes_to_unicode().items()
}
unicode_to_bytes = {v: k for k, v in tokenization_gpt2.bytes_to_unicode().items()}
def convert_token_to_string(token: str) -> str:
string = tokenizer.convert_tokens_to_string([token])
# A hack to handle missing spaces to HF's Llama tokenizers
if (type(token) is str
and token.startswith(file_utils.SPIECE_UNDERLINE)
or token == "<0x20>"):
if (
type(token) is str
and token.startswith(file_utils.SPIECE_UNDERLINE)
or token == "<0x20>"
):
return " " + string
return string
@@ -226,8 +229,7 @@ def _reduced_vocabulary(
# by this point.
token_bytes = bytes(token_str) # type: ignore[arg-type]
elif "\ufffd" in token_str and not re_replacement_seq.match(
token_str):
elif "\ufffd" in token_str and not re_replacement_seq.match(token_str):
# Handle tokens with invalid UTF-8 sequences.
if re_llama_byte_token.match(token):
# Llama-like tokenizers use <0xXX> for incomplete sequences.
@@ -238,12 +240,13 @@ def _reduced_vocabulary(
if None in byte_vals:
raise RuntimeError(
f"Cannot convert token `{token}`"
f" ({token_idx}) to bytes: {token_str}")
f" ({token_idx}) to bytes: {token_str}"
)
# safe to ignore, since if None in byte_vals,
# an error is thrown.
token_bytes = bytes(byte_vals) # type: ignore[arg-type]
else:
token_bytes = token_str.encode('utf-8')
token_bytes = token_str.encode("utf-8")
if token_idx != eos_token_id:
vocabulary.setdefault(token_bytes, []).append(token_idx)
@@ -254,16 +257,18 @@ def _reduced_vocabulary(
def get_outlines_vocabulary(tokenizer: AnyTokenizer) -> oc.Vocabulary:
"""Get the `Vocabulary` object for a given tokenizer.
"""
"""Get the `Vocabulary` object for a given tokenizer."""
if hasattr(tokenizer, "_outlines_vocabulary"):
return tokenizer._outlines_vocabulary # type: ignore
try:
if hasattr(
if (
hasattr(
tokenizer,
"eos_token_id",
) and tokenizer.eos_token_id is not None:
)
and tokenizer.eos_token_id is not None
):
eos_token_id = tokenizer.eos_token_id
else:
raise ValueError(
@@ -272,17 +277,18 @@ def get_outlines_vocabulary(tokenizer: AnyTokenizer) -> oc.Vocabulary:
reduced_vocab = _reduced_vocabulary(
tokenizer,
eos_token_id #type: ignore
eos_token_id, # type: ignore
)
vocabulary = OutlinesVocabulary(
oc.Vocabulary(eos_token_id, reduced_vocab))
vocabulary = OutlinesVocabulary(oc.Vocabulary(eos_token_id, reduced_vocab))
tokenizer._outlines_vocabulary = vocabulary # type: ignore
return vocabulary
except AttributeError as e:
raise ValueError(f"Cannot get the vocabulary of the tokenizer "
f"({type(tokenizer)}). The tokenizer should have a "
"get_vocab method.") from e
raise ValueError(
f"Cannot get the vocabulary of the tokenizer "
f"({type(tokenizer)}). The tokenizer should have a "
"get_vocab method."
) from e
def grammar_is_likely_lark(grammar_str: str) -> bool:
@@ -304,14 +310,14 @@ def grammar_is_likely_lark(grammar_str: str) -> bool:
if not grammar_str or not isinstance(grammar_str, str):
return False
for line in grammar_str.split('\n'):
for line in grammar_str.split("\n"):
# Remove both comment styles
line = re.sub(r'(#|//).*$', '', line).strip()
line = re.sub(r"(#|//).*$", "", line).strip()
if not line:
continue
# Look for EBNF rule definition
if '::=' in line:
if "::=" in line:
return False
return True
@@ -348,40 +354,41 @@ def convert_lark_to_ebnf(grammar_str: str) -> str:
def clean_line(line: str) -> str:
"""Remove comments and whitespace from line."""
return re.sub(r'(#|//).*$', '', line).strip()
return re.sub(r"(#|//).*$", "", line).strip()
def check_quotes(text: str, rule_name: str, line_num: int) -> None:
"""Validate quote matching in text."""
if text.count("'") % 2 != 0 or text.count('"') % 2 != 0:
raise ValueError(
f"Mismatched quotes in {rule_name} on line {line_num}")
raise ValueError(f"Mismatched quotes in {rule_name} on line {line_num}")
def extract_references(text: str) -> set[str]:
"""Extract rule references from text."""
# Remove quoted strings and special characters
text = re.sub(r'"[^"]*"', '', text)
text = re.sub(r'[+*?()|\[\]{}]', ' ', text)
return set(re.findall(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b', text))
text = re.sub(r'"[^"]*"', "", text)
text = re.sub(r"[+*?()|\[\]{}]", " ", text)
return set(re.findall(r"\b[a-zA-Z_][a-zA-Z0-9_]*\b", text))
# First pass: Find root rule and validate rule definitions
lines = [clean_line(line) for line in grammar_str.split('\n')]
lines = [clean_line(line) for line in grammar_str.split("\n")]
first_rule = None
for line_num, line in enumerate(lines, 1):
if not line or line.startswith('|'):
if not line or line.startswith("|"):
continue
if ':' in line:
if ":" in line:
try:
name = line.split(':', 1)[0].strip().strip('?')
name = line.split(":", 1)[0].strip().strip("?")
defined_rules.add(name)
if first_rule is None:
first_rule = name
if name == 'start':
first_rule = 'start'
if name == "start":
first_rule = "start"
except IndexError as e:
raise ValueError(f"Invalid rule format on line {line_num}. "
"Expected 'rule_name: definition'") from e
raise ValueError(
f"Invalid rule format on line {line_num}. "
"Expected 'rule_name: definition'"
) from e
if not defined_rules:
raise ValueError("No valid rules found in grammar")
@@ -398,29 +405,33 @@ def convert_lark_to_ebnf(grammar_str: str) -> str:
continue
try:
if ':' in line and not line.startswith('|'):
if ":" in line and not line.startswith("|"):
# Save previous rule if exists
if current_rule:
output_lines.append(
f"{current_rule} ::= {' | '.join(current_definition)}")
f"{current_rule} ::= {' | '.join(current_definition)}"
)
# Process new rule
name, definition = line.split(':', 1)
current_rule = name.strip().strip('?')
name, definition = line.split(":", 1)
current_rule = name.strip().strip("?")
check_quotes(definition, f"rule '{current_rule}'", line_num)
definition = re.sub(r"'([^']*)'", r'"\1"', definition)
referenced_rules.update(extract_references(definition))
current_definition = [definition.strip()]
elif line.startswith('|'):
elif line.startswith("|"):
if not current_rule:
raise ValueError(f"Alternative '|' on line {line_num} "
"without a preceding rule definition")
raise ValueError(
f"Alternative '|' on line {line_num} "
"without a preceding rule definition"
)
alt_def = line[1:].strip()
check_quotes(alt_def, f"alternative for rule '{current_rule}'",
line_num)
check_quotes(
alt_def, f"alternative for rule '{current_rule}'", line_num
)
alt_def = re.sub(r"'([^']*)'", r'"\1"', alt_def)
referenced_rules.update(extract_references(alt_def))
current_definition.append(alt_def)
@@ -430,25 +441,24 @@ def convert_lark_to_ebnf(grammar_str: str) -> str:
# Add final rule if exists
if current_rule:
output_lines.append(
f"{current_rule} ::= {' | '.join(current_definition)}")
output_lines.append(f"{current_rule} ::= {' | '.join(current_definition)}")
# Validate all rules are defined
undefined_rules = referenced_rules - defined_rules - {'root'}
undefined_rules = referenced_rules - defined_rules - {"root"}
if undefined_rules:
raise ValueError("Referenced rules are not defined: "
f"{', '.join(sorted(undefined_rules))}")
raise ValueError(
f"Referenced rules are not defined: {', '.join(sorted(undefined_rules))}"
)
return '\n'.join(output_lines)
return "\n".join(output_lines)
def choice_as_grammar(choice: list[str]) -> str:
def escape_ebnf_string(s: str) -> str:
"""Escape special characters in a EBNF string."""
# Escape double quotes and backslashes
return re.sub(r'(["\\])', r'\\\1', s)
return re.sub(r'(["\\])', r"\\\1", s)
escaped_choices = (escape_ebnf_string(c) for c in choice)
grammar = ('root ::= ' + ' | '.join(f'"{c}"' for c in escaped_choices))
grammar = "root ::= " + " | ".join(f'"{c}"' for c in escaped_choices)
return grammar