Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -12,8 +12,10 @@ from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
|
||||
from vllm.utils import LazyLoader
|
||||
from vllm.v1.structured_output.backend_guidance import GuidanceBackend
|
||||
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
|
||||
StructuredOutputGrammar)
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
)
|
||||
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -48,8 +50,7 @@ class StructuredOutputManager:
|
||||
# - at least 1 CPU
|
||||
# - at most half the number of CPUs or 8, whichever is less
|
||||
max_workers = max(1, min(multiprocessing.cpu_count() // 2, 8))
|
||||
self.executor_for_fillmask = ThreadPoolExecutor(
|
||||
max_workers=max_workers)
|
||||
self.executor_for_fillmask = ThreadPoolExecutor(max_workers=max_workers)
|
||||
|
||||
if not self.vllm_config.model_config.skip_tokenizer_init:
|
||||
# The default max_workers if not specified is the number of
|
||||
@@ -60,12 +61,15 @@ class StructuredOutputManager:
|
||||
max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
|
||||
self.executor = ThreadPoolExecutor(max_workers=max_workers)
|
||||
self.tokenizer = init_tokenizer_from_configs(
|
||||
model_config=self.vllm_config.model_config)
|
||||
reasoning_parser = \
|
||||
self.vllm_config.structured_outputs_config.reasoning_parser
|
||||
model_config=self.vllm_config.model_config
|
||||
)
|
||||
reasoning_parser = (
|
||||
self.vllm_config.structured_outputs_config.reasoning_parser
|
||||
)
|
||||
if reasoning_parser:
|
||||
reasoner_cls = ReasoningParserManager.get_reasoning_parser(
|
||||
reasoning_parser)
|
||||
reasoning_parser
|
||||
)
|
||||
self.reasoner = reasoner_cls(tokenizer=self.tokenizer)
|
||||
|
||||
def grammar_init(self, request: Request) -> None:
|
||||
@@ -73,8 +77,10 @@ class StructuredOutputManager:
|
||||
return
|
||||
|
||||
if TYPE_CHECKING:
|
||||
assert request.sampling_params is not None and \
|
||||
request.sampling_params.structured_outputs is not None
|
||||
assert (
|
||||
request.sampling_params is not None
|
||||
and request.sampling_params.structured_outputs is not None
|
||||
)
|
||||
|
||||
# Initialize the backend the first time it is needed.
|
||||
#
|
||||
@@ -98,8 +104,7 @@ class StructuredOutputManager:
|
||||
vocab_size=vocab_size,
|
||||
)
|
||||
elif backend == "outlines":
|
||||
from vllm.v1.structured_output.backend_outlines import (
|
||||
OutlinesBackend)
|
||||
from vllm.v1.structured_output.backend_outlines import OutlinesBackend
|
||||
|
||||
self.backend = OutlinesBackend(
|
||||
self.vllm_config,
|
||||
@@ -108,15 +113,16 @@ class StructuredOutputManager:
|
||||
)
|
||||
elif backend == "lm-format-enforcer":
|
||||
from vllm.v1.structured_output.backend_lm_format_enforcer import ( # noqa: E501
|
||||
LMFormatEnforcerBackend)
|
||||
LMFormatEnforcerBackend,
|
||||
)
|
||||
|
||||
self.backend = LMFormatEnforcerBackend(
|
||||
self.vllm_config,
|
||||
tokenizer=self.tokenizer,
|
||||
vocab_size=vocab_size,
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported structured output backend: {backend}")
|
||||
raise ValueError(f"Unsupported structured output backend: {backend}")
|
||||
|
||||
grammar = self.executor.submit(self._async_create_grammar, request)
|
||||
request.structured_output_request.grammar = grammar # type: ignore[assignment]
|
||||
@@ -169,8 +175,9 @@ class StructuredOutputManager:
|
||||
|
||||
max_num_spec_tokens = 0
|
||||
if self.vllm_config.speculative_config is not None:
|
||||
max_num_spec_tokens = \
|
||||
max_num_spec_tokens = (
|
||||
self.vllm_config.speculative_config.num_speculative_tokens
|
||||
)
|
||||
|
||||
if self._grammar_bitmask is None:
|
||||
assert self.backend is not None
|
||||
@@ -179,22 +186,23 @@ class StructuredOutputManager:
|
||||
# Allocate a bitmask for each token needing to be checked:
|
||||
# one for each speculative position, and one more for the
|
||||
# bonus token / non-speculative token.
|
||||
self._grammar_bitmask = \
|
||||
self.backend.allocate_token_bitmask(
|
||||
max_batch_size * (1 + max_num_spec_tokens))
|
||||
self._grammar_bitmask = self.backend.allocate_token_bitmask(
|
||||
max_batch_size * (1 + max_num_spec_tokens)
|
||||
)
|
||||
|
||||
# Generate a batched bitmask for all structured output requests.
|
||||
# When speculative decoding is enabled, we need to include multiple
|
||||
# masks for each request, one for each possible bonus token position.
|
||||
# These are stored inline in the tensor and unpacked by the gpu runner.
|
||||
cumulative_index = 0
|
||||
ordered_seq = sorted(structured_output_request_ids.items(),
|
||||
key=lambda x: x[1])
|
||||
ordered_seq = sorted(structured_output_request_ids.items(), key=lambda x: x[1])
|
||||
|
||||
# Optimized parallel filling of bitmasks for
|
||||
# non-spec, large-batch-size cases
|
||||
if len(ordered_seq) > self.fill_bitmask_parallel_threshold and \
|
||||
max_num_spec_tokens == 0:
|
||||
if (
|
||||
len(ordered_seq) > self.fill_bitmask_parallel_threshold
|
||||
and max_num_spec_tokens == 0
|
||||
):
|
||||
promises = []
|
||||
batch = []
|
||||
for req_id, _ in ordered_seq:
|
||||
@@ -205,8 +213,9 @@ class StructuredOutputManager:
|
||||
assert structured_output_request.grammar is not None
|
||||
|
||||
apply_bitmask = self.should_fill_bitmask(request)
|
||||
batch.append((structured_output_request.grammar,
|
||||
cumulative_index, apply_bitmask))
|
||||
batch.append(
|
||||
(structured_output_request.grammar, cumulative_index, apply_bitmask)
|
||||
)
|
||||
if len(batch) == self.fill_bitmask_parallel_batch_size:
|
||||
promises.append(self._async_submit_fill_bitmask(batch))
|
||||
batch = []
|
||||
@@ -232,18 +241,28 @@ class StructuredOutputManager:
|
||||
state_advancements = 0
|
||||
req_tokens = scheduled_spec_decode_tokens.get(req_id, [])
|
||||
for i, token in enumerate(req_tokens + [None]):
|
||||
self._fill_bitmasks([(structured_output_request.grammar,
|
||||
cumulative_index, apply_bitmask)])
|
||||
self._fill_bitmasks(
|
||||
[
|
||||
(
|
||||
structured_output_request.grammar,
|
||||
cumulative_index,
|
||||
apply_bitmask,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
if apply_bitmask and token is not None and \
|
||||
not structured_output_request.grammar.is_terminated():
|
||||
if (
|
||||
apply_bitmask
|
||||
and token is not None
|
||||
and not structured_output_request.grammar.is_terminated()
|
||||
):
|
||||
assert structured_output_request.grammar.accept_tokens(
|
||||
req_id, [token])
|
||||
req_id, [token]
|
||||
)
|
||||
state_advancements += 1
|
||||
cumulative_index += 1
|
||||
if state_advancements > 0:
|
||||
structured_output_request.grammar.rollback(
|
||||
state_advancements)
|
||||
structured_output_request.grammar.rollback(state_advancements)
|
||||
|
||||
bitmask_tensor = self._grammar_bitmask
|
||||
if cumulative_index < bitmask_tensor.shape[0]:
|
||||
@@ -258,8 +277,9 @@ class StructuredOutputManager:
|
||||
if self.reasoner is not None:
|
||||
assert request.structured_output_request is not None
|
||||
if request.structured_output_request.reasoning_ended is None:
|
||||
request.structured_output_request.reasoning_ended = \
|
||||
request.structured_output_request.reasoning_ended = (
|
||||
self.reasoner.is_reasoning_end(request.prompt_token_ids)
|
||||
)
|
||||
return request.structured_output_request.reasoning_ended
|
||||
return True
|
||||
|
||||
|
||||
@@ -14,9 +14,11 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import LazyLoader
|
||||
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
StructuredOutputOptions)
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
StructuredOutputOptions,
|
||||
)
|
||||
from vllm.v1.structured_output.request import get_structured_output_key
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -26,8 +28,7 @@ if TYPE_CHECKING:
|
||||
else:
|
||||
llguidance = LazyLoader("llguidance", globals(), "llguidance")
|
||||
llguidance_hf = LazyLoader("llguidance.hf", globals(), "llguidance.hf")
|
||||
llguidance_torch = LazyLoader("llguidance.torch", globals(),
|
||||
"llguidance.torch")
|
||||
llguidance_torch = LazyLoader("llguidance.torch", globals(), "llguidance.torch")
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -36,16 +37,18 @@ def _walk_json_for_additional_properties(data: object):
|
||||
if isinstance(data, dict):
|
||||
for value in data.values():
|
||||
_walk_json_for_additional_properties(value)
|
||||
if 'additionalProperties' not in data and \
|
||||
('properties' in data or 'patternProperties' in data):
|
||||
data['additionalProperties'] = False
|
||||
if "additionalProperties" not in data and (
|
||||
"properties" in data or "patternProperties" in data
|
||||
):
|
||||
data["additionalProperties"] = False
|
||||
elif isinstance(data, list):
|
||||
for item in data:
|
||||
_walk_json_for_additional_properties(item)
|
||||
|
||||
|
||||
def process_for_additional_properties(
|
||||
guide_json: Union[str, dict[str, Any]]) -> dict[str, Any]:
|
||||
guide_json: Union[str, dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
if isinstance(guide_json, str):
|
||||
guide_json_obj = json.loads(guide_json)
|
||||
else:
|
||||
@@ -57,21 +60,27 @@ def process_for_additional_properties(
|
||||
|
||||
@dataclass
|
||||
class GuidanceBackend(StructuredOutputBackend):
|
||||
|
||||
def __post_init__(self):
|
||||
self.disable_any_whitespace = \
|
||||
self.disable_any_whitespace = (
|
||||
self.vllm_config.structured_outputs_config.disable_any_whitespace
|
||||
self.disable_additional_properties = \
|
||||
)
|
||||
self.disable_additional_properties = (
|
||||
self.vllm_config.structured_outputs_config.disable_additional_properties
|
||||
)
|
||||
|
||||
self.ll_tokenizer = llguidance_hf.from_tokenizer(
|
||||
self.tokenizer, self.vocab_size)
|
||||
self.tokenizer, self.vocab_size
|
||||
)
|
||||
|
||||
def compile_grammar(self, request_type: StructuredOutputOptions,
|
||||
grammar_spec: str) -> StructuredOutputGrammar:
|
||||
def compile_grammar(
|
||||
self, request_type: StructuredOutputOptions, grammar_spec: str
|
||||
) -> StructuredOutputGrammar:
|
||||
self.serialized_grammar = serialize_guidance_grammar(
|
||||
request_type, grammar_spec, self.disable_any_whitespace,
|
||||
self.disable_additional_properties)
|
||||
request_type,
|
||||
grammar_spec,
|
||||
self.disable_any_whitespace,
|
||||
self.disable_additional_properties,
|
||||
)
|
||||
|
||||
ll_matcher = llguidance.LLMatcher(
|
||||
self.ll_tokenizer,
|
||||
@@ -90,7 +99,8 @@ class GuidanceBackend(StructuredOutputBackend):
|
||||
|
||||
def allocate_token_bitmask(self, max_num_seqs: int):
|
||||
return llguidance_torch.allocate_token_bitmask(
|
||||
max_num_seqs, self.ll_tokenizer.vocab_size)
|
||||
max_num_seqs, self.ll_tokenizer.vocab_size
|
||||
)
|
||||
|
||||
def destroy(self):
|
||||
pass
|
||||
@@ -178,15 +188,17 @@ def serialize_guidance_grammar(
|
||||
disable_any_whitespace: bool = False,
|
||||
disable_additional_properties: bool = False,
|
||||
) -> str:
|
||||
|
||||
def _process_schema(grammar_spec: Union[str, dict[str, Any]], ) -> str:
|
||||
def _process_schema(
|
||||
grammar_spec: Union[str, dict[str, Any]],
|
||||
) -> str:
|
||||
if disable_additional_properties:
|
||||
grammar_spec = process_for_additional_properties(grammar_spec)
|
||||
return llguidance.LLMatcher.grammar_from_json_schema(
|
||||
grammar_spec,
|
||||
defaults={
|
||||
"whitespace_flexible": not disable_any_whitespace,
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
if request_type == StructuredOutputOptions.JSON:
|
||||
return _process_schema(grammar_spec)
|
||||
@@ -195,7 +207,8 @@ def serialize_guidance_grammar(
|
||||
'{"type": "object"}',
|
||||
defaults={
|
||||
"whitespace_flexible": not disable_any_whitespace,
|
||||
})
|
||||
},
|
||||
)
|
||||
else:
|
||||
if request_type == StructuredOutputOptions.REGEX:
|
||||
tp = "regex"
|
||||
@@ -215,29 +228,32 @@ def serialize_guidance_grammar(
|
||||
trig = next((t for t in triggers if begin.startswith(t)), None)
|
||||
if trig is None:
|
||||
raise ValueError(
|
||||
f"Trigger {begin} not found in triggers {triggers}")
|
||||
f"Trigger {begin} not found in triggers {triggers}"
|
||||
)
|
||||
tags.append(
|
||||
llguidance.StructTag(
|
||||
trigger=trig,
|
||||
begin=s["begin"],
|
||||
grammar=_process_schema(s["schema"]),
|
||||
end=s["end"],
|
||||
))
|
||||
)
|
||||
)
|
||||
if not tags:
|
||||
raise ValueError(
|
||||
"No structural tags found in the grammar spec.")
|
||||
raise ValueError("No structural tags found in the grammar spec.")
|
||||
return llguidance.StructTag.to_grammar(tags)
|
||||
else:
|
||||
logger.error("Validation should have already occurred. "
|
||||
"Please file an issue.")
|
||||
raise ValueError("grammar is not of valid supported types. "
|
||||
f"({request_type!s})")
|
||||
logger.error(
|
||||
"Validation should have already occurred. Please file an issue."
|
||||
)
|
||||
raise ValueError(
|
||||
f"grammar is not of valid supported types. ({request_type!s})"
|
||||
)
|
||||
return llguidance.grammar_from(tp, grammar_spec)
|
||||
|
||||
|
||||
def validate_guidance_grammar(
|
||||
sampling_params: SamplingParams,
|
||||
tokenizer: Optional[llguidance.LLTokenizer] = None) -> None:
|
||||
sampling_params: SamplingParams, tokenizer: Optional[llguidance.LLTokenizer] = None
|
||||
) -> None:
|
||||
tp, grm = get_structured_output_key(sampling_params)
|
||||
guidance_grm = serialize_guidance_grammar(tp, grm)
|
||||
err = llguidance.LLMatcher.validate_grammar(guidance_grm, tokenizer)
|
||||
|
||||
@@ -13,26 +13,31 @@ from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import LazyLoader
|
||||
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
StructuredOutputOptions)
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
StructuredOutputOptions,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import lmformatenforcer
|
||||
import lmformatenforcer.integrations.vllm as lmfe_vllm
|
||||
else:
|
||||
lmformatenforcer = LazyLoader("lmformatenforcer", globals(),
|
||||
"lmformatenforcer")
|
||||
lmfe_vllm = LazyLoader("lmformatenforcer.integrations.vllm", globals(),
|
||||
"lmformatenforcer.integrations.vllm")
|
||||
lmformatenforcer = LazyLoader("lmformatenforcer", globals(), "lmformatenforcer")
|
||||
lmfe_vllm = LazyLoader(
|
||||
"lmformatenforcer.integrations.vllm",
|
||||
globals(),
|
||||
"lmformatenforcer.integrations.vllm",
|
||||
)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _cached_build_vllm_token_enforcer_tokenizer_data(
|
||||
tokenizer: PreTrainedTokenizerBase,
|
||||
vocab_size: int) -> lmfe_vllm.TokenEnforcerTokenizerData:
|
||||
tokenizer: PreTrainedTokenizerBase, vocab_size: int
|
||||
) -> lmfe_vllm.TokenEnforcerTokenizerData:
|
||||
return lmfe_vllm.build_vllm_token_enforcer_tokenizer_data(
|
||||
tokenizer, use_bitmask=True, vocab_size=vocab_size)
|
||||
tokenizer, use_bitmask=True, vocab_size=vocab_size
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -44,7 +49,8 @@ class LMFormatEnforcerGrammar(StructuredOutputGrammar):
|
||||
original_len = len(self.current_tokens_prefix)
|
||||
for token in tokens:
|
||||
if not self.token_enforcer.get_allowed_tokens(
|
||||
self.current_tokens_prefix).is_token_allowed(token):
|
||||
self.current_tokens_prefix
|
||||
).is_token_allowed(token):
|
||||
# Rollback partial updates to ensure atomicity.
|
||||
del self.current_tokens_prefix[original_len:]
|
||||
return False
|
||||
@@ -56,8 +62,8 @@ class LMFormatEnforcerGrammar(StructuredOutputGrammar):
|
||||
prefix = tokens[:prefix_length]
|
||||
next_token = tokens[prefix_length]
|
||||
if not self.token_enforcer.get_allowed_tokens(
|
||||
self.current_tokens_prefix +
|
||||
prefix).is_token_allowed(next_token):
|
||||
self.current_tokens_prefix + prefix
|
||||
).is_token_allowed(next_token):
|
||||
break
|
||||
else:
|
||||
return tokens
|
||||
@@ -69,14 +75,16 @@ class LMFormatEnforcerGrammar(StructuredOutputGrammar):
|
||||
|
||||
def fill_bitmask(self, bitmask: torch.Tensor, batch_index: int) -> None:
|
||||
allowed_tokens = self.token_enforcer.get_allowed_tokens(
|
||||
self.current_tokens_prefix)
|
||||
self.current_tokens_prefix
|
||||
)
|
||||
bitmask[batch_index] = allowed_tokens.allowed_tokens
|
||||
|
||||
def is_terminated(self) -> bool:
|
||||
# We are considered terminated if the prefix ends with eos_token_id
|
||||
return_value = len(
|
||||
self.current_tokens_prefix) > 0 and self.current_tokens_prefix[
|
||||
-1] == self.token_enforcer.eos_token_id
|
||||
return_value = (
|
||||
len(self.current_tokens_prefix) > 0
|
||||
and self.current_tokens_prefix[-1] == self.token_enforcer.eos_token_id
|
||||
)
|
||||
return return_value
|
||||
|
||||
def reset(self):
|
||||
@@ -85,18 +93,18 @@ class LMFormatEnforcerGrammar(StructuredOutputGrammar):
|
||||
|
||||
@dataclass
|
||||
class LMFormatEnforcerBackend(StructuredOutputBackend):
|
||||
|
||||
def __post_init__(self):
|
||||
self.tokenizer_data = _cached_build_vllm_token_enforcer_tokenizer_data(
|
||||
self.tokenizer, self.vocab_size)
|
||||
self.tokenizer, self.vocab_size
|
||||
)
|
||||
|
||||
def compile_grammar(self, request_type: StructuredOutputOptions,
|
||||
grammar_spec: str) -> StructuredOutputGrammar:
|
||||
def compile_grammar(
|
||||
self, request_type: StructuredOutputOptions, grammar_spec: str
|
||||
) -> StructuredOutputGrammar:
|
||||
character_level_parser: lmformatenforcer.CharacterLevelParser
|
||||
if request_type == StructuredOutputOptions.JSON:
|
||||
spec_dict = json.loads(grammar_spec)
|
||||
character_level_parser = lmformatenforcer.JsonSchemaParser(
|
||||
spec_dict)
|
||||
character_level_parser = lmformatenforcer.JsonSchemaParser(spec_dict)
|
||||
elif request_type == StructuredOutputOptions.JSON_OBJECT:
|
||||
character_level_parser = lmformatenforcer.JsonSchemaParser(None)
|
||||
elif request_type == StructuredOutputOptions.REGEX:
|
||||
@@ -104,14 +112,17 @@ class LMFormatEnforcerBackend(StructuredOutputBackend):
|
||||
elif request_type == StructuredOutputOptions.CHOICE:
|
||||
choices = ast.literal_eval(grammar_spec)
|
||||
character_level_parser = lmformatenforcer.UnionParser(
|
||||
[lmformatenforcer.StringParser(choice) for choice in choices])
|
||||
[lmformatenforcer.StringParser(choice) for choice in choices]
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Invalid request type for LM Format Enforcer backend"
|
||||
f"({request_type!s})")
|
||||
f"Invalid request type for LM Format Enforcer backend({request_type!s})"
|
||||
)
|
||||
max_rollback_tokens = (
|
||||
self.vllm_config.speculative_config.num_speculative_tokens
|
||||
if self.vllm_config.speculative_config is not None else 0)
|
||||
if self.vllm_config.speculative_config is not None
|
||||
else 0
|
||||
)
|
||||
|
||||
if max_rollback_tokens > 0:
|
||||
raise ValueError(
|
||||
@@ -136,8 +147,7 @@ class LMFormatEnforcerBackend(StructuredOutputBackend):
|
||||
pass
|
||||
|
||||
|
||||
def validate_structured_output_request_lm_format_enforcer(
|
||||
params: SamplingParams):
|
||||
def validate_structured_output_request_lm_format_enforcer(params: SamplingParams):
|
||||
if params.structured_outputs is None:
|
||||
return
|
||||
|
||||
@@ -163,5 +173,7 @@ def validate_structured_output_request_lm_format_enforcer(
|
||||
elif so_params.choice:
|
||||
return
|
||||
elif so_params.grammar:
|
||||
raise ValueError("LM Format Enforcer structured outputs backend "
|
||||
"does not support grammar specifications")
|
||||
raise ValueError(
|
||||
"LM Format Enforcer structured outputs backend "
|
||||
"does not support grammar specifications"
|
||||
)
|
||||
|
||||
@@ -15,20 +15,23 @@ from regex import escape as regex_escape
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import LazyLoader
|
||||
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
StructuredOutputOptions)
|
||||
from vllm.v1.structured_output.utils import (OutlinesVocabulary,
|
||||
get_outlines_cache,
|
||||
get_outlines_vocabulary)
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
StructuredOutputOptions,
|
||||
)
|
||||
from vllm.v1.structured_output.utils import (
|
||||
OutlinesVocabulary,
|
||||
get_outlines_cache,
|
||||
get_outlines_vocabulary,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import outlines_core as oc
|
||||
import outlines_core.json_schema as json_schema
|
||||
else:
|
||||
oc = LazyLoader("oc", globals(), "outlines_core")
|
||||
json_schema = LazyLoader("json_schema", globals(),
|
||||
"outlines_core.json_schema")
|
||||
json_schema = LazyLoader("json_schema", globals(), "outlines_core.json_schema")
|
||||
|
||||
# Python 3.11+ sre_parse and sre_constants
|
||||
# are deprecated, so we must import them from re
|
||||
@@ -46,13 +49,13 @@ else:
|
||||
|
||||
@dataclass
|
||||
class OutlinesBackend(StructuredOutputBackend):
|
||||
|
||||
def __post_init__(self):
|
||||
self.vocabulary = get_outlines_vocabulary(self.tokenizer)
|
||||
self.cache = get_outlines_cache()
|
||||
|
||||
def _compile_index(self, regex_string: str,
|
||||
vocabulary: OutlinesVocabulary) -> oc.Index:
|
||||
def _compile_index(
|
||||
self, regex_string: str, vocabulary: OutlinesVocabulary
|
||||
) -> oc.Index:
|
||||
cache_key = f"{vocabulary._hash}_{regex_string}"
|
||||
if cache_key in self.cache:
|
||||
return self.cache[cache_key]
|
||||
@@ -62,8 +65,9 @@ class OutlinesBackend(StructuredOutputBackend):
|
||||
|
||||
return index
|
||||
|
||||
def compile_grammar(self, request_type: StructuredOutputOptions,
|
||||
grammar_spec: str) -> StructuredOutputGrammar:
|
||||
def compile_grammar(
|
||||
self, request_type: StructuredOutputOptions, grammar_spec: str
|
||||
) -> StructuredOutputGrammar:
|
||||
if request_type == StructuredOutputOptions.JSON:
|
||||
regex = json_schema.build_regex_from_schema(grammar_spec)
|
||||
elif request_type == StructuredOutputOptions.REGEX:
|
||||
@@ -79,10 +83,13 @@ class OutlinesBackend(StructuredOutputBackend):
|
||||
index = self._compile_index(regex, self.vocabulary)
|
||||
max_rollback_tokens = (
|
||||
self.vllm_config.speculative_config.num_speculative_tokens
|
||||
if self.vllm_config.speculative_config is not None else 0)
|
||||
return OutlinesGrammar(vocab_size=self.vocab_size,
|
||||
guide=oc.Guide(
|
||||
index, max_rollback=max_rollback_tokens))
|
||||
if self.vllm_config.speculative_config is not None
|
||||
else 0
|
||||
)
|
||||
return OutlinesGrammar(
|
||||
vocab_size=self.vocab_size,
|
||||
guide=oc.Guide(index, max_rollback=max_rollback_tokens),
|
||||
)
|
||||
|
||||
def allocate_token_bitmask(self, max_num_seqs: int) -> torch.Tensor:
|
||||
return torch.full(
|
||||
@@ -98,20 +105,15 @@ class OutlinesBackend(StructuredOutputBackend):
|
||||
|
||||
@dataclass
|
||||
class OutlinesGrammar(StructuredOutputGrammar):
|
||||
|
||||
vocab_size: int
|
||||
guide: oc.Guide = field(hash=False)
|
||||
num_processed_tokens: int = field(default_factory=lambda: 0,
|
||||
repr=False,
|
||||
hash=False,
|
||||
init=False)
|
||||
num_processed_tokens: int = field(
|
||||
default_factory=lambda: 0, repr=False, hash=False, init=False
|
||||
)
|
||||
|
||||
# outlines_core signals done on DFA accept; vLLM expects done after EOS.
|
||||
# We delay the finished flag by one step so EOS can still be emitted.
|
||||
_prev_finished: bool = field(default=False,
|
||||
init=False,
|
||||
repr=False,
|
||||
hash=False)
|
||||
_prev_finished: bool = field(default=False, init=False, repr=False, hash=False)
|
||||
|
||||
def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
|
||||
"""Accepts a list of tokens and advances the FSM.
|
||||
@@ -142,8 +144,7 @@ class OutlinesGrammar(StructuredOutputGrammar):
|
||||
|
||||
def fill_bitmask(self, bitmask: torch.Tensor, idx: int) -> None:
|
||||
mask = bitmask[idx]
|
||||
self.guide.write_mask_into(mask.data_ptr(), mask.numel(),
|
||||
mask.element_size())
|
||||
self.guide.write_mask_into(mask.data_ptr(), mask.numel(), mask.element_size())
|
||||
|
||||
def is_terminated(self) -> bool:
|
||||
curr = self.guide.is_finished()
|
||||
@@ -187,8 +188,10 @@ def validate_structured_output_request_outlines(params: SamplingParams):
|
||||
regex = "(" + "|".join(choices) + ")"
|
||||
validate_regex_is_buildable(regex)
|
||||
elif so_params.grammar:
|
||||
raise ValueError("Outlines structured outputs backend "
|
||||
"does not support grammar specifications")
|
||||
raise ValueError(
|
||||
"Outlines structured outputs backend "
|
||||
"does not support grammar specifications"
|
||||
)
|
||||
|
||||
|
||||
def _prefix_needs_context(parsed) -> bool:
|
||||
@@ -196,7 +199,7 @@ def _prefix_needs_context(parsed) -> bool:
|
||||
|
||||
def subpattern_consumes(parsed) -> bool:
|
||||
"""Return True if subpattern can consume at least one character."""
|
||||
tokens = parsed.data if hasattr(parsed, 'data') else parsed
|
||||
tokens = parsed.data if hasattr(parsed, "data") else parsed
|
||||
for ttype, tval in tokens:
|
||||
# literal, character class, or dot always consumes
|
||||
if ttype in (sre_parse.LITERAL, sre_parse.IN, sre_parse.ANY):
|
||||
@@ -212,17 +215,18 @@ def _prefix_needs_context(parsed) -> bool:
|
||||
if any(subpattern_consumes(br) for br in branches):
|
||||
return True
|
||||
# grouped subpattern: recurse into its contents
|
||||
elif ttype == sre_parse.SUBPATTERN and subpattern_consumes(
|
||||
tval[3]):
|
||||
elif ttype == sre_parse.SUBPATTERN and subpattern_consumes(tval[3]):
|
||||
return True
|
||||
# No consumers, return False
|
||||
return False
|
||||
|
||||
tokens = parsed.data if hasattr(parsed, 'data') else parsed
|
||||
tokens = parsed.data if hasattr(parsed, "data") else parsed
|
||||
for ttype, tval in tokens:
|
||||
# Direct anchors or look-around
|
||||
if ttype == sre_parse.AT or ttype in (sre_constants.ASSERT,
|
||||
sre_constants.ASSERT_NOT):
|
||||
if ttype == sre_parse.AT or ttype in (
|
||||
sre_constants.ASSERT,
|
||||
sre_constants.ASSERT_NOT,
|
||||
):
|
||||
return True
|
||||
|
||||
# Nested subpattern: check
|
||||
@@ -261,9 +265,8 @@ def _prefix_needs_context(parsed) -> bool:
|
||||
|
||||
def _check_unsupported(parsed) -> None:
|
||||
"""Check for regex features unsupported by regex-automata"""
|
||||
tokens = parsed.data if hasattr(parsed, 'data') else parsed
|
||||
tokens = parsed.data if hasattr(parsed, "data") else parsed
|
||||
for ttype, tval in tokens:
|
||||
|
||||
# backreference
|
||||
if ttype in (sre_parse.GROUPREF, sre_parse.GROUPREF_EXISTS):
|
||||
raise ValueError("Backreferences are unsupported.")
|
||||
@@ -274,8 +277,7 @@ def _check_unsupported(parsed) -> None:
|
||||
|
||||
# unicode word boundaries
|
||||
elif ttype == sre_parse.AT:
|
||||
if tval in (sre_constants.AT_BOUNDARY,
|
||||
sre_constants.AT_NON_BOUNDARY):
|
||||
if tval in (sre_constants.AT_BOUNDARY, sre_constants.AT_NON_BOUNDARY):
|
||||
raise ValueError("Unicode word boundaries are unsupported.")
|
||||
|
||||
elif ttype == sre_parse.BRANCH:
|
||||
@@ -308,7 +310,8 @@ def validate_regex_is_buildable(pattern: str) -> None:
|
||||
raise ValueError(
|
||||
f"Regex uses unsupported feature for structured outputs: {e}. "
|
||||
"Only basic matching constructs are supported—lookarounds, "
|
||||
"backreferences, and unicode boundaries are not.") from e
|
||||
"backreferences, and unicode boundaries are not."
|
||||
) from e
|
||||
|
||||
if _prefix_needs_context(parsed):
|
||||
raise ValueError(
|
||||
@@ -317,4 +320,5 @@ def validate_regex_is_buildable(pattern: str) -> None:
|
||||
"in a way which requires context before any token is matched."
|
||||
"structured outputs needs regexes that can match without needing "
|
||||
"that context. Try rewriting the pattern without using these "
|
||||
f"constructs. Pattern:\n{pattern}")
|
||||
f"constructs. Pattern:\n{pattern}"
|
||||
)
|
||||
|
||||
@@ -103,8 +103,9 @@ class StructuredOutputBackend(ABC):
|
||||
vocab_size: int
|
||||
|
||||
@abstractmethod
|
||||
def compile_grammar(self, request_type: StructuredOutputOptions,
|
||||
grammar_spec: str) -> StructuredOutputGrammar:
|
||||
def compile_grammar(
|
||||
self, request_type: StructuredOutputOptions, grammar_spec: str
|
||||
) -> StructuredOutputGrammar:
|
||||
"""
|
||||
Compiles a grammar specification into a structured output grammar.
|
||||
|
||||
|
||||
@@ -14,12 +14,16 @@ from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.utils import LazyLoader
|
||||
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
StructuredOutputOptions)
|
||||
from vllm.v1.structured_output.utils import (choice_as_grammar,
|
||||
convert_lark_to_ebnf,
|
||||
grammar_is_likely_lark)
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputBackend,
|
||||
StructuredOutputGrammar,
|
||||
StructuredOutputOptions,
|
||||
)
|
||||
from vllm.v1.structured_output.utils import (
|
||||
choice_as_grammar,
|
||||
convert_lark_to_ebnf,
|
||||
grammar_is_likely_lark,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import xgrammar as xgr
|
||||
@@ -31,10 +35,10 @@ logger = init_logger(__name__)
|
||||
|
||||
@dataclass
|
||||
class XgrammarBackend(StructuredOutputBackend):
|
||||
|
||||
def __post_init__(self):
|
||||
self.disable_any_whitespace = \
|
||||
self.disable_any_whitespace = (
|
||||
self.vllm_config.structured_outputs_config.disable_any_whitespace
|
||||
)
|
||||
|
||||
if isinstance(self.tokenizer, MistralTokenizer):
|
||||
# NOTE: ideally, xgrammar should handle this accordingly.
|
||||
@@ -44,27 +48,33 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
encoded_vocab = self.tokenizer._vocab
|
||||
else:
|
||||
encoded_vocab = [
|
||||
token for token, _ in sorted(
|
||||
token
|
||||
for token, _ in sorted(
|
||||
self.tokenizer.get_vocab().items(),
|
||||
key=lambda x: x[1],
|
||||
)
|
||||
]
|
||||
stop_token_ids = None
|
||||
if (hasattr(
|
||||
if (
|
||||
hasattr(
|
||||
self.tokenizer,
|
||||
"eos_token_id",
|
||||
) and self.tokenizer.eos_token_id is not None):
|
||||
)
|
||||
and self.tokenizer.eos_token_id is not None
|
||||
):
|
||||
stop_token_ids = [self.tokenizer.eos_token_id]
|
||||
except AttributeError as e:
|
||||
raise ValueError(
|
||||
f"Cannot get the vocabulary of the tokenizer "
|
||||
f"{type(self.tokenizer)}. The tokenizer should have a "
|
||||
"get_vocab method.") from e
|
||||
"get_vocab method."
|
||||
) from e
|
||||
tokenizer_info = xgr.TokenizerInfo( # type: ignore
|
||||
encoded_vocab=encoded_vocab,
|
||||
# NOTE: https://github.com/mlc-ai/xgrammar/blob/5e141f6ff1ca02bc31f9e512e68b61f2a8ae88e5/tests/python/test_tokenizer_info.py#L43 # noqa: E501
|
||||
vocab_type=xgr.VocabType.RAW
|
||||
if self.tokenizer.is_tekken else xgr.VocabType.BYTE_FALLBACK,
|
||||
if self.tokenizer.is_tekken
|
||||
else xgr.VocabType.BYTE_FALLBACK,
|
||||
vocab_size=self.vocab_size,
|
||||
stop_token_ids=stop_token_ids,
|
||||
add_prefix_space=True,
|
||||
@@ -83,18 +93,21 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
|
||||
self.num_speculative_tokens = 0
|
||||
if self.vllm_config.speculative_config is not None:
|
||||
self.num_speculative_tokens = \
|
||||
self.num_speculative_tokens = (
|
||||
self.vllm_config.speculative_config.num_speculative_tokens
|
||||
)
|
||||
|
||||
def compile_grammar(self, request_type: StructuredOutputOptions,
|
||||
grammar_spec: str) -> StructuredOutputGrammar:
|
||||
def compile_grammar(
|
||||
self, request_type: StructuredOutputOptions, grammar_spec: str
|
||||
) -> StructuredOutputGrammar:
|
||||
if request_type == StructuredOutputOptions.JSON:
|
||||
ctx = self.compiler.compile_json_schema(
|
||||
grammar_spec, any_whitespace=not self.disable_any_whitespace)
|
||||
grammar_spec, any_whitespace=not self.disable_any_whitespace
|
||||
)
|
||||
elif request_type == StructuredOutputOptions.JSON_OBJECT:
|
||||
ctx = self.compiler.compile_json_schema(
|
||||
'{"type": "object"}',
|
||||
any_whitespace=not self.disable_any_whitespace)
|
||||
'{"type": "object"}', any_whitespace=not self.disable_any_whitespace
|
||||
)
|
||||
elif request_type == StructuredOutputOptions.GRAMMAR:
|
||||
ctx = self.compiler.compile_grammar(grammar_spec)
|
||||
elif request_type == StructuredOutputOptions.REGEX:
|
||||
@@ -106,17 +119,20 @@ class XgrammarBackend(StructuredOutputBackend):
|
||||
begin=s["begin"],
|
||||
schema=json.dumps(s["schema"]),
|
||||
end=s["end"],
|
||||
) for s in s_tag["structures"]
|
||||
)
|
||||
for s in s_tag["structures"]
|
||||
]
|
||||
structural_tag = xgr.StructuralTag.from_legacy_structural_tag(
|
||||
tags, s_tag["triggers"])
|
||||
tags, s_tag["triggers"]
|
||||
)
|
||||
ctx = self.compiler.compile_structural_tag(structural_tag)
|
||||
else:
|
||||
logger.error(
|
||||
"Validation should have already occurred. Please file an issue."
|
||||
)
|
||||
raise ValueError(
|
||||
f"grammar is not of valid supported types. ({request_type!s})")
|
||||
f"grammar is not of valid supported types. ({request_type!s})"
|
||||
)
|
||||
|
||||
return XgrammarGrammar(
|
||||
matcher=xgr.GrammarMatcher(
|
||||
@@ -146,10 +162,9 @@ class XgrammarGrammar(StructuredOutputGrammar):
|
||||
vocab_size: int
|
||||
matcher: xgr.GrammarMatcher = field(hash=False)
|
||||
ctx: xgr.CompiledGrammar = field(hash=False)
|
||||
num_processed_tokens: int = field(default_factory=lambda: 0,
|
||||
repr=False,
|
||||
hash=False,
|
||||
init=False)
|
||||
num_processed_tokens: int = field(
|
||||
default_factory=lambda: 0, repr=False, hash=False, init=False
|
||||
)
|
||||
_is_terminated: bool = field(default=False, repr=False, hash=False)
|
||||
|
||||
def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
|
||||
@@ -164,7 +179,10 @@ class XgrammarGrammar(StructuredOutputGrammar):
|
||||
if not self.matcher.accept_token(token):
|
||||
logger.error(
|
||||
"Failed to advance FSM for request %s "
|
||||
"for tokens %s. Please file an issue.", request_id, token)
|
||||
"for tokens %s. Please file an issue.",
|
||||
request_id,
|
||||
token,
|
||||
)
|
||||
return False
|
||||
self.num_processed_tokens += 1
|
||||
self._is_terminated = self.matcher.is_terminated()
|
||||
@@ -216,8 +234,9 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
|
||||
|
||||
# Check for array unsupported keywords
|
||||
if obj.get("type") == "array" and any(
|
||||
key in obj for key in ("uniqueItems", "contains",
|
||||
"minContains", "maxContains")):
|
||||
key in obj
|
||||
for key in ("uniqueItems", "contains", "minContains", "maxContains")
|
||||
):
|
||||
return True
|
||||
|
||||
# Unsupported keywords for strings
|
||||
@@ -226,8 +245,14 @@ def has_xgrammar_unsupported_json_features(schema: dict[str, Any]) -> bool:
|
||||
|
||||
# Unsupported keywords for objects
|
||||
if obj.get("type") == "object" and any(
|
||||
key in obj for key in ("minProperties", "maxProperties",
|
||||
"propertyNames", "patternProperties")):
|
||||
key in obj
|
||||
for key in (
|
||||
"minProperties",
|
||||
"maxProperties",
|
||||
"propertyNames",
|
||||
"patternProperties",
|
||||
)
|
||||
):
|
||||
return True
|
||||
|
||||
# Recursively check all nested objects and arrays
|
||||
@@ -259,16 +284,18 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
try:
|
||||
xgr.Grammar.from_regex(so_params.regex)
|
||||
except Exception as err:
|
||||
raise ValueError("Failed to transform regex into a grammar: "
|
||||
f"{err}") from err
|
||||
raise ValueError(
|
||||
f"Failed to transform regex into a grammar: {err}"
|
||||
) from err
|
||||
|
||||
if so_params.choice:
|
||||
choice_grammar = choice_as_grammar(so_params.choice)
|
||||
try:
|
||||
xgr.Grammar.from_ebnf(choice_grammar)
|
||||
except Exception as err:
|
||||
raise ValueError("Failed to transform choices into a grammar: "
|
||||
"{err}") from err
|
||||
raise ValueError(
|
||||
"Failed to transform choices into a grammar: {err}"
|
||||
) from err
|
||||
so_params.choice = None
|
||||
so_params.grammar = choice_grammar
|
||||
return
|
||||
@@ -285,12 +312,14 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
try:
|
||||
xgr.Grammar.from_json_schema(schema)
|
||||
except Exception as err:
|
||||
raise ValueError("Failed to transform json schema into a grammar: "
|
||||
f"{err}") from err
|
||||
raise ValueError(
|
||||
f"Failed to transform json schema into a grammar: {err}"
|
||||
) from err
|
||||
|
||||
if has_xgrammar_unsupported_json_features(schema):
|
||||
raise ValueError("The provided JSON schema contains features not "
|
||||
"supported by xgrammar.")
|
||||
raise ValueError(
|
||||
"The provided JSON schema contains features not supported by xgrammar."
|
||||
)
|
||||
return
|
||||
|
||||
if so_params.grammar:
|
||||
@@ -300,7 +329,8 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
so_params.grammar = convert_lark_to_ebnf(so_params.grammar)
|
||||
except ValueError as e:
|
||||
raise ValueError(
|
||||
"Failed to convert the grammar from Lark to EBNF. ") from e
|
||||
"Failed to convert the grammar from Lark to EBNF. "
|
||||
) from e
|
||||
|
||||
# Test parsing EBNF grammar, possibly already converted from Lark
|
||||
try:
|
||||
@@ -318,10 +348,12 @@ def validate_xgrammar_grammar(sampling_params: SamplingParams) -> None:
|
||||
begin=s["begin"],
|
||||
schema=json.dumps(s["schema"]),
|
||||
end=s["end"],
|
||||
) for s in s_tag["structures"]
|
||||
)
|
||||
for s in s_tag["structures"]
|
||||
]
|
||||
structural_tag = xgr.StructuralTag.from_legacy_structural_tag(
|
||||
tags, s_tag["triggers"])
|
||||
tags, s_tag["triggers"]
|
||||
)
|
||||
xgr.Grammar.from_structural_tag(structural_tag)
|
||||
except Exception as e:
|
||||
raise ValueError("Invalid structural tag specification.") from e
|
||||
|
||||
@@ -10,17 +10,19 @@ from concurrent.futures._base import TimeoutError
|
||||
from typing import Optional, Union, cast
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.structured_output.backend_types import (StructuredOutputGrammar,
|
||||
StructuredOutputKey,
|
||||
StructuredOutputOptions)
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputGrammar,
|
||||
StructuredOutputKey,
|
||||
StructuredOutputOptions,
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class StructuredOutputRequest:
|
||||
|
||||
sampling_params: SamplingParams
|
||||
_grammar: Optional[Union[Future[StructuredOutputGrammar],
|
||||
StructuredOutputGrammar]] = None
|
||||
_grammar: Optional[
|
||||
Union[Future[StructuredOutputGrammar], StructuredOutputGrammar]
|
||||
] = None
|
||||
reasoning_ended: Optional[bool] = None
|
||||
|
||||
def _check_grammar_completion(self) -> bool:
|
||||
@@ -43,13 +45,15 @@ class StructuredOutputRequest:
|
||||
@property
|
||||
def grammar(self) -> Optional[StructuredOutputGrammar]:
|
||||
completed = self._check_grammar_completion()
|
||||
return cast(Optional[StructuredOutputGrammar],
|
||||
self._grammar) if completed else None
|
||||
return (
|
||||
cast(Optional[StructuredOutputGrammar], self._grammar)
|
||||
if completed
|
||||
else None
|
||||
)
|
||||
|
||||
@grammar.setter
|
||||
def grammar(
|
||||
self, grammar: Union[StructuredOutputGrammar,
|
||||
Future[StructuredOutputGrammar]]
|
||||
self, grammar: Union[StructuredOutputGrammar, Future[StructuredOutputGrammar]]
|
||||
) -> None:
|
||||
self._grammar = grammar
|
||||
|
||||
@@ -58,8 +62,7 @@ class StructuredOutputRequest:
|
||||
return get_structured_output_key(self.sampling_params)
|
||||
|
||||
|
||||
def get_structured_output_key(
|
||||
sampling_params: SamplingParams) -> StructuredOutputKey:
|
||||
def get_structured_output_key(sampling_params: SamplingParams) -> StructuredOutputKey:
|
||||
params = sampling_params.structured_outputs
|
||||
assert params is not None, "params can't be None."
|
||||
if params.json is not None:
|
||||
|
||||
@@ -76,27 +76,31 @@ def apply_grammar_bitmask(
|
||||
for req_id, batch_index in seq:
|
||||
logit_index = batch_index + cumulative_offset
|
||||
cumulative_offset += len(
|
||||
scheduler_output.scheduled_spec_decode_tokens.get(req_id, []))
|
||||
scheduler_output.scheduled_spec_decode_tokens.get(req_id, [])
|
||||
)
|
||||
if req_id in scheduler_output.structured_output_request_ids:
|
||||
struct_out_req_batch_indices[req_id] = logit_index
|
||||
|
||||
out_indices = []
|
||||
|
||||
# Reorder the bitmask to match the order of the requests in the batch.
|
||||
sorted_bitmask = np.full(shape=(logits.shape[0], grammar_bitmask.shape[1]),
|
||||
fill_value=-1,
|
||||
dtype=grammar_bitmask.dtype)
|
||||
sorted_bitmask = np.full(
|
||||
shape=(logits.shape[0], grammar_bitmask.shape[1]),
|
||||
fill_value=-1,
|
||||
dtype=grammar_bitmask.dtype,
|
||||
)
|
||||
cumulative_index = 0
|
||||
seq = sorted(scheduler_output.structured_output_request_ids.items(),
|
||||
key=lambda x: x[1])
|
||||
seq = sorted(
|
||||
scheduler_output.structured_output_request_ids.items(), key=lambda x: x[1]
|
||||
)
|
||||
for req_id, _ in seq:
|
||||
num_spec_tokens = len(
|
||||
scheduler_output.scheduled_spec_decode_tokens.get(req_id, []))
|
||||
scheduler_output.scheduled_spec_decode_tokens.get(req_id, [])
|
||||
)
|
||||
if req_id in struct_out_req_batch_indices:
|
||||
logit_index = struct_out_req_batch_indices[req_id]
|
||||
for i in range(1 + num_spec_tokens):
|
||||
sorted_bitmask[logit_index + i] = \
|
||||
grammar_bitmask[cumulative_index + i]
|
||||
sorted_bitmask[logit_index + i] = grammar_bitmask[cumulative_index + i]
|
||||
out_indices.append(logit_index + i)
|
||||
cumulative_index += 1 + num_spec_tokens
|
||||
grammar_bitmask = sorted_bitmask
|
||||
@@ -128,8 +132,7 @@ class OutlinesVocabulary:
|
||||
self.inner = vocabulary
|
||||
# Have to do abs(hash()) because python hashes can
|
||||
# be negative, and we are using hash as a cache key.
|
||||
hex_str = hashlib.sha256(
|
||||
vocabulary.__repr__().encode('utf-8')).hexdigest()
|
||||
hex_str = hashlib.sha256(vocabulary.__repr__().encode("utf-8")).hexdigest()
|
||||
hash_int = int(hex_str, 16)
|
||||
self._hash = hash_int
|
||||
|
||||
@@ -165,16 +168,18 @@ def get_outlines_cache():
|
||||
|
||||
cache_dir = get_outlines_cache_path()
|
||||
if envs.VLLM_V1_USE_OUTLINES_CACHE:
|
||||
logger.warning("Enabling outlines cache. This is an unbounded on-disk "
|
||||
"cache. It may consume a lot of disk space and should "
|
||||
"not be used with untrusted clients.")
|
||||
logger.warning(
|
||||
"Enabling outlines cache. This is an unbounded on-disk "
|
||||
"cache. It may consume a lot of disk space and should "
|
||||
"not be used with untrusted clients."
|
||||
)
|
||||
cache = Cache(cache_dir, eviction_policy="none", cull_limit=0)
|
||||
outlines_version = importlib.metadata.version("outlines_core")
|
||||
|
||||
cached_version = cache.get('__version__', None)
|
||||
cached_version = cache.get("__version__", None)
|
||||
if cached_version != outlines_version:
|
||||
cache.clear()
|
||||
cache.set('__version__', outlines_version)
|
||||
cache.set("__version__", outlines_version)
|
||||
return cache
|
||||
else:
|
||||
return LRUCache(maxsize=128)
|
||||
@@ -194,19 +199,17 @@ def _reduced_vocabulary(
|
||||
A Dict of token string -> equivalent token ids
|
||||
"""
|
||||
|
||||
unicode_to_bytes = {
|
||||
v: k
|
||||
for k, v in tokenization_gpt2.bytes_to_unicode().items()
|
||||
}
|
||||
unicode_to_bytes = {v: k for k, v in tokenization_gpt2.bytes_to_unicode().items()}
|
||||
|
||||
def convert_token_to_string(token: str) -> str:
|
||||
|
||||
string = tokenizer.convert_tokens_to_string([token])
|
||||
|
||||
# A hack to handle missing spaces to HF's Llama tokenizers
|
||||
if (type(token) is str
|
||||
and token.startswith(file_utils.SPIECE_UNDERLINE)
|
||||
or token == "<0x20>"):
|
||||
if (
|
||||
type(token) is str
|
||||
and token.startswith(file_utils.SPIECE_UNDERLINE)
|
||||
or token == "<0x20>"
|
||||
):
|
||||
return " " + string
|
||||
|
||||
return string
|
||||
@@ -226,8 +229,7 @@ def _reduced_vocabulary(
|
||||
# by this point.
|
||||
token_bytes = bytes(token_str) # type: ignore[arg-type]
|
||||
|
||||
elif "\ufffd" in token_str and not re_replacement_seq.match(
|
||||
token_str):
|
||||
elif "\ufffd" in token_str and not re_replacement_seq.match(token_str):
|
||||
# Handle tokens with invalid UTF-8 sequences.
|
||||
if re_llama_byte_token.match(token):
|
||||
# Llama-like tokenizers use <0xXX> for incomplete sequences.
|
||||
@@ -238,12 +240,13 @@ def _reduced_vocabulary(
|
||||
if None in byte_vals:
|
||||
raise RuntimeError(
|
||||
f"Cannot convert token `{token}`"
|
||||
f" ({token_idx}) to bytes: {token_str}")
|
||||
f" ({token_idx}) to bytes: {token_str}"
|
||||
)
|
||||
# safe to ignore, since if None in byte_vals,
|
||||
# an error is thrown.
|
||||
token_bytes = bytes(byte_vals) # type: ignore[arg-type]
|
||||
else:
|
||||
token_bytes = token_str.encode('utf-8')
|
||||
token_bytes = token_str.encode("utf-8")
|
||||
|
||||
if token_idx != eos_token_id:
|
||||
vocabulary.setdefault(token_bytes, []).append(token_idx)
|
||||
@@ -254,16 +257,18 @@ def _reduced_vocabulary(
|
||||
|
||||
|
||||
def get_outlines_vocabulary(tokenizer: AnyTokenizer) -> oc.Vocabulary:
|
||||
"""Get the `Vocabulary` object for a given tokenizer.
|
||||
"""
|
||||
"""Get the `Vocabulary` object for a given tokenizer."""
|
||||
if hasattr(tokenizer, "_outlines_vocabulary"):
|
||||
return tokenizer._outlines_vocabulary # type: ignore
|
||||
|
||||
try:
|
||||
if hasattr(
|
||||
if (
|
||||
hasattr(
|
||||
tokenizer,
|
||||
"eos_token_id",
|
||||
) and tokenizer.eos_token_id is not None:
|
||||
)
|
||||
and tokenizer.eos_token_id is not None
|
||||
):
|
||||
eos_token_id = tokenizer.eos_token_id
|
||||
else:
|
||||
raise ValueError(
|
||||
@@ -272,17 +277,18 @@ def get_outlines_vocabulary(tokenizer: AnyTokenizer) -> oc.Vocabulary:
|
||||
|
||||
reduced_vocab = _reduced_vocabulary(
|
||||
tokenizer,
|
||||
eos_token_id #type: ignore
|
||||
eos_token_id, # type: ignore
|
||||
)
|
||||
vocabulary = OutlinesVocabulary(
|
||||
oc.Vocabulary(eos_token_id, reduced_vocab))
|
||||
vocabulary = OutlinesVocabulary(oc.Vocabulary(eos_token_id, reduced_vocab))
|
||||
tokenizer._outlines_vocabulary = vocabulary # type: ignore
|
||||
|
||||
return vocabulary
|
||||
except AttributeError as e:
|
||||
raise ValueError(f"Cannot get the vocabulary of the tokenizer "
|
||||
f"({type(tokenizer)}). The tokenizer should have a "
|
||||
"get_vocab method.") from e
|
||||
raise ValueError(
|
||||
f"Cannot get the vocabulary of the tokenizer "
|
||||
f"({type(tokenizer)}). The tokenizer should have a "
|
||||
"get_vocab method."
|
||||
) from e
|
||||
|
||||
|
||||
def grammar_is_likely_lark(grammar_str: str) -> bool:
|
||||
@@ -304,14 +310,14 @@ def grammar_is_likely_lark(grammar_str: str) -> bool:
|
||||
if not grammar_str or not isinstance(grammar_str, str):
|
||||
return False
|
||||
|
||||
for line in grammar_str.split('\n'):
|
||||
for line in grammar_str.split("\n"):
|
||||
# Remove both comment styles
|
||||
line = re.sub(r'(#|//).*$', '', line).strip()
|
||||
line = re.sub(r"(#|//).*$", "", line).strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Look for EBNF rule definition
|
||||
if '::=' in line:
|
||||
if "::=" in line:
|
||||
return False
|
||||
|
||||
return True
|
||||
@@ -348,40 +354,41 @@ def convert_lark_to_ebnf(grammar_str: str) -> str:
|
||||
|
||||
def clean_line(line: str) -> str:
|
||||
"""Remove comments and whitespace from line."""
|
||||
return re.sub(r'(#|//).*$', '', line).strip()
|
||||
return re.sub(r"(#|//).*$", "", line).strip()
|
||||
|
||||
def check_quotes(text: str, rule_name: str, line_num: int) -> None:
|
||||
"""Validate quote matching in text."""
|
||||
if text.count("'") % 2 != 0 or text.count('"') % 2 != 0:
|
||||
raise ValueError(
|
||||
f"Mismatched quotes in {rule_name} on line {line_num}")
|
||||
raise ValueError(f"Mismatched quotes in {rule_name} on line {line_num}")
|
||||
|
||||
def extract_references(text: str) -> set[str]:
|
||||
"""Extract rule references from text."""
|
||||
# Remove quoted strings and special characters
|
||||
text = re.sub(r'"[^"]*"', '', text)
|
||||
text = re.sub(r'[+*?()|\[\]{}]', ' ', text)
|
||||
return set(re.findall(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b', text))
|
||||
text = re.sub(r'"[^"]*"', "", text)
|
||||
text = re.sub(r"[+*?()|\[\]{}]", " ", text)
|
||||
return set(re.findall(r"\b[a-zA-Z_][a-zA-Z0-9_]*\b", text))
|
||||
|
||||
# First pass: Find root rule and validate rule definitions
|
||||
lines = [clean_line(line) for line in grammar_str.split('\n')]
|
||||
lines = [clean_line(line) for line in grammar_str.split("\n")]
|
||||
first_rule = None
|
||||
|
||||
for line_num, line in enumerate(lines, 1):
|
||||
if not line or line.startswith('|'):
|
||||
if not line or line.startswith("|"):
|
||||
continue
|
||||
|
||||
if ':' in line:
|
||||
if ":" in line:
|
||||
try:
|
||||
name = line.split(':', 1)[0].strip().strip('?')
|
||||
name = line.split(":", 1)[0].strip().strip("?")
|
||||
defined_rules.add(name)
|
||||
if first_rule is None:
|
||||
first_rule = name
|
||||
if name == 'start':
|
||||
first_rule = 'start'
|
||||
if name == "start":
|
||||
first_rule = "start"
|
||||
except IndexError as e:
|
||||
raise ValueError(f"Invalid rule format on line {line_num}. "
|
||||
"Expected 'rule_name: definition'") from e
|
||||
raise ValueError(
|
||||
f"Invalid rule format on line {line_num}. "
|
||||
"Expected 'rule_name: definition'"
|
||||
) from e
|
||||
|
||||
if not defined_rules:
|
||||
raise ValueError("No valid rules found in grammar")
|
||||
@@ -398,29 +405,33 @@ def convert_lark_to_ebnf(grammar_str: str) -> str:
|
||||
continue
|
||||
|
||||
try:
|
||||
if ':' in line and not line.startswith('|'):
|
||||
if ":" in line and not line.startswith("|"):
|
||||
# Save previous rule if exists
|
||||
if current_rule:
|
||||
output_lines.append(
|
||||
f"{current_rule} ::= {' | '.join(current_definition)}")
|
||||
f"{current_rule} ::= {' | '.join(current_definition)}"
|
||||
)
|
||||
|
||||
# Process new rule
|
||||
name, definition = line.split(':', 1)
|
||||
current_rule = name.strip().strip('?')
|
||||
name, definition = line.split(":", 1)
|
||||
current_rule = name.strip().strip("?")
|
||||
|
||||
check_quotes(definition, f"rule '{current_rule}'", line_num)
|
||||
definition = re.sub(r"'([^']*)'", r'"\1"', definition)
|
||||
referenced_rules.update(extract_references(definition))
|
||||
current_definition = [definition.strip()]
|
||||
|
||||
elif line.startswith('|'):
|
||||
elif line.startswith("|"):
|
||||
if not current_rule:
|
||||
raise ValueError(f"Alternative '|' on line {line_num} "
|
||||
"without a preceding rule definition")
|
||||
raise ValueError(
|
||||
f"Alternative '|' on line {line_num} "
|
||||
"without a preceding rule definition"
|
||||
)
|
||||
|
||||
alt_def = line[1:].strip()
|
||||
check_quotes(alt_def, f"alternative for rule '{current_rule}'",
|
||||
line_num)
|
||||
check_quotes(
|
||||
alt_def, f"alternative for rule '{current_rule}'", line_num
|
||||
)
|
||||
alt_def = re.sub(r"'([^']*)'", r'"\1"', alt_def)
|
||||
referenced_rules.update(extract_references(alt_def))
|
||||
current_definition.append(alt_def)
|
||||
@@ -430,25 +441,24 @@ def convert_lark_to_ebnf(grammar_str: str) -> str:
|
||||
|
||||
# Add final rule if exists
|
||||
if current_rule:
|
||||
output_lines.append(
|
||||
f"{current_rule} ::= {' | '.join(current_definition)}")
|
||||
output_lines.append(f"{current_rule} ::= {' | '.join(current_definition)}")
|
||||
|
||||
# Validate all rules are defined
|
||||
undefined_rules = referenced_rules - defined_rules - {'root'}
|
||||
undefined_rules = referenced_rules - defined_rules - {"root"}
|
||||
if undefined_rules:
|
||||
raise ValueError("Referenced rules are not defined: "
|
||||
f"{', '.join(sorted(undefined_rules))}")
|
||||
raise ValueError(
|
||||
f"Referenced rules are not defined: {', '.join(sorted(undefined_rules))}"
|
||||
)
|
||||
|
||||
return '\n'.join(output_lines)
|
||||
return "\n".join(output_lines)
|
||||
|
||||
|
||||
def choice_as_grammar(choice: list[str]) -> str:
|
||||
|
||||
def escape_ebnf_string(s: str) -> str:
|
||||
"""Escape special characters in a EBNF string."""
|
||||
# Escape double quotes and backslashes
|
||||
return re.sub(r'(["\\])', r'\\\1', s)
|
||||
return re.sub(r'(["\\])', r"\\\1", s)
|
||||
|
||||
escaped_choices = (escape_ebnf_string(c) for c in choice)
|
||||
grammar = ('root ::= ' + ' | '.join(f'"{c}"' for c in escaped_choices))
|
||||
grammar = "root ::= " + " | ".join(f'"{c}"' for c in escaped_choices)
|
||||
return grammar
|
||||
|
||||
Reference in New Issue
Block a user