# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Sampling parameters for text generation.""" import copy import json as json_mod import math from dataclasses import field from enum import Enum, IntEnum from functools import cached_property from typing import Annotated, Any import msgspec import numpy as np from pydantic import BeforeValidator, GetPydanticSchema, StrictInt from pydantic.dataclasses import dataclass import vllm.envs as envs from vllm.config import ModelConfig, SpeculativeConfig, StructuredOutputsConfig from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike from vllm.utils.mistral import is_mistral_tokenizer from vllm.utils.torch_utils import make_ndarray_with_pad from vllm.v1.serial_utils import PydanticMsgspecMixin logger = init_logger(__name__) _SAMPLING_EPS = 1e-5 _MAX_TEMP = 1e-2 MAX_LOGPROB_TOKEN_IDS = 128 """Upper bound on `SamplingParams.logprob_token_ids` list length. Must match the per-request row width allocated by the sampler's `LogprobTokenIdsState`.""" MAX_STRUCTURED_OUTPUT_JSON_NESTING = 64 """Upper bound on nested objects and arrays in a structured-output JSON schema or structural tag. Grammar conversion runs on the API server and its cost grows steeply with nesting. OpenAI allows 10 levels of schema nesting.""" STRUCTURAL_TAG_WRAPPER_NESTING = 16 """Extra nesting allowed for a structural tag, which wraps its JSON schemas in tag structure (up to 15 levels for tool-call tags generated by tool parsers).""" def _verify_num_sequences(value: int, parameter_name: str) -> None: if not isinstance(value, int): raise VLLMValidationError( f"{parameter_name} must be an int, but is of type {type(value)}" ) if value < 1: raise VLLMValidationError(f"{parameter_name} must be at least 1, got {value}.") max_n = envs.VLLM_MAX_N_SEQUENCES if value > max_n: raise VLLMValidationError( f"{parameter_name} must be at most {max_n}, got {value}. " "To increase this limit, set the VLLM_MAX_N_SEQUENCES " "environment variable." ) def validate_thinking_token_budget(value: int | float | bool | None) -> int | None: """Validate ``thinking_token_budget``; return ``None`` if unset.""" if value is None: return None if isinstance(value, (bool, float)) or not isinstance(value, int): raise VLLMValidationError( "`thinking_token_budget` must be a non-negative integer " "or -1 for unlimited.", parameter="thinking_token_budget", value=value, ) if value == -1: return None if value < 0: raise VLLMValidationError( "`thinking_token_budget` must be a non-negative integer " "or -1 for unlimited.", parameter="thinking_token_budget", value=value, ) return value ThinkingTokenBudget = Annotated[ int | None, BeforeValidator(validate_thinking_token_budget), ] class SamplingType(IntEnum): GREEDY = 0 RANDOM = 1 RANDOM_SEED = 2 # maybe make msgspec? @dataclass class StructuredOutputsParams: # One of these fields will be used to build a logit processor. json: str | dict | None = None regex: str | None = None choice: list[str] | None = None grammar: str | None = None json_object: bool | None = None # These are other options that can be set. disable_any_whitespace: bool = False disable_additional_properties: bool = False whitespace_pattern: str | None = None structural_tag: str | None = None _backend: str | None = field(default=None, init=False) """CAUTION: Should only be set by Processor._validate_structured_output""" _backend_was_auto: bool = field(default=False, init=False) """CAUTION: Should only be set by Processor._validate_structured_output""" def __post_init__(self): """Validate that some fields are mutually exclusive.""" count = sum( [ self.json is not None, self.regex is not None, self.choice is not None, self.grammar is not None, self.json_object is not None, self.structural_tag is not None, ] ) if count > 1: raise VLLMValidationError( "You can only use one kind of structured outputs constraint " f"but multiple are specified: {self.__dict__}" ) if count < 1: raise VLLMValidationError( "You must use one kind of structured outputs constraint " f"but none are specified: {self.__dict__}" ) def all_constraints_none(self) -> bool: """Returns True if all structured-output constraint fields are None.""" return all( getattr(self, field) is None for field in ( "json", "regex", "choice", "grammar", "json_object", "structural_tag", ) ) def all_non_structural_tag_constraints_none(self) -> bool: """Returns True if all structured-output constraint fields are None.""" return all( getattr(self, field) is None for field in ( "json", "regex", "choice", "grammar", "json_object", ) ) @dataclass class RepetitionDetectionParams: """Parameters for detecting repetitive N-gram patterns in output tokens.""" max_pattern_size: int = 0 """Maximum size of N-gram pattern to detect for sequence repetition. Set to 0 to disable. Must be used together with min_count.""" min_pattern_size: int = 0 """Minimum N-gram pattern size to check for sequence repetition. If set to 0, it defaults to 1. Must be <= max_pattern_size.""" min_count: int = 0 """Minimum number of times an N-gram pattern must repeat to trigger detection. Must be >= 2. Example: 3 for detecting a phrase repeated 3 times. Must be used together with max_pattern_size.""" def __post_init__(self): if ( self.max_pattern_size < 0 or self.min_pattern_size < 0 or self.min_pattern_size > self.max_pattern_size ): raise VLLMValidationError( "max_pattern_size, min_pattern_size must be >=0, " "with min_pattern_size <= max_pattern_size. " "Set both to 0 to disable repetitive pattern detection." ) if self.max_pattern_size > 0 and self.min_count < 2: raise VLLMValidationError( "min_count must be >= 2 to detect repetitive patterns " "in engine output. If you do not wish to detect repetitive " "patterns, set max_pattern_size to 0." ) class RequestOutputKind(Enum): # Return entire output so far in every RequestOutput CUMULATIVE = 0 # Return only deltas in each RequestOutput DELTA = 1 # Do not return intermediate RequestOutput FINAL_ONLY = 2 def _exceeds_json_nesting(spec: str | dict[str, Any], limit: int) -> bool: """Whether a JSON document nests objects and arrays deeper than `limit`.""" if isinstance(spec, str): try: spec = json_mod.loads(spec) except RecursionError: return True except ValueError: # Malformed JSON is reported by the backend validation. return False level: list[Any] = [spec] for _ in range(limit): level = [ child for node in level if isinstance(node, (dict, list)) for child in (node.values() if isinstance(node, dict) else node) if isinstance(child, (dict, list)) ] if not level: return False return True def check_json_nesting( spec: str | dict[str, Any] | None, *, structural_tag: bool = False ) -> None: """Reject a JSON schema nested more than `MAX_STRUCTURED_OUTPUT_JSON_NESTING` levels deep, or a structural tag nested deeper than that plus `STRUCTURAL_TAG_WRAPPER_NESTING`. Raises: VLLMValidationError: If `spec` is nested too deeply. """ limit = MAX_STRUCTURED_OUTPUT_JSON_NESTING if structural_tag: limit += STRUCTURAL_TAG_WRAPPER_NESTING if spec and _exceeds_json_nesting(spec, limit): raise VLLMValidationError( "The JSON schema is nested too deeply: objects and arrays may nest at " f"most {MAX_STRUCTURED_OUTPUT_JSON_NESTING} levels" ) def _is_non_tekken_mistral(tokenizer: TokenizerLike) -> bool: return is_mistral_tokenizer(tokenizer) and not tokenizer.is_tekken def _get_llg_tokenizer(tokenizer: TokenizerLike) -> Any: return tokenizer.llg_tokenizer if is_mistral_tokenizer(tokenizer) else None class SamplingParams( PydanticMsgspecMixin, msgspec.Struct, omit_defaults=True, # type: ignore[call-arg] # required for @cached_property. dict=True, ): # type: ignore[call-arg] """Sampling parameters for text generation. Overall, we follow the sampling parameters from the OpenAI text completion API (https://platform.openai.com/docs/api-reference/completions/create). In addition, we support beam search, which is not supported by OpenAI. """ n: int = 1 """Number of outputs to return for the given prompt request. The maximum allowed value is controlled by the ``VLLM_MAX_N_SEQUENCES`` environment variable (default: 16384). NOTE: `AsyncLLM` streams outputs by default. When `n > 1`, all `n` outputs are generated and streamed cumulatively per request. To see all `n` outputs upon completion, use `output_kind=RequestOutputKind.FINAL_ONLY` in `SamplingParams`.""" presence_penalty: float = 0.0 """Penalizes new tokens based on whether they appear in the generated text so far. Values > 0 encourage the model to use new tokens, while values < 0 encourage the model to repeat tokens.""" frequency_penalty: float = 0.0 """Penalizes new tokens based on their frequency in the generated text so far. Values > 0 encourage the model to use new tokens, while values < 0 encourage the model to repeat tokens.""" repetition_penalty: float = 1.0 """Penalizes new tokens based on whether they appear in the prompt and the generated text so far. Values > 1 encourage the model to use new tokens, while values < 1 encourage the model to repeat tokens.""" temperature: float = 1.0 """Controls the randomness of the sampling. Lower values make the model more deterministic, while higher values make the model more random. Zero means greedy sampling.""" watermarking: bool | None = None """Whether to apply the engine's configured watermark to this request. ``None`` defaults to enabled.""" top_p: float = 1.0 """Controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to 1 to consider all tokens.""" top_k: int = 0 """Controls the number of top tokens to consider. Set to 0 (or -1) to consider all tokens.""" min_p: float = 0.0 """Represents the minimum probability for a token to be considered, relative to the probability of the most likely token. Must be in [0, 1]. Set to 0 to disable this.""" seed: int | None = None """Random seed to use for the generation.""" stop: str | list[str] | None = None """String(s) that stop the generation when they are generated. The returned output will not contain the stop strings.""" stop_token_ids: list[int] | None = None """Token IDs that stop the generation when they are generated. The returned output will contain the stop tokens unless the stop tokens are special tokens.""" ignore_eos: bool = False """Whether to ignore the EOS token and continue generating tokens after the EOS token is generated.""" max_tokens: int | None = 16 """Maximum number of tokens to generate per output sequence.""" min_tokens: int = 0 """Minimum number of tokens to generate per output sequence before EOS or `stop_token_ids` can be generated""" logprobs: int | None = None """Number of log probabilities to return per output token. When set to `None`, no probability is returned. If set to a non-`None` value, the result includes the log probabilities of the specified number of most likely tokens, as well as the chosen tokens. Note that the implementation follows the OpenAI API: The API will always return the log probability of the sampled token, so there may be up to `logprobs+1` elements in the response. When set to -1, return all `vocab_size` log probabilities.""" prompt_logprobs: int | None = None """Number of log probabilities to return per prompt token. When set to -1, return all `vocab_size` log probabilities.""" prompt_logprob_token_ids: ( Annotated[np.ndarray, GetPydanticSchema(lambda _, h: h(list[list[StrictInt]]))] | None ) = None """Token IDs to score per causal prompt row: an integer array of shape [num_rows, num_ids] or nested lists; -1 entries (padding) score -inf. Row i scores its IDs as predictions of prompt token prompt_logprob_start + i + 1, so the last prompt row is excluded.""" prompt_logprob_start: int | None = None """First causal prompt row to score; defaults to the first row.""" logprob_token_ids: list[int] | None = None """Specific token IDs to return logprobs for. More efficient than logprobs=-1 when you only need logprobs for a small set of tokens. When set, logprobs for exactly these token IDs will be returned, in addition to the sampled token. This is useful for scoring tasks where you want to compare probabilities of specific label tokens.""" flat_logprobs: bool = False """Whether to return logprobs in flatten format (i.e. FlatLogprob) for better performance. NOTE: GC costs of FlatLogprobs is significantly smaller than list[dict[int, Logprob]]. After enabled, PromptLogprobs and SampleLogprobs would populated as FlatLogprobs.""" # NOTE: This parameter is only exposed at the engine level for now. # It is not exposed in the OpenAI API server, as the OpenAI API does # not support returning only a list of token IDs. detokenize: bool = True """Whether to detokenize the output.""" skip_special_tokens: bool = True """Whether to skip special tokens in the output.""" spaces_between_special_tokens: bool = True """Whether to add spaces between special tokens in the output.""" include_stop_str_in_output: bool = False """Whether to include the stop strings in output text.""" output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE stream_interval: int | None = None """Number of newly generated tokens to batch into each streamed `RequestOutput`. Raises the interval above the engine-level `--stream-interval`. Values below engine setting are clamped up to it. The first and final outputs are always emitted immediately.""" skip_clone: bool = False """Internal flag indicating that this SamplingParams instance is safe to reuse without cloning. When True, clone() will return self without performing a deep copy. This should only be set when the params object is guaranteed to be dedicated to a single request and won't be modified in ways that would affect other uses.""" # The below fields are not supposed to be used as an input. # They are set in post_init. output_text_buffer_length: int = 0 _eos_token_id: int | None = None _all_stop_token_ids: set[int] = msgspec.field(default_factory=set) _watermarking_skipped: bool = False # Fields used to construct logits processors structured_outputs: StructuredOutputsParams | None = None """Parameters for configuring structured outputs.""" logit_bias: dict[int, float] | None = None """If provided, the engine will construct a logits processor that applies these logit biases.""" allowed_token_ids: list[int] | None = None """If provided, the engine will construct a logits processor which only retains scores for the given token ids.""" extra_args: dict[str, Any] | None = None """Arbitrary additional args, that can be used by custom sampling implementations, plugins, etc. Not used by any in-tree sampling implementations.""" # Fields used for bad words bad_words: list[str] | None = None """Words that are not allowed to be generated. More precisely, only the last token of a corresponding token sequence is not allowed when the next generated token can complete the sequence.""" _bad_words_token_ids: list[list[int]] | None = None skip_reading_prefix_cache: bool | None = None thinking_token_budget: int | None = None """Maximum number of tokens allowed for thinking operations.""" repetition_detection: RepetitionDetectionParams | None = None """Parameters for detecting repetitive N-gram patterns in output tokens. If such repetition is detected, generation will be ended early. LLMs can sometimes generate repetitive, unhelpful token patterns, stopping only when they hit the maximum output length (e.g. 'abcdabcdabcd...' or '\\emoji \\emoji \\emoji ...'). This feature can detect such behavior and terminate early, saving time and tokens.""" # Debugging / RL-specific parameters. Not intended for production serving. routed_experts_prompt_start: int = 0 """When enable_return_routed_experts is active, skip the first routed_experts_prompt_start prompt tokens from the returned routing data. In multi-turn agent scenarios, set this to the length of the already-returned prefix to avoid duplicating routing for prompt tokens covered by earlier turns. Default 0 returns routing for all prompt tokens.""" trace_decode_token_ids: list[int] | None = None """If provided, forces the engine to emit this predetermined sequence of token IDs during decoding instead of sampling randomly. Real logprobs are still computed. Conflict checking is performed at the engine level.""" @staticmethod def from_optional( n: int | None = 1, presence_penalty: float | None = 0.0, frequency_penalty: float | None = 0.0, repetition_penalty: float | None = 1.0, temperature: float | None = 1.0, watermarking: bool | None = None, top_p: float | None = 1.0, top_k: int = 0, min_p: float = 0.0, seed: int | None = None, stop: str | list[str] | None = None, stop_token_ids: list[int] | None = None, bad_words: list[str] | None = None, thinking_token_budget: int | None = None, include_stop_str_in_output: bool = False, ignore_eos: bool = False, max_tokens: int | None = 16, min_tokens: int = 0, logprobs: int | None = None, prompt_logprobs: int | None = None, prompt_logprob_token_ids: np.ndarray | list[list[int]] | None = None, prompt_logprob_start: int | None = None, detokenize: bool = True, skip_special_tokens: bool = True, spaces_between_special_tokens: bool = True, output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE, stream_interval: int | None = None, structured_outputs: StructuredOutputsParams | None = None, logit_bias: dict[int, float] | dict[str, float] | None = None, allowed_token_ids: list[int] | None = None, extra_args: dict[str, Any] | None = None, skip_clone: bool = False, repetition_detection: RepetitionDetectionParams | None = None, logprob_token_ids: list[int] | None = None, routed_experts_prompt_start: int = 0, # Debugging / RL-specific parameters. trace_decode_token_ids: list[int] | None = None, ) -> "SamplingParams": if logit_bias is not None: # Fast path uses a dict comprehension; on failure we iterate once # to identify the exact offending entry for the error message. try: logit_bias = { int(token): min(100.0, max(-100.0, bias)) for token, bias in logit_bias.items() } except (ValueError, TypeError): invalid_keys = [] converted_logit_bias = {} for token, bias in logit_bias.items(): try: token_id = int(token) except (ValueError, TypeError): invalid_keys.append(token) continue converted_logit_bias[token_id] = min(100.0, max(-100.0, bias)) if invalid_keys: raise VLLMValidationError( f"logit_bias contains key(s) that cannot be " f"converted to integer token IDs: {invalid_keys!r}", parameter="logit_bias", value=invalid_keys, ) from None logit_bias = converted_logit_bias return SamplingParams( n=1 if n is None else n, presence_penalty=0.0 if presence_penalty is None else presence_penalty, frequency_penalty=0.0 if frequency_penalty is None else frequency_penalty, repetition_penalty=1.0 if repetition_penalty is None else repetition_penalty, temperature=1.0 if temperature is None else temperature, watermarking=watermarking, top_p=1.0 if top_p is None else top_p, top_k=top_k, min_p=min_p, seed=seed, stop=stop, stop_token_ids=stop_token_ids, bad_words=bad_words, thinking_token_budget=thinking_token_budget, include_stop_str_in_output=include_stop_str_in_output, ignore_eos=ignore_eos, max_tokens=max_tokens, min_tokens=min_tokens, logprobs=logprobs, prompt_logprobs=prompt_logprobs, prompt_logprob_token_ids=prompt_logprob_token_ids, prompt_logprob_start=prompt_logprob_start, logprob_token_ids=logprob_token_ids, detokenize=detokenize, skip_special_tokens=skip_special_tokens, spaces_between_special_tokens=spaces_between_special_tokens, output_kind=output_kind, stream_interval=stream_interval, structured_outputs=structured_outputs, logit_bias=logit_bias, allowed_token_ids=allowed_token_ids, extra_args=extra_args, skip_clone=skip_clone, repetition_detection=repetition_detection, routed_experts_prompt_start=routed_experts_prompt_start, trace_decode_token_ids=trace_decode_token_ids, ) def __post_init__(self) -> None: if 0 < self.temperature < _MAX_TEMP: logger.warning( "temperature %s is less than %s, which may cause numerical " "errors nan or inf in tensors. We have maxed it out to %s.", self.temperature, _MAX_TEMP, _MAX_TEMP, ) self.temperature = max(self.temperature, _MAX_TEMP) if self.seed == -1: self.seed = None self.thinking_token_budget = validate_thinking_token_budget( self.thinking_token_budget ) if self.stop is None: self.stop = [] elif isinstance(self.stop, str): self.stop = [self.stop] if self.stop_token_ids is None: self.stop_token_ids = [] else: self.stop_token_ids = list(dict.fromkeys(self.stop_token_ids)) if self.bad_words is None: self.bad_words = [] else: self.bad_words = list(dict.fromkeys(self.bad_words)) if self.logprobs is True: self.logprobs = 1 if self.prompt_logprobs is True: self.prompt_logprobs = 1 # Number of characters to hold back for stop string evaluation # until sequence is finished. if self.stop and not self.include_stop_str_in_output: self.output_text_buffer_length = max(len(s) for s in self.stop) - 1 self._verify_args() if ( envs.VLLM_BATCH_INVARIANT and self.temperature >= _SAMPLING_EPS and self.seed is None ): logger.warning_once( "Random sampling without an explicit seed may not be batch " "invariant. Set seed in SamplingParams or use temperature=0." ) if self.temperature < _SAMPLING_EPS: # Zero temperature means greedy sampling. self.top_p = 1.0 self.top_k = 0 self.min_p = 0.0 self._verify_greedy_sampling() # eos_token_id is added to this by the engine self._all_stop_token_ids.update(self.stop_token_ids) if self.skip_reading_prefix_cache is None: # If prefix caching is enabled, # the output of prompt logprobs may less than n_prompt_tokens, # we need to skip reading cache at this request. self.skip_reading_prefix_cache = ( self.prompt_logprobs is not None or self.prompt_logprob_token_ids is not None ) def _verify_args(self) -> None: _verify_num_sequences(self.n, "n") if self.extra_args: self._verify_extra_args() if not -2.0 <= self.presence_penalty <= 2.0: raise VLLMValidationError( f"presence_penalty must be in [-2, 2], got {self.presence_penalty}." ) if not -2.0 <= self.frequency_penalty <= 2.0: raise VLLMValidationError( f"frequency_penalty must be in [-2, 2], got {self.frequency_penalty}." ) if not math.isfinite(self.repetition_penalty): raise VLLMValidationError( "repetition_penalty must be a finite number, " f"got {self.repetition_penalty}." ) if self.repetition_penalty <= 0.0: raise VLLMValidationError( "repetition_penalty must be greater than zero, got " f"{self.repetition_penalty}." ) if not math.isfinite(self.temperature): raise VLLMValidationError( f"temperature must be a finite number, got {self.temperature}.", parameter="temperature", value=self.temperature, ) if self.temperature < 0.0: raise VLLMValidationError( f"temperature must be non-negative, got {self.temperature}.", parameter="temperature", value=self.temperature, ) if self.temperature > 2.0: raise VLLMValidationError( f"temperature must be in [0, 2], got {self.temperature}.", parameter="temperature", value=self.temperature, ) if not 0.0 < self.top_p <= 1.0: raise VLLMValidationError( f"top_p must be in (0, 1], got {self.top_p}.", parameter="top_p", value=self.top_p, ) # quietly accept -1 as disabled, but prefer 0 if self.top_k < -1: raise VLLMValidationError( f"top_k must be 0 (disable), or at least 1, got {self.top_k}." ) if not isinstance(self.top_k, int): raise VLLMValidationError( f"top_k must be an integer, got {type(self.top_k).__name__}" ) if not 0.0 <= self.min_p <= 1.0: raise VLLMValidationError(f"min_p must be in [0, 1], got {self.min_p}.") if self.max_tokens is not None and self.max_tokens < 1: raise VLLMValidationError( f"max_tokens must be at least 1, got {self.max_tokens}.", parameter="max_tokens", value=self.max_tokens, ) if self.min_tokens < 0: raise VLLMValidationError( f"min_tokens must be greater than or equal to 0, got {self.min_tokens}." ) if self.max_tokens is not None and self.min_tokens > self.max_tokens: raise VLLMValidationError( f"min_tokens must be less than or equal to " f"max_tokens={self.max_tokens}, got {self.min_tokens}." ) if self.stream_interval is not None and self.stream_interval < 1: raise VLLMValidationError( f"stream_interval must be at least 1, got {self.stream_interval}.", parameter="stream_interval", value=self.stream_interval, ) if self.logprobs is not None and self.logprobs != -1 and self.logprobs < 0: raise VLLMValidationError( f"logprobs must be non-negative or -1, got {self.logprobs}.", parameter="logprobs", value=self.logprobs, ) if ( self.prompt_logprobs is not None and self.prompt_logprobs != -1 and self.prompt_logprobs < 0 ): raise VLLMValidationError( f"prompt_logprobs must be non-negative or -1, got " f"{self.prompt_logprobs}.", parameter="prompt_logprobs", value=self.prompt_logprobs, ) assert isinstance(self.stop_token_ids, list) if not all(isinstance(st_id, int) for st_id in self.stop_token_ids): raise VLLMValidationError( f"stop_token_ids must contain only integers, got {self.stop_token_ids}." ) assert isinstance(self.stop, list) if any(not stop_str for stop_str in self.stop): raise VLLMValidationError("stop cannot contain an empty string.") if self.stop and not self.detokenize: raise VLLMValidationError( "stop strings are only supported when detokenize is True. " "Set detokenize=True to use stop." ) assert isinstance(self.bad_words, list) if any(not bad_word for bad_word in self.bad_words): raise VLLMValidationError( f"bad_words cannot contain an empty string. " f"Got bad_words={self.bad_words}" ) def _verify_extra_args(self) -> None: # JSON accepts arbitrary integers, but the engine's MessagePack # transport only supports signed/unsigned 64-bit integers. pending: list[Any] = [self.extra_args] visited: set[int] = set() while pending: value = pending.pop() if isinstance(value, int) and not -(2**63) <= value < 2**64: raise VLLMValidationError( "extra_args integers must be between -2**63 and 2**64 - 1.", parameter="extra_args", ) if isinstance(value, (dict, list, tuple)) and id(value) not in visited: visited.add(id(value)) if isinstance(value, dict): pending.extend(value.keys()) pending.extend(value.values()) else: pending.extend(value) def _verify_greedy_sampling(self) -> None: if self.n > 1: raise VLLMValidationError( f"n must be 1 when using greedy sampling, got {self.n}." ) def update_from_generation_config( self, generation_config: dict[str, Any], eos_token_id: int | None = None, ) -> None: """Update if there are non-default values from generation_config.""" if not self.ignore_eos: self._eos_token_id = eos_token_id if eos_token_id is not None: # Add the eos token id into the sampling_params to support # min_tokens processing. self._all_stop_token_ids.add(eos_token_id) # Update eos_token_id for generation if (eos_ids := generation_config.get("eos_token_id")) is not None: # it can be either int or list of int eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids) if eos_token_id is not None: # We don't need to include the primary eos_token_id in # stop_token_ids since it's handled separately for stopping # purposes. eos_ids.discard(eos_token_id) if eos_ids: self._all_stop_token_ids.update(eos_ids) if not self.ignore_eos: assert self.stop_token_ids is not None eos_ids.update(self.stop_token_ids) self.stop_token_ids = list(eos_ids) def update_from_tokenizer(self, tokenizer: TokenizerLike) -> None: if not self.bad_words: return self._bad_words_token_ids = [] max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS for bad_word in self.bad_words: # To prohibit words both at the beginning # and in the middle of text # (related to add_prefix_space tokenizer parameter) for add_prefix_space in [False, True]: prefix = " " if add_prefix_space else "" prompt = prefix + bad_word.lstrip() prompt_token_ids = tokenizer.encode( text=prompt, add_special_tokens=False ) if not prompt_token_ids: if not add_prefix_space: raise VLLMValidationError( "bad_words entries must tokenize to at least one token.", parameter="bad_words", value=self.bad_words, ) # The unprefixed form is still enforceable when only the # optional space-prefixed form tokenizes to nothing. continue # If no space at the beginning # or if prefix space produces a new word token if (not add_prefix_space) or ( add_prefix_space and prompt_token_ids[0] != self._bad_words_token_ids[-1][0] and len(prompt_token_ids) == len(self._bad_words_token_ids[-1]) ): self._bad_words_token_ids.append(prompt_token_ids) if len(self._bad_words_token_ids) > max_num_bad_words: raise VLLMValidationError( f"Too many bad words after tokenization: " f"{len(self._bad_words_token_ids)}. " f"The max number is {max_num_bad_words}.", parameter="bad_words", value=self.bad_words, ) invalid_token_ids = [ token_id for bad_words_token_ids in self._bad_words_token_ids for token_id in bad_words_token_ids if token_id < 0 or token_id > tokenizer.max_token_id ] if len(invalid_token_ids) > 0: raise VLLMValidationError( f"The model vocabulary size is {tokenizer.max_token_id + 1}," f" but the following tokens" f" were specified as bad: {invalid_token_ids}." f" All token id values should be integers satisfying:" f" 0 <= token_id <= {tokenizer.max_token_id}.", parameter="bad_words", value=self.bad_words, ) @cached_property def sampling_type(self) -> SamplingType: if self.temperature < _SAMPLING_EPS: return SamplingType.GREEDY if self.seed is not None: return SamplingType.RANDOM_SEED return SamplingType.RANDOM @property def eos_token_id(self) -> int | None: return self._eos_token_id @property def all_stop_token_ids(self) -> set[int]: return self._all_stop_token_ids @property def bad_words_token_ids(self) -> list[list[int]] | None: # For internal use only. Backward compatibility not guaranteed return self._bad_words_token_ids @property def num_logprobs(self) -> int | None: """Number of sample logprobs to return per output token, or `None` if no sample logprobs were requested. Takes `logprob_token_ids` into account: when `logprobs` is unset but `logprob_token_ids` is set, returns `len(logprob_token_ids)`.""" if self.logprobs is not None: return self.logprobs return len(self.logprob_token_ids) if self.logprob_token_ids else None def clone(self) -> "SamplingParams": """If skip_clone is True, uses shallow copy instead of deep copy.""" if self.skip_clone: return copy.copy(self) return copy.deepcopy(self) def verify( self, model_config: ModelConfig, speculative_config: SpeculativeConfig | None, structured_outputs_config: StructuredOutputsConfig | None, tokenizer: TokenizerLike | None, ) -> None: self._validate_logprobs(model_config) self._validate_logit_bias(model_config) self._validate_trace_replay(model_config, speculative_config) self._validate_stop_token_ids(model_config) self._validate_allowed_token_ids(model_config) self._validate_spec_decode(speculative_config) self._validate_diffusion(model_config) self._validate_structured_outputs( model_config, structured_outputs_config, tokenizer ) def _validate_logprobs(self, model_config: ModelConfig) -> None: max_logprobs = model_config.max_logprobs if max_logprobs == -1: max_logprobs = model_config.get_vocab_size() # Validate sample logprobs. if num_logprobs := self.logprobs: if num_logprobs == -1: num_logprobs = model_config.get_vocab_size() if num_logprobs > max_logprobs: raise VLLMValidationError( f"Requested sample logprobs of {num_logprobs}, " f"which is greater than max allowed: {max_logprobs}", parameter="logprobs", value=num_logprobs, ) # Validate logprob_token_ids. if self.logprob_token_ids is not None: n = len(self.logprob_token_ids) if n > MAX_LOGPROB_TOKEN_IDS: raise VLLMValidationError( f"Requested logprob_token_ids of length {n}, " f"which is greater than max allowed: {MAX_LOGPROB_TOKEN_IDS}", parameter="logprob_token_ids", value=n, ) vocab_size = model_config.get_vocab_size() invalid_token_ids = [ token_id for token_id in self.logprob_token_ids if token_id < 0 or token_id >= vocab_size ] if invalid_token_ids: raise VLLMValidationError( f"token_id(s) {invalid_token_ids} in logprob_token_ids " f"contain out-of-vocab token ids. Vocabulary size: " f"{vocab_size}", parameter="logprob_token_ids", value=invalid_token_ids, ) if self.logprobs is not None and self.logprobs != n: raise VLLMValidationError( f"When both logprobs and logprob_token_ids are set, " f"logprobs must equal len(logprob_token_ids). Got " f"logprobs={self.logprobs}, len(logprob_token_ids)={n}.", parameter="logprob_token_ids", value=n, ) # Validate prompt logprobs. if num_prompt_logprobs := self.prompt_logprobs: if num_prompt_logprobs == -1: num_prompt_logprobs = model_config.get_vocab_size() if num_prompt_logprobs > max_logprobs: raise VLLMValidationError( f"Requested prompt logprobs of {num_prompt_logprobs}, " f"which is greater than max allowed: {max_logprobs}", parameter="prompt_logprobs", value=num_prompt_logprobs, ) # Validate prompt_logprob_token_ids. ids = self.prompt_logprob_token_ids if ids is not None: shape: tuple[int, ...] = () if isinstance(ids, list) and all(isinstance(row, list) for row in ids): shape = (len(ids), max(map(len, ids), default=0)) elif isinstance(ids, np.ndarray) and ids.dtype.kind in "iu": shape = ids.shape if len(shape) != 2 or 0 in shape: raise VLLMValidationError( "prompt_logprob_token_ids must be a non-empty integer array " "of shape [num_rows, num_ids].", parameter="prompt_logprob_token_ids", value=getattr(ids, "shape", type(ids).__name__), ) max_rows = model_config.max_model_len - 1 if shape[0] > max_rows: raise VLLMValidationError( f"prompt_logprob_token_ids has {shape[0]} rows, but a prompt " f"has at most {max_rows} scored rows (max_model_len - 1).", parameter="prompt_logprob_token_ids", value=shape[0], ) num_ids = shape[1] if num_ids > max_logprobs: raise VLLMValidationError( f"Requested {num_ids} token ids per row in " f"prompt_logprob_token_ids, which is greater than max allowed: " f"{max_logprobs}. " f"Set max_logprobs (--max-logprobs) to at least {num_ids}.", parameter="prompt_logprob_token_ids", value=num_ids, ) if isinstance(ids, list): # Pad only after the checks above bound the allocation. try: ids = make_ndarray_with_pad(ids, -1, np.int64, max_len=num_ids) except (TypeError, ValueError, OverflowError) as e: raise VLLMValidationError( "prompt_logprob_token_ids must contain integer token ids.", parameter="prompt_logprob_token_ids", ) from e vocab_size = model_config.get_vocab_size() lo, hi = int(ids.min()), int(ids.max()) if lo < -1 or hi >= vocab_size: raise VLLMValidationError( "prompt_logprob_token_ids contain out-of-vocab token ids " f"(-1 pads a row). Vocabulary size: {vocab_size}", parameter="prompt_logprob_token_ids", value=[lo, hi], ) # In-vocab IDs fit int32, like the runner's other token ID buffers. self.prompt_logprob_token_ids = np.ascontiguousarray(ids, dtype=np.int32) if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0: raise VLLMValidationError( "prompt_logprob_start must be non-negative.", parameter="prompt_logprob_start", value=self.prompt_logprob_start, ) elif self.prompt_logprob_start is not None: raise VLLMValidationError( "prompt_logprob_start requires prompt_logprob_token_ids.", parameter="prompt_logprob_start", value=self.prompt_logprob_start, ) def _validate_stop_token_ids(self, model_config: ModelConfig) -> None: """Validate stop_token_ids are within vocabulary range.""" if not self.stop_token_ids: return # stop_token_ids are used as column indices into the logits tensor, # whose width is the model's vocab size (LogitsProcessor is built from # config.vocab_size, InputBatch.vocab_size comes from # model_config.get_vocab_size()), so use the same bound here — like # _validate_logit_bias, which indexes the same tensor. vocab_size = model_config.get_vocab_size() invalid_token_ids = [ token_id for token_id in self.stop_token_ids if token_id < 0 or token_id >= vocab_size ] if invalid_token_ids: raise VLLMValidationError( f"token_id(s) {invalid_token_ids} in stop_token_ids contain " f"out-of-vocab token ids. Vocabulary size: {vocab_size}", parameter="stop_token_ids", value=invalid_token_ids, ) def _validate_logit_bias(self, model_config: ModelConfig) -> None: """Validate logit_bias token IDs are within vocabulary range.""" if not self.logit_bias: return vocab_size = model_config.get_vocab_size() invalid_token_ids = [ token_id for token_id in self.logit_bias if token_id < 0 or token_id >= vocab_size ] if invalid_token_ids: raise VLLMValidationError( f"token_id(s) {invalid_token_ids} in logit_bias contain " f"out-of-vocab token ids. Vocabulary size: {vocab_size}", parameter="logit_bias", value=invalid_token_ids, ) def _validate_trace_replay( self, model_config: ModelConfig, speculative_config: SpeculativeConfig | None, ) -> None: """Validate trace replay request compatibility.""" if self.trace_decode_token_ids is None: return if len(self.trace_decode_token_ids) == 0: raise ValueError("trace_decode_token_ids must be a non-empty list.") if self.n != 1: raise ValueError("trace_decode_token_ids requires n=1.") if not all(isinstance(t, int) and t >= 0 for t in self.trace_decode_token_ids): raise ValueError( "trace_decode_token_ids must contain non-negative integers." ) if self.prompt_logprobs is not None: raise ValueError( "trace_decode_token_ids is not supported with prompt_logprobs." ) if self.prompt_logprob_token_ids is not None: raise ValueError( "trace_decode_token_ids is not supported with prompt_logprob_token_ids." ) if speculative_config is not None: raise ValueError( "trace_decode_token_ids is not supported with speculative decoding." ) if self.structured_outputs is not None: raise ValueError( "trace_decode_token_ids is not supported with structured outputs." ) if self.repetition_detection is not None: raise ValueError( "trace_decode_token_ids is not supported with repetition_detection." ) if self.thinking_token_budget is not None: raise ValueError( "trace_decode_token_ids is not supported with thinking_token_budget." ) if self.bad_words: raise ValueError("trace_decode_token_ids is not supported with bad_words.") vocab_size = model_config.get_vocab_size() invalid_token_ids = [ token_id for token_id in self.trace_decode_token_ids if token_id < 0 or token_id >= vocab_size ] if invalid_token_ids: raise VLLMValidationError( f"token_id(s) {invalid_token_ids} in trace_decode_token_ids " f"contain out-of-vocab token ids. Vocabulary size: {vocab_size}", parameter="trace_decode_token_ids", value=invalid_token_ids, ) def _validate_allowed_token_ids(self, model_config: ModelConfig) -> None: allowed_token_ids = self.allowed_token_ids if allowed_token_ids is None: return if len(allowed_token_ids) == 0: raise VLLMValidationError( "allowed_token_ids is not None and empty!", parameter="allowed_token_ids", value=allowed_token_ids, ) # allowed_token_ids are client-supplied ids used as column indices # into the logits tensor (the mask in InputBatch is sized by # model_config.get_vocab_size(), and the ids are written as # mask[req_index][allowed_token_ids]), so use the same bound here — # like _validate_stop_token_ids and _validate_logit_bias, which # index the same tensor. vocab_size = model_config.get_vocab_size() invalid_token_ids = [ token_id for token_id in allowed_token_ids if token_id < 0 or token_id >= vocab_size ] if invalid_token_ids: raise VLLMValidationError( "allowed_token_ids contains out-of-vocab token id!", parameter="allowed_token_ids", value=invalid_token_ids, ) def _validate_spec_decode( self, speculative_config: SpeculativeConfig | None, ) -> None: if speculative_config is None: return # Some sampling parameters are not yet compatible with spec decoding. if self.min_p > _SAMPLING_EPS or self.logit_bias: raise VLLMValidationError( "The min_p and logit_bias sampling parameters " "are not yet supported with speculative decoding." ) def _validate_diffusion(self, model_config: ModelConfig) -> None: if not model_config.is_diffusion: return # Diffusion models denoise a whole canvas per step with a fixed # temperature schedule, so per-request sampling parameters are not # supported. Penalties are ignored by the sampler with a warning. if ( self.temperature != 1.0 or self.min_p > _SAMPLING_EPS or self.seed is not None or self.min_tokens > 0 or self.logit_bias or self.bad_words or self.allowed_token_ids ): raise VLLMValidationError( "The temperature, min_p, seed, min_tokens, logit_bias, " "bad_words, and allowed_token_ids sampling parameters " "are not yet supported with diffusion models." ) def _validate_structured_outputs( self, model_config: ModelConfig, structured_outputs_config: StructuredOutputsConfig | None, tokenizer: TokenizerLike | None, ) -> None: if structured_outputs_config is None or self.structured_outputs is None: return if model_config.is_diffusion: # Diffusion LLMs denoise a whole canvas of tokens in parallel # rather than sampling left-to-right, which the grammar FSM # requires. Without this check, requests fail mid-generation # with an FSM rejection (HTTP 500). See issue #45436. raise VLLMValidationError( "Structured outputs are not yet supported for diffusion " "language models. Remove the structured output constraint " "(e.g. `response_format`, `structured_outputs`) from the " "request." ) if tokenizer is None: raise VLLMValidationError( "Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501 ) backend = structured_outputs_config.backend if _backend := self.structured_outputs._backend: # Request-level backend selection is not supported. # The values may differ if `params` is reused and was set # to a specific backend based on `auto` behavior in a previous # request. We remember that it was set as a result of `auto` # using the `_backend_was_auto` field set in the params. if backend != _backend and not ( backend == "auto" and self.structured_outputs._backend_was_auto ): raise VLLMValidationError( "Request-level structured output backend selection is not " f"supported. The request specified '{_backend}', but vLLM " f"was initialised with '{backend}'. This error can be " "resolved by removing '_backend' from the request." ) else: self.structured_outputs._backend = backend # Request content validation if ( isinstance(self.structured_outputs.choice, list) and not self.structured_outputs.choice ): # It is invalid for choice to be an empty list raise VLLMValidationError( f"Choice '{self.structured_outputs.choice}' cannot be an empty list" # noqa: E501 ) # Reject empty string grammar early to avoid engine-side crashes if ( isinstance(self.structured_outputs.grammar, str) and self.structured_outputs.grammar.strip() == "" ): raise VLLMValidationError( "structured_outputs.grammar cannot be an empty string" ) # Reject empty string json schema early to avoid engine-side crashes if ( isinstance(self.structured_outputs.json, str) and self.structured_outputs.json.strip() == "" ): raise VLLMValidationError( "structured_outputs.json cannot be an empty string" ) # Reject deep nesting before any backend walks or converts the schema check_json_nesting(self.structured_outputs.json) check_json_nesting(self.structured_outputs.structural_tag, structural_tag=True) # Reject json_object=False early to avoid engine-side crashes if self.structured_outputs.json_object is False: raise VLLMValidationError( "structured_outputs.json_object must be True if set; omit " "structured_outputs to disable structured outputs" ) # Reject a regex containing a NUL byte early, in every backend mode. A # NUL is never meaningful in a regex pattern and is not handled by the # regex-to-grammar conversion. Checked here, before backend selection, # so it is a clean 400 rather than a silent fallback in the default # "auto" mode. if self.structured_outputs.regex and "\x00" in self.structured_outputs.regex: raise VLLMValidationError( "structured_outputs.regex must not contain a NUL character ('\\x00')" ) # Note(arpera): # We do NOT check here structured output regex on emptiness because # empty regex is indeed compiles to a valid grammar as well as # whitespace-only regexps, for instance, regex="\n" or regex=" " # are valid patterns and we MUST process them. if ( isinstance(self.structured_outputs.structural_tag, str) and self.structured_outputs.structural_tag.strip() == "" ): raise VLLMValidationError( "structured_outputs.structural_tag cannot be an empty string" ) from vllm.v1.structured_output.backend_guidance import ( GuidanceBackend, has_guidance_unsupported_json_features, validate_guidance_grammar, ) from vllm.v1.structured_output.backend_lm_format_enforcer import ( validate_structured_output_request_lm_format_enforcer, ) from vllm.v1.structured_output.backend_outlines import ( validate_structured_output_request_outlines, ) from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar from vllm.v1.structured_output.utils import grammar_is_likely_lark if backend.startswith("xgrammar"): # xgrammar with no fallback validate_xgrammar_grammar(self) elif backend.startswith("guidance"): if _is_non_tekken_mistral(tokenizer=tokenizer): raise VLLMValidationError( "Non-tekken Mistral tokenizers are not supported for the 'guidance'" " structured output backend. Please either use a more recent " "Mistral model, the ['xgrammar', 'outlines'] " "backends or tokenizer_mode='hf' instead." ) if not GuidanceBackend.is_tokenizer_supported(tokenizer): raise VLLMValidationError( "The 'guidance' structured output backend only supports fast " "Hugging Face tokenizers and Tekken-based Mistral tokenizers. " "Please use the 'xgrammar' backend or configure a fast " "tokenizer instead." ) # TODO: ideally we would have the LLTokenizer here as Lark syntax # allows <|special_token|> and similar, see # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens # Without tokenizer these are disallowed in grammars. validate_guidance_grammar( self, tokenizer=_get_llg_tokenizer(tokenizer), ) elif backend == "outlines": # outlines backend validate_structured_output_request_outlines(self) elif backend == "lm-format-enforcer": # lm format enforcer backend if is_mistral_tokenizer(tokenizer): raise VLLMValidationError( "Mistral tokenizer is not supported for the 'lm-format-enforcer' " "structured output backend. Please use ['xgrammar', 'outlines'] " "backends or tokenizer_mode='hf' instead." ) validate_structured_output_request_lm_format_enforcer(self) elif ( backend == "auto" and is_mistral_tokenizer(tokenizer) and tokenizer.is_tekken and self.structured_outputs.grammar and grammar_is_likely_lark(self.structured_outputs.grammar) ): # Lark grammars for Tekken Mistral tokenizers, including the ones # the Mistral tool parser generates, go to guidance. llguidance # takes the tokenizer from mistral-common and never lets a regex # match a special token. The xgrammar backend declares no special # tokens for Mistral tokenizers, so a regex like `.` can match # `[TOOL_CALLS]` or `[INST]` as plain text. validate_guidance_grammar(self, tokenizer=_get_llg_tokenizer(tokenizer)) self.structured_outputs._backend = "guidance" self.structured_outputs._backend_was_auto = True else: # NOTE: backend must be "auto" here, because we have # checked supported_backends above. # In this mode, we set opinionated defaults based on what we think # will satisfy the most use cases without having to worry about # this setting. We include fallback behavior here, but not with any # other setting where a specific backend was specified. try: validate_xgrammar_grammar(self) self.structured_outputs._backend = "xgrammar" except VLLMValidationError: # The request either failed validation # or includes some jsonschema feature(s) that # are not supported in xgrammar. skip_guidance = _is_non_tekken_mistral(tokenizer) # Check if schema has features unsupported by guidance so_params = self.structured_outputs if not skip_guidance and so_params.json: if isinstance(so_params.json, str): try: schema = json_mod.loads(so_params.json) except json_mod.JSONDecodeError as e: raise VLLMValidationError( "Invalid JSON grammar specification." ) from e else: schema = so_params.json skip_guidance = has_guidance_unsupported_json_features(schema) if skip_guidance: # Fall back to outlines for non-Tekken Mistral tokenizers or # schemas containing features unsupported by guidance. validate_structured_output_request_outlines(self) self.structured_outputs._backend = "outlines" else: # Fall back to guidance by default. if not GuidanceBackend.is_tokenizer_supported(tokenizer): raise VLLMValidationError( "No compatible structured output backend was found for " "this request. XGrammar rejected the constraint, and " "Guidance does not support this tokenizer. Configure a " "fast tokenizer or select a compatible backend explicitly." ) from None validate_guidance_grammar( self, tokenizer=_get_llg_tokenizer(tokenizer), ) self.structured_outputs._backend = "guidance" # Remember that this backend was set automatically self.structured_outputs._backend_was_auto = True # Run post-init validation. This is also important to ensure subsequent # roundtrip serialization/deserialization won't fail. self.structured_outputs.__post_init__() def __repr__(self) -> str: return ( f"SamplingParams(n={self.n}, " f"presence_penalty={self.presence_penalty}, " f"frequency_penalty={self.frequency_penalty}, " f"repetition_penalty={self.repetition_penalty}, " f"temperature={self.temperature}, " f"watermarking={self.watermarking}, " f"top_p={self.top_p}, " f"top_k={self.top_k}, " f"min_p={self.min_p}, " f"seed={self.seed}, " f"stop={self.stop}, " f"stop_token_ids={self.stop_token_ids}, " f"bad_words={self.bad_words}, " f"thinking_token_budget={self.thinking_token_budget}, " f"include_stop_str_in_output={self.include_stop_str_in_output}, " f"ignore_eos={self.ignore_eos}, " f"max_tokens={self.max_tokens}, " f"min_tokens={self.min_tokens}, " f"logprobs={self.logprobs}, " f"prompt_logprobs={self.prompt_logprobs}, " f"skip_special_tokens={self.skip_special_tokens}, " "spaces_between_special_tokens=" f"{self.spaces_between_special_tokens}, " f"structured_outputs={self.structured_outputs}, " f"extra_args={self.extra_args})" ) @staticmethod def for_sampler_warmup() -> "SamplingParams": """Set parameters to exercise all sampler logic.""" return SamplingParams( temperature=0.9, top_p=0.9, top_k=50, min_p=0.1, frequency_penalty=0.5, presence_penalty=0.5, repetition_penalty=1.2, min_tokens=2, logit_bias={0: -1.0, 1: 0.5}, _bad_words_token_ids=[[0], [1, 2]], logprobs=5, prompt_logprobs=1, watermarking=True, ) class BeamSearchParams( msgspec.Struct, omit_defaults=True, # type: ignore[call-arg] # required for @cached_property. dict=True, ): # type: ignore[call-arg] """Beam search parameters for text generation.""" beam_width: int max_tokens: int ignore_eos: bool = False temperature: float = 0.0 length_penalty: float = 1.0 include_stop_str_in_output: bool = False structured_outputs: StructuredOutputsParams | None = None skip_special_tokens: bool = True watermarking: bool | None = None def __post_init__(self) -> None: _verify_num_sequences(self.beam_width, "beam_width")