fix(security): Add VLLM_MAX_N_SEQUENCES environment variable and enforce limit (#37952)
Signed-off-by: jperezde <jperezde@redhat.com> Signed-off-by: Russell Bryant <rbryant@redhat.com> Co-authored-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
co-authored by
Russell Bryant
parent
497e234d38
commit
b111f8a61f
@@ -86,6 +86,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
|
||||
VLLM_RPC_TIMEOUT: int = 10000 # ms
|
||||
VLLM_HTTP_TIMEOUT_KEEP_ALIVE: int = 5 # seconds
|
||||
VLLM_MAX_N_SEQUENCES: int = 16384
|
||||
VLLM_PLUGINS: list[str] | None = None
|
||||
VLLM_LORA_RESOLVER_CACHE_DIR: str | None = None
|
||||
VLLM_LORA_RESOLVER_HF_REPO_LIST: str | None = None
|
||||
@@ -870,6 +871,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_HTTP_TIMEOUT_KEEP_ALIVE": lambda: int(
|
||||
os.environ.get("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "5")
|
||||
),
|
||||
# Maximum allowed value for the `n` sampling parameter (number of output
|
||||
# sequences per request). Limits resource consumption to prevent
|
||||
# denial-of-service via excessively large fan-out. Default: 16384.
|
||||
"VLLM_MAX_N_SEQUENCES": lambda: int(
|
||||
os.environ.get("VLLM_MAX_N_SEQUENCES", "16384")
|
||||
),
|
||||
# a list of plugin names to load, separated by commas.
|
||||
# if this is not set, it means all plugins will be loaded
|
||||
# if this is set to an empty string, no plugins will be loaded
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Any
|
||||
import msgspec
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ModelConfig, SpeculativeConfig, StructuredOutputsConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
@@ -169,6 +170,9 @@ class SamplingParams(
|
||||
n: int = 1
|
||||
"""Number of outputs to return for the given prompt request.
|
||||
|
||||
The maximum allowed value is controlled by the ``VLLM_MAX_N_SEQUENCES``
|
||||
environment variable (default: 16384).
|
||||
|
||||
NOTE:
|
||||
`AsyncLLM` streams outputs by default. When `n > 1`, all `n` outputs
|
||||
are generated and streamed cumulatively per request. To see all `n`
|
||||
@@ -425,6 +429,13 @@ class SamplingParams(
|
||||
raise ValueError(f"n must be an int, but is of type {type(self.n)}")
|
||||
if self.n < 1:
|
||||
raise ValueError(f"n must be at least 1, got {self.n}.")
|
||||
max_n = envs.VLLM_MAX_N_SEQUENCES
|
||||
if self.n > max_n:
|
||||
raise ValueError(
|
||||
f"n must be at most {max_n}, got {self.n}. "
|
||||
"To increase this limit, set the VLLM_MAX_N_SEQUENCES "
|
||||
"environment variable."
|
||||
)
|
||||
if not -2.0 <= self.presence_penalty <= 2.0:
|
||||
raise ValueError(
|
||||
f"presence_penalty must be in [-2, 2], got {self.presence_penalty}."
|
||||
|
||||
Reference in New Issue
Block a user